diff --git a/apps/simple/assemblyai.ts b/apps/simple/assemblyai.ts new file mode 100644 index 0000000000000000000000000000000000000000..697e83073cb904cf27667e234eca1f5b35feed9c --- /dev/null +++ b/apps/simple/assemblyai.ts @@ -0,0 +1,58 @@ +import { program } from "commander"; +import { AudioTranscriptReader, CreateTranscriptParameters } from "../../packages/core/src/readers/AssemblyAI"; +import { stdin as input, stdout as output } from "node:process"; +// readline/promises is still experimental so not in @types/node yet +// @ts-ignore +import readline from "node:readline/promises"; +import { VectorStoreIndex } from "../../packages/core/src/indices"; + +program + .option("-a, --audio-url [string]", "URL or path of the audio file to transcribe") + .option('-i, --transcript-id [string]', "ID of the AssemblyAI transcript") + .action(async (options) => { + if (!process.env.ASSEMBLYAI_API_KEY) { + console.log( + "No ASSEMBLYAI_API_KEY found in environment variables.", + ); + return; + } + + const reader = new AudioTranscriptReader(); + let params: CreateTranscriptParameters | string; + console.log(options) + if (options.audioUrl) { + params = { + audio_url: options.audioUrl + }; + } else if (options.transcriptId) { + params = options.transcriptId; + } + else { + console.log("You must provide either an --audio-url or a --transcript-id"); + return; + } + + const documents = await reader.loadData(params); + console.log(documents); + + // Split text and create embeddings. Store them in a VectorStoreIndex + const index = await VectorStoreIndex.fromDocuments(documents); + + // Create query engine + const queryEngine = index.asQueryEngine(); + + const rl = readline.createInterface({ input, output }); + while (true) { + const query = await rl.question("Ask a question: "); + + if (!query) { + break; + } + + const response = await queryEngine.query(query); + + console.log(response.toString()); + } + }); + +program.parse(); diff --git a/examples/assemblyai.ts b/examples/assemblyai.ts new file mode 100644 index 0000000000000000000000000000000000000000..283b4f308ce4a9112d3a19c4ed8481702e8cb8e2 --- /dev/null +++ b/examples/assemblyai.ts @@ -0,0 +1,58 @@ +import { program } from "commander"; +import { stdin as input, stdout as output } from "node:process"; +// readline/promises is still experimental so not in @types/node yet +// @ts-ignore +import readline from "node:readline/promises"; +import { AudioTranscriptReader, CreateTranscriptParameters } from "../packages/core/src/readers/AssemblyAI"; +import { VectorStoreIndex } from "../packages/core/src/indices"; + +program + .option("-a, --audio-url [string]", "URL or path of the audio file to transcribe") + .option('-i, --transcript-id [string]', "ID of the AssemblyAI transcript") + .action(async (options) => { + if (!process.env.ASSEMBLYAI_API_KEY) { + console.log( + "No ASSEMBLYAI_API_KEY found in environment variables.", + ); + return; + } + + const reader = new AudioTranscriptReader(); + let params: CreateTranscriptParameters | string; + console.log(options) + if (options.audioUrl) { + params = { + audio_url: options.audioUrl + }; + } else if (options.transcriptId) { + params = options.transcriptId; + } + else { + console.log("You must provide either an --audio-url or a --transcript-id"); + return; + } + + const documents = await reader.loadData(params); + console.log(documents); + + // Split text and create embeddings. Store them in a VectorStoreIndex + const index = await VectorStoreIndex.fromDocuments(documents); + + // Create query engine + const queryEngine = index.asQueryEngine(); + + const rl = readline.createInterface({ input, output }); + while (true) { + const query = await rl.question("Ask a question: "); + + if (!query) { + break; + } + + const response = await queryEngine.query(query); + + console.log(response.toString()); + } + }); + +program.parse(); diff --git a/package.json b/package.json index b344807e573312521310524ae2626777b52cdee4..a882f74689a12df77ac52d03347be24aab4dfdbb 100644 --- a/package.json +++ b/package.json @@ -22,7 +22,7 @@ "ts-jest": "^29.1.1", "turbo": "^1.10.16" }, - "packageManager": "pnpm@7.15.0", + "packageManager": "pnpm@8.10.0", "dependencies": { "@changesets/cli": "^2.26.2" }, diff --git a/packages/core/package.json b/packages/core/package.json index 0fa85b45ee69307b1499bcacc6f8d2a68b691ffb..a2f51e97f7baf625862fa5e9771bf8b119b3de21 100644 --- a/packages/core/package.json +++ b/packages/core/package.json @@ -5,6 +5,7 @@ "dependencies": { "@anthropic-ai/sdk": "^0.8.1", "@notionhq/client": "^2.2.13", + "assemblyai": "^3.0.1", "js-tiktoken": "^1.0.7", "lodash": "^4.17.21", "mammoth": "^1.6.0", diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts index f428f99bc933851690a515ba560e5e54e67b3e4f..fa1758bc7b75db86f5f070f442fffa7f103a4cc0 100644 --- a/packages/core/src/index.ts +++ b/packages/core/src/index.ts @@ -15,6 +15,7 @@ export * from "./QueryEngine"; export * from "./QuestionGenerator"; export * from "./readers/base"; export * from "./readers/CSVReader"; +export * from "./readers/AssemblyAI"; export * from "./readers/MarkdownReader"; export * from "./readers/NotionReader"; export * from "./readers/PDFReader"; diff --git a/packages/core/src/readers/AssemblyAI.ts b/packages/core/src/readers/AssemblyAI.ts new file mode 100644 index 0000000000000000000000000000000000000000..6eea143bb0c529cb47b2aa4c9572417b975bdf6a --- /dev/null +++ b/packages/core/src/readers/AssemblyAI.ts @@ -0,0 +1,148 @@ +import { + AssemblyAI, + BaseServiceParams, + CreateTranscriptParameters, + SubtitleFormat, + TranscriptParagraph, + TranscriptSentence, +} from "assemblyai"; +import { Document } from "../Node"; +import { BaseReader } from "./base"; + +type AssemblyAIOptions = Partial<BaseServiceParams>; + +/** + * Base class for AssemblyAI Readers. + */ +abstract class AssemblyAIReader implements BaseReader { + protected client: AssemblyAI; + + /** + * Creates a new AssemblyAI Reader. + * @param assemblyAIOptions The options to configure the AssemblyAI Reader. + * Configure the `assemblyAIOptions.apiKey` with your AssemblyAI API key, or configure it as the `ASSEMBLYAI_API_KEY` environment variable. + */ + constructor(assemblyAIOptions?: AssemblyAIOptions) { + let options = assemblyAIOptions; + if (!options) { + options = {}; + } + if (!options.apiKey) { + options.apiKey = process.env.ASSEMBLYAI_API_KEY; + } + if (!options.apiKey) { + throw new Error("No AssemblyAI API key provided. Pass an `apiKey` option, or configure the `ASSEMBLYAI_API_KEY` environment variable."); + } + + this.client = new AssemblyAI(options as BaseServiceParams); + } + + abstract loadData(...args: any[]): Promise<Document[]>; + + protected async getOrCreateTranscript(params: CreateTranscriptParameters | string) { + if (typeof params === "string") { + return await this.client.transcripts.get(params); + } + else { + return await this.client.transcripts.create(params); + } + } + + protected async getTranscriptId(params: CreateTranscriptParameters | string) { + if (typeof params === "string") { + return params; + } + else { + return (await this.client.transcripts.create(params)).id; + } + } +} + +/** + * Creates and reads the transcript as a document using AssemblyAI. + */ +class AudioTranscriptReader extends AssemblyAIReader { + /** + * Creates or gets a transcript and loads the transcript as a document using AssemblyAI. + * @param params The parameters to create or get the transcript. + * @returns A promise that resolves to a single document containing the transcript text. + */ + async loadData(params: CreateTranscriptParameters | string): Promise<Document[]> { + const transcript = await this.getOrCreateTranscript(params); + return [ + new Document({ text: transcript.text || undefined }), + ]; + } +} + +/** + * Creates a transcript and returns a document for each paragraph. + */ +class AudioTranscriptParagraphsReader extends AssemblyAIReader { + /** + * Creates or gets a transcript, and returns a document for each paragraph. + * @param params The parameters to create or get the transcript. + * @returns A promise that resolves to an array of documents, each containing a paragraph of the transcript. + */ + async loadData(params: CreateTranscriptParameters | string): Promise<Document[]> { + let transcriptId = await this.getTranscriptId(params); + const paragraphsResponse = await this.client.transcripts.paragraphs( + transcriptId + ); + return paragraphsResponse.paragraphs.map((p: TranscriptParagraph) => + new Document({ text: p.text }), + ); + } +} + +/** + * Creates a transcript and returns a document for each sentence. + */ +class AudioTranscriptSentencesReader extends AssemblyAIReader { + /** + * Creates or gets a transcript, and returns a document for each sentence. + * @param params The parameters to create or get the transcript. + * @returns A promise that resolves to an array of documents, each containing a sentence of the transcript. + */ + async loadData(params: CreateTranscriptParameters | string): Promise<Document[]> { + let transcriptId = await this.getTranscriptId(params); + const sentencesResponse = await this.client.transcripts.sentences( + transcriptId + ); + return sentencesResponse.sentences.map((p: TranscriptSentence) => + new Document({ text: p.text }), + ); + } +} + +/** + * Creates a transcript and reads subtitles for the transcript as `srt` or `vtt` format. + */ +class AudioSubtitlesReader extends AssemblyAIReader { + /** + * Creates or gets a transcript and reads subtitles for the transcript as `srt` or `vtt` format. + * @param params The parameters to create or get the transcript. + * @param subtitleFormat The format of the subtitles, either `srt` or `vtt`. + * @returns A promise that resolves a document containing the subtitles as the page content. + */ + async loadData( + params: CreateTranscriptParameters | string, + subtitleFormat: SubtitleFormat = 'srt' + ): Promise<Document[]> { + let transcriptId = await this.getTranscriptId(params); + const subtitles = await this.client.transcripts.subtitles(transcriptId, subtitleFormat); + return [new Document({ text: subtitles })]; + } +} + +export { + AudioTranscriptReader, + AudioTranscriptParagraphsReader, + AudioTranscriptSentencesReader, + AudioSubtitlesReader, +} +export type { + AssemblyAIOptions, + CreateTranscriptParameters, + SubtitleFormat +} diff --git a/packages/eslint-config-custom/index.js b/packages/eslint-config-custom/index.js index a4d327e57c817abdb802dbc0de516b3d87401e4f..d93af2127f28f5a34406af37b548ec6311b3ad50 100644 --- a/packages/eslint-config-custom/index.js +++ b/packages/eslint-config-custom/index.js @@ -9,6 +9,7 @@ module.exports = { "OPENAI_API_KEY", "REPLICATE_API_TOKEN", "ANTHROPIC_API_KEY", + "ASSEMBLYAI_API_KEY", "AZURE_OPENAI_KEY", "AZURE_OPENAI_ENDPOINT", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 1eb9f0151c2b8f0bc463b1bcdf3d6fcfe026a6db..2988f4a7c4cacd47d7bd7e46b72d8608e23c4452 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -18,7 +18,7 @@ importers: devDependencies: '@turbo/gen': specifier: ^1.10.16 - version: 1.10.16(@types/node@18.18.7)(typescript@4.9.5) + version: 1.10.16(@types/node@18.18.7)(typescript@5.2.2) '@types/jest': specifier: ^29.5.6 version: 29.5.6 @@ -39,10 +39,10 @@ importers: version: 3.0.3 prettier-plugin-organize-imports: specifier: ^3.2.3 - version: 3.2.3(prettier@3.0.3)(typescript@4.9.5) + version: 3.2.3(prettier@3.0.3)(typescript@5.2.2) ts-jest: specifier: ^29.1.1 - version: 29.1.1(@babel/core@7.23.0)(jest@29.7.0)(typescript@4.9.5) + version: 29.1.1(@babel/core@7.23.0)(jest@29.7.0)(typescript@5.2.2) turbo: specifier: ^1.10.16 version: 1.10.16 @@ -122,7 +122,7 @@ importers: version: 18.18.7 ts-node: specifier: ^10.9.1 - version: 10.9.1(@types/node@18.18.7)(typescript@4.9.5) + version: 10.9.1(@types/node@18.18.7)(typescript@5.2.2) packages/core: dependencies: @@ -132,6 +132,9 @@ importers: '@notionhq/client': specifier: ^2.2.13 version: 2.2.13 + assemblyai: + specifier: ^3.0.1 + version: 3.0.1 js-tiktoken: specifier: ^1.0.7 version: 1.0.7 @@ -207,7 +210,7 @@ importers: dependencies: eslint-config-next: specifier: ^13.4.1 - version: 13.4.1(eslint@8.52.0)(typescript@4.9.5) + version: 13.4.1(eslint@8.52.0)(typescript@5.2.2) eslint-config-prettier: specifier: ^8.3.0 version: 8.8.0(eslint@8.52.0) @@ -3751,7 +3754,7 @@ packages: resolution: {integrity: sha512-vxhUy4J8lyeyinH7Azl1pdd43GJhZH/tP2weN8TntQblOY+A0XbT8DJk1/oCPuOOyg/Ja757rG0CgHcWC8OfMA==} dev: true - /@turbo/gen@1.10.16(@types/node@18.18.7)(typescript@4.9.5): + /@turbo/gen@1.10.16(@types/node@18.18.7)(typescript@5.2.2): resolution: {integrity: sha512-PzyluADjVuy5OcIi+/aRcD70OElQpRVRDdfZ9fH8G5Fv75lQcNrjd1bBGKmhjSw+g+eTEkXMGnY7s6gsCYjYTQ==} hasBin: true dependencies: @@ -3763,7 +3766,7 @@ packages: minimatch: 9.0.3 node-plop: 0.26.3 proxy-agent: 6.3.1 - ts-node: 10.9.1(@types/node@18.18.7)(typescript@4.9.5) + ts-node: 10.9.1(@types/node@18.18.7)(typescript@5.2.2) update-check: 1.5.4 validate-npm-package-name: 5.0.0 transitivePeerDependencies: @@ -4170,7 +4173,7 @@ packages: dependencies: '@types/yargs-parser': 21.0.1 - /@typescript-eslint/parser@5.59.2(eslint@8.52.0)(typescript@4.9.5): + /@typescript-eslint/parser@5.59.2(eslint@8.52.0)(typescript@5.2.2): resolution: {integrity: sha512-uq0sKyw6ao1iFOZZGk9F8Nro/8+gfB5ezl1cA06SrqbgJAt0SRoFhb9pXaHvkrxUpZaoLxt8KlovHNk8Gp6/HQ==} engines: {node: ^12.22.0 || ^14.17.0 || >=16.0.0} peerDependencies: @@ -4182,10 +4185,10 @@ packages: dependencies: '@typescript-eslint/scope-manager': 5.59.2 '@typescript-eslint/types': 5.59.2 - '@typescript-eslint/typescript-estree': 5.59.2(typescript@4.9.5) + '@typescript-eslint/typescript-estree': 5.59.2(typescript@5.2.2) debug: 4.3.4 eslint: 8.52.0 - typescript: 4.9.5 + typescript: 5.2.2 transitivePeerDependencies: - supports-color dev: false @@ -4203,7 +4206,7 @@ packages: engines: {node: ^12.22.0 || ^14.17.0 || >=16.0.0} dev: false - /@typescript-eslint/typescript-estree@5.59.2(typescript@4.9.5): + /@typescript-eslint/typescript-estree@5.59.2(typescript@5.2.2): resolution: {integrity: sha512-+j4SmbwVmZsQ9jEyBMgpuBD0rKwi9RxRpjX71Brr73RsYnEr3Lt5QZ624Bxphp8HUkSKfqGnPJp1kA5nl0Sh7Q==} engines: {node: ^12.22.0 || ^14.17.0 || >=16.0.0} peerDependencies: @@ -4218,8 +4221,8 @@ packages: globby: 11.1.0 is-glob: 4.0.3 semver: 7.5.4 - tsutils: 3.21.0(typescript@4.9.5) - typescript: 4.9.5 + tsutils: 3.21.0(typescript@5.2.2) + typescript: 5.2.2 transitivePeerDependencies: - supports-color dev: false @@ -4667,6 +4670,15 @@ packages: safer-buffer: 2.1.2 dev: true + /assemblyai@3.0.1: + resolution: {integrity: sha512-7EW3v00o8s3TSweQERJjoNbXMDU8abwYwpE3gYKQhNY4xlcxb2in344rQi6MsMLVxRtLe8HBXGVRUlZkzfxC3w==} + dependencies: + ws: 8.14.2 + transitivePeerDependencies: + - bufferutil + - utf-8-validate + dev: false + /assert@2.1.0: resolution: {integrity: sha512-eLHpSK/Y4nhMJ07gDaAzoX/XAKS8PSaojml3M0DM4JpV1LAi5JOJ/p6H/XWrl8L+DzVEvVCW1z3vWAaB9oTsQw==} dependencies: @@ -6820,7 +6832,7 @@ packages: source-map: 0.6.1 dev: true - /eslint-config-next@13.4.1(eslint@8.52.0)(typescript@4.9.5): + /eslint-config-next@13.4.1(eslint@8.52.0)(typescript@5.2.2): resolution: {integrity: sha512-ajuxjCkW1hvirr0EQZb3/B/bFH52Z7CT89uCtTcICFL9l30i5c8hN4p0LXvTjdOXNPV5fEDcxBgGHgXdzTj1/A==} peerDependencies: eslint: ^7.23.0 || ^8.0.0 @@ -6831,7 +6843,7 @@ packages: dependencies: '@next/eslint-plugin-next': 13.4.1 '@rushstack/eslint-patch': 1.2.0 - '@typescript-eslint/parser': 5.59.2(eslint@8.52.0)(typescript@4.9.5) + '@typescript-eslint/parser': 5.59.2(eslint@8.52.0)(typescript@5.2.2) eslint: 8.52.0 eslint-import-resolver-node: 0.3.7 eslint-import-resolver-typescript: 3.5.5(@typescript-eslint/parser@5.59.2)(eslint-import-resolver-node@0.3.7)(eslint-plugin-import@2.27.5)(eslint@8.52.0) @@ -6839,7 +6851,7 @@ packages: eslint-plugin-jsx-a11y: 6.7.1(eslint@8.52.0) eslint-plugin-react: 7.32.2(eslint@8.52.0) eslint-plugin-react-hooks: 4.6.0(eslint@8.52.0) - typescript: 4.9.5 + typescript: 5.2.2 transitivePeerDependencies: - eslint-import-resolver-webpack - supports-color @@ -6918,7 +6930,7 @@ packages: eslint-import-resolver-webpack: optional: true dependencies: - '@typescript-eslint/parser': 5.59.2(eslint@8.52.0)(typescript@4.9.5) + '@typescript-eslint/parser': 5.59.2(eslint@8.52.0)(typescript@5.2.2) debug: 3.2.7 eslint: 8.52.0 eslint-import-resolver-node: 0.3.7 @@ -6937,7 +6949,7 @@ packages: '@typescript-eslint/parser': optional: true dependencies: - '@typescript-eslint/parser': 5.59.2(eslint@8.52.0)(typescript@4.9.5) + '@typescript-eslint/parser': 5.59.2(eslint@8.52.0)(typescript@5.2.2) array-includes: 3.1.6 array.prototype.flat: 1.3.1 array.prototype.flatmap: 1.3.1 @@ -11308,7 +11320,7 @@ packages: engines: {node: '>=4'} dev: false - /prettier-plugin-organize-imports@3.2.3(prettier@3.0.3)(typescript@4.9.5): + /prettier-plugin-organize-imports@3.2.3(prettier@3.0.3)(typescript@5.2.2): resolution: {integrity: sha512-KFvk8C/zGyvUaE3RvxN2MhCLwzV6OBbFSkwZ2OamCrs9ZY4i5L77jQ/w4UmUr+lqX8qbaqVq6bZZkApn+IgJSg==} peerDependencies: '@volar/vue-language-plugin-pug': ^1.0.4 @@ -11322,7 +11334,7 @@ packages: optional: true dependencies: prettier: 3.0.3 - typescript: 4.9.5 + typescript: 5.2.2 dev: true /prettier@2.8.8: @@ -13267,7 +13279,7 @@ packages: resolution: {integrity: sha512-Y/arvbn+rrz3JCKl9C4kVNfTfSm2/mEp5FSz5EsZSANGPSlQrpRI5M4PKF+mJnE52jOO90PnPSc3Ur3bTQw0gA==} dev: true - /ts-jest@29.1.1(@babel/core@7.23.0)(jest@29.7.0)(typescript@4.9.5): + /ts-jest@29.1.1(@babel/core@7.23.0)(jest@29.7.0)(typescript@5.2.2): resolution: {integrity: sha512-D6xjnnbP17cC85nliwGiL+tpoKN0StpgE0TeOjXQTU6MVCfsB4v7aW05CgQ/1OywGb0x/oy9hHFnN+sczTiRaA==} engines: {node: ^14.15.0 || ^16.10.0 || >=18.0.0} hasBin: true @@ -13297,11 +13309,11 @@ packages: lodash.memoize: 4.1.2 make-error: 1.3.6 semver: 7.5.4 - typescript: 4.9.5 + typescript: 5.2.2 yargs-parser: 21.1.1 dev: true - /ts-node@10.9.1(@types/node@18.18.7)(typescript@4.9.5): + /ts-node@10.9.1(@types/node@18.18.7)(typescript@5.2.2): resolution: {integrity: sha512-NtVysVPkxxrwFGUUxGYhfux8k78pQB3JqYBXlLRZgdGUqTO5wU/UyHop5p70iEbGhB7q5KmiZiU0Y3KlJrScEw==} hasBin: true peerDependencies: @@ -13327,7 +13339,7 @@ packages: create-require: 1.1.1 diff: 4.0.2 make-error: 1.3.6 - typescript: 4.9.5 + typescript: 5.2.2 v8-compile-cache-lib: 3.0.1 yn: 3.1.1 dev: true @@ -13387,14 +13399,14 @@ packages: - ts-node dev: true - /tsutils@3.21.0(typescript@4.9.5): + /tsutils@3.21.0(typescript@5.2.2): resolution: {integrity: sha512-mHKK3iUXL+3UF6xL5k0PEhKRUBKPBCv/+RkEOpjRWxxx27KKRBmmA60A9pgOUvMi8GKhRMPEmjBRPzs2W7O1OA==} engines: {node: '>= 6'} peerDependencies: typescript: '>=2.8.0 || >= 3.2.0-dev || >= 3.3.0-dev || >= 3.4.0-dev || >= 3.5.0-dev || >= 3.6.0-dev || >= 3.6.0-beta || >= 3.7.0-dev || >= 3.7.0-beta' dependencies: tslib: 1.14.1 - typescript: 4.9.5 + typescript: 5.2.2 dev: false /tty-browserify@0.0.1: @@ -13599,7 +13611,6 @@ packages: resolution: {integrity: sha512-mI4WrpHsbCIcwT9cF4FZvr80QUeKvsUsUvKDoR+X/7XHQH98xYD8YHZg7ANtz2GtZt/CBq2QJ0thkGJMHfqc1w==} engines: {node: '>=14.17'} hasBin: true - dev: true /ua-parser-js@1.0.36: resolution: {integrity: sha512-znuyCIXzl8ciS3+y3fHJI/2OhQIXbXw9MWC/o3qwyR+RGppjZHrM27CGFSKCJXi2Kctiz537iOu2KnXs1lMQhw==}