diff --git a/.github/workflows/dev-build.yaml b/.github/workflows/dev-build.yaml index 1eaf5b03fd19ab2cff3c1bc3bf10f17f9b210108..62e94e807e3021710fdbf60193f92de92b9adfe7 100644 --- a/.github/workflows/dev-build.yaml +++ b/.github/workflows/dev-build.yaml @@ -6,7 +6,7 @@ concurrency: on: push: - branches: ['agent-builder'] # put your current branch to create a build. Core team only. + branches: ['ocr-parse-pdfs'] # put your current branch to create a build. Core team only. paths-ignore: - '**.md' - 'cloud-deployments/*' diff --git a/collector/package.json b/collector/package.json index 1509645a10481893ba21cc90230a2dfdd110eac8..b67951df321a16fd1907b555f467fc974c7937f8 100644 --- a/collector/package.json +++ b/collector/package.json @@ -19,6 +19,7 @@ "@xenova/transformers": "^2.11.0", "bcrypt": "^5.1.0", "body-parser": "^1.20.2", + "canvas": "^2.11.2", "cors": "^2.8.5", "dotenv": "^16.0.3", "epub2": "^3.0.2", @@ -40,6 +41,7 @@ "puppeteer": "~21.5.2", "sharp": "^0.33.5", "slugify": "^1.6.6", + "tesseract.js": "^6.0.0", "url-pattern": "^1.0.3", "uuid": "^9.0.0", "wavefile": "^11.0.0", @@ -50,4 +52,4 @@ "nodemon": "^2.0.22", "prettier": "^2.4.1" } -} +} \ No newline at end of file diff --git a/collector/processSingleFile/convert/asPDF/index.js b/collector/processSingleFile/convert/asPDF/index.js index e3e42d3bd7039ed9e63b9c7b9705202e6d145bfe..350f217f4fa2322f3c66b7872b39e83213a9e613 100644 --- a/collector/processSingleFile/convert/asPDF/index.js +++ b/collector/processSingleFile/convert/asPDF/index.js @@ -7,6 +7,7 @@ const { const { tokenizeString } = require("../../../utils/tokenizer"); const { default: slugify } = require("slugify"); const PDFLoader = require("./PDFLoader"); +const OCRLoader = require("../../../utils/OCRLoader"); async function asPdf({ fullFilePath = "", filename = "" }) { const pdfLoader = new PDFLoader(fullFilePath, { @@ -15,7 +16,14 @@ async function asPdf({ fullFilePath = "", filename = "" }) { console.log(`-- Working ${filename} --`); const pageContent = []; - const docs = await pdfLoader.load(); + let docs = await pdfLoader.load(); + + if (docs.length === 0) { + console.log( + `[asPDF] No text content found for ${filename}. Will attempt OCR parse.` + ); + docs = await new OCRLoader().ocrPDF(fullFilePath); + } for (const doc of docs) { console.log( @@ -28,7 +36,7 @@ async function asPdf({ fullFilePath = "", filename = "" }) { } if (!pageContent.length) { - console.error(`Resulting text content was empty for ${filename}.`); + console.error(`[asPDF] Resulting text content was empty for ${filename}.`); trashFile(fullFilePath); return { success: false, diff --git a/collector/utils/OCRLoader/CanvasFactory.js b/collector/utils/OCRLoader/CanvasFactory.js new file mode 100644 index 0000000000000000000000000000000000000000..067917e51be4c4616245652153f8d1cb4cb37300 --- /dev/null +++ b/collector/utils/OCRLoader/CanvasFactory.js @@ -0,0 +1,52 @@ +/** + * This is a factory for creating a canvas and context in Node.js + * it is used to create a canvas and context for the PDFLoader for turning the PDF into an image + * so we can later use the image to extract text from the PDF. + */ +class NodeCanvasFactory { + constructor() { + this.CanvasModule = null; + } + + async init() { + this.CanvasModule = await import("canvas"); + this.Image = this.CanvasModule.Image; + } + + /** + * Creates a canvas and context for the PDFLoader + * @param {number} width - The width of the canvas + * @param {number} height - The height of the canvas + * @param {boolean} transparent - Whether the canvas is transparent + * @returns {{canvas: HTMLCanvasElement, context: CanvasRenderingContext2D}} - The canvas and context + */ + create(width, height, transparent = false) { + const canvas = this.CanvasModule.createCanvas(width, height); + const context = canvas.getContext("2d", { alpha: transparent }); + if (transparent) context.clearRect(0, 0, width, height); + return { + canvas, + context, + }; + } + + /** + * Required for the PDFLoader pdfjs interation - do not remove or use directly. + */ + reset(canvasAndContext, width, height) { + canvasAndContext.canvas.width = width; + canvasAndContext.canvas.height = height; + } + + /** + * Required for the PDFLoader pdfjs interation - do not remove or use directly. + */ + destroy(canvasAndContext) { + canvasAndContext.canvas.width = 0; + canvasAndContext.canvas.height = 0; + canvasAndContext.canvas = null; + canvasAndContext.context = null; + } +} + +module.exports = NodeCanvasFactory; diff --git a/collector/utils/OCRLoader/index.js b/collector/utils/OCRLoader/index.js new file mode 100644 index 0000000000000000000000000000000000000000..725033b611acfa43e972302cbabb0ff915e51bb2 --- /dev/null +++ b/collector/utils/OCRLoader/index.js @@ -0,0 +1,190 @@ +const fs = require("fs"); +const os = require("os"); +const path = require("path"); +const NodeCanvasFactory = require("./CanvasFactory"); + +class OCRLoader { + constructor() { + this.cacheDir = path.resolve( + process.env.STORAGE_DIR + ? path.resolve(process.env.STORAGE_DIR, `models`, `tesseract`) + : path.resolve(__dirname, `../../../server/storage/models/tesseract`) + ); + } + + log(text, ...args) { + console.log(`\x1b[36m[OCRLoader]\x1b[0m ${text}`, ...args); + } + + /** + * Loads a PDF file and returns an array of documents. + * This function is reserved to parsing for SCANNED documents - digital documents are not supported in this function + * @returns {Promise<{pageContent: string, metadata: object}[]>} An array of documents with page content and metadata. + */ + async ocrPDF( + filePath, + { maxExecutionTime = 300_000, batchSize = 10, maxWorkers = null } = {} + ) { + if ( + !filePath || + !fs.existsSync(filePath) || + !fs.statSync(filePath).isFile() + ) { + this.log(`File ${filePath} does not exist. Skipping OCR.`); + return []; + } + + const documentTitle = path.basename(filePath); + this.log(`Starting OCR of ${documentTitle}`); + const pdfjs = await import("pdf-parse/lib/pdf.js/v2.0.550/build/pdf.js"); + let buffer = fs.readFileSync(filePath); + const canvasFactory = new NodeCanvasFactory(); + await canvasFactory.init(); + global.Image = canvasFactory.Image; + + const pdfDocument = await pdfjs.getDocument({ + data: new Uint8Array(buffer), + canvasFactory, + }).promise; + buffer = null; + + const documents = []; + const meta = await pdfDocument.getMetadata().catch(() => null); + const metadata = { + source: filePath, + pdf: { + version: "v2.0.550", + info: meta?.info, + metadata: meta?.metadata, + totalPages: pdfDocument.numPages, + }, + }; + + async function getPageAsBuffer(pageNumber, scale = 1) { + let canvas = null; + let context = null; + try { + const page = await pdfDocument.getPage(pageNumber); + const viewport = page.getViewport(scale); + ({ canvas, context } = canvasFactory.create( + viewport.width, + viewport.height + )); + await page.render({ + canvasFactory, + canvasContext: context, + viewport, + }).promise; + return canvas.toBuffer(); + } catch (e) { + this.log(`Error getting page as buffer: ${e.message}`); + return null; + } finally { + canvas = null; + context = null; + } + } + + const { createWorker, OEM } = require("tesseract.js"); + const BATCH_SIZE = batchSize; + const MAX_EXECUTION_TIME = maxExecutionTime; + const NUM_WORKERS = maxWorkers ?? Math.min(os.cpus().length, 4); + const totalPages = pdfDocument.numPages; + const workerPool = await Promise.all( + Array(NUM_WORKERS) + .fill(0) + .map(() => + createWorker("eng", OEM.LSTM_ONLY, { + cachePath: this.cacheDir, + }) + ) + ); + + const startTime = Date.now(); + try { + this.log("Bootstrapping OCR completed successfully!", { + MAX_EXECUTION_TIME_MS: MAX_EXECUTION_TIME, + BATCH_SIZE, + MAX_CONCURRENT_WORKERS: NUM_WORKERS, + TOTAL_PAGES: totalPages, + }); + const timeoutPromise = new Promise((_, reject) => { + setTimeout(() => { + reject( + new Error( + `OCR job took too long to complete (${ + MAX_EXECUTION_TIME / 1000 + } seconds)` + ) + ); + }, MAX_EXECUTION_TIME); + }); + + const processPages = async () => { + for ( + let startPage = 1; + startPage <= totalPages; + startPage += BATCH_SIZE + ) { + const endPage = Math.min(startPage + BATCH_SIZE - 1, totalPages); + const pageNumbers = Array.from( + { length: endPage - startPage + 1 }, + (_, i) => startPage + i + ); + this.log(`Working on pages ${startPage} - ${endPage}`); + + const pageQueue = [...pageNumbers]; + const results = []; + const workerPromises = workerPool.map(async (worker, workerIndex) => { + while (pageQueue.length > 0) { + const pageNum = pageQueue.shift(); + this.log( + `\x1b[34m[Worker ${ + workerIndex + 1 + }]\x1b[0m assigned pg${pageNum}` + ); + const imageBuffer = await getPageAsBuffer(pageNum, 5); + const { data } = await worker.recognize(imageBuffer, {}, "text"); + this.log( + `✅ \x1b[34m[Worker ${ + workerIndex + 1 + }]\x1b[0m completed pg${pageNum}` + ); + results.push({ + pageContent: data.text, + metadata: { + ...metadata, + loc: { pageNumber: pageNum }, + }, + }); + } + }); + + await Promise.all(workerPromises); + documents.push( + ...results.sort( + (a, b) => a.metadata.loc.pageNumber - b.metadata.loc.pageNumber + ) + ); + } + return documents; + }; + + await Promise.race([timeoutPromise, processPages()]); + } catch (e) { + this.log(`Error: ${e.message}`); + } finally { + global.Image = undefined; + await Promise.all(workerPool.map((worker) => worker.terminate())); + } + + this.log(`Completed OCR of ${documentTitle}!`, { + documentsParsed: documents.length, + totalPages: totalPages, + executionTime: `${((Date.now() - startTime) / 1000).toFixed(2)}s`, + }); + return documents; + } +} + +module.exports = OCRLoader; diff --git a/collector/yarn.lock b/collector/yarn.lock index 402fee3ea7e1902b54d4786f07cfb081377852d8..611b395673ac319709a5139b44e2d54b8c7487d8 100644 --- a/collector/yarn.lock +++ b/collector/yarn.lock @@ -280,7 +280,7 @@ "@langchain/core" "~0.1" js-tiktoken "^1.0.11" -"@mapbox/node-pre-gyp@^1.0.11": +"@mapbox/node-pre-gyp@^1.0.0", "@mapbox/node-pre-gyp@^1.0.11": version "1.0.11" resolved "https://registry.yarnpkg.com/@mapbox/node-pre-gyp/-/node-pre-gyp-1.0.11.tgz#417db42b7f5323d79e93b34a6d7a2a12c0df43fa" integrity sha512-Yhlar6v9WQgUp/He7BdgzOz8lqMQ8sU+jkCq7Wx8Myc5YFJLbEe7lgui/V7G1qB1DJykHSGwreceSaD60Y0PUQ== @@ -693,6 +693,11 @@ bluebird@~3.4.0: resolved "https://registry.yarnpkg.com/bluebird/-/bluebird-3.4.7.tgz#f72d760be09b7f76d08ed8fae98b289a8d05fab3" integrity sha512-iD3898SR7sWVRHbiQv+sHUtHnMvC1o3nW5rAcqnq3uOn07DSAppZYUkIGslDz6gXC7HfunPe7YVBgoEJASPcHA== +bmp-js@^0.1.0: + version "0.1.0" + resolved "https://registry.yarnpkg.com/bmp-js/-/bmp-js-0.1.0.tgz#e05a63f796a6c1ff25f4771ec7adadc148c07233" + integrity sha512-vHdS19CnY3hwiNdkaqk93DvjVLfbEcI8mys4UjuWrlX1haDmroo8o4xCzh4wD6DGV6HxRCyauwhHRqMTfERtjw== + body-parser@1.20.2, body-parser@^1.20.2: version "1.20.2" resolved "https://registry.yarnpkg.com/body-parser/-/body-parser-1.20.2.tgz#6feb0e21c4724d06de7ff38da36dad4f57a747fd" @@ -788,6 +793,15 @@ camelcase@6: resolved "https://registry.yarnpkg.com/camelcase/-/camelcase-6.3.0.tgz#5685b95eb209ac9c0c177467778c9c84df58ba9a" integrity sha512-Gmy6FhYlCY7uOElZUSbxo2UCDH8owEk996gkbrpsgGtrJLM3J7jGxl9Ic7Qwwj4ivOE5AWZWRMecDdF7hqGjFA== +canvas@^2.11.2: + version "2.11.2" + resolved "https://registry.yarnpkg.com/canvas/-/canvas-2.11.2.tgz#553d87b1e0228c7ac0fc72887c3adbac4abbd860" + integrity sha512-ItanGBMrmRV7Py2Z+Xhs7cT+FNt5K0vPL4p9EZ/UX/Mu7hFbkxSjKF2KVtPwX7UYWp7dRKnrTvReflgrItJbdw== + dependencies: + "@mapbox/node-pre-gyp" "^1.0.0" + nan "^2.17.0" + simple-get "^3.0.3" + chalk@^2.4.2: version "2.4.2" resolved "https://registry.yarnpkg.com/chalk/-/chalk-2.4.2.tgz#cd42541677a54333cf541a49108c1432b44c9424" @@ -1043,6 +1057,13 @@ decamelize@1.2.0: resolved "https://registry.yarnpkg.com/decamelize/-/decamelize-1.2.0.tgz#f6534d15148269b20352e7bee26f501f9a191290" integrity sha512-z2S+W9X73hAUUki+N+9Za2lBlun89zigOyGrsax+KUQ6wKW4ZoWpEYBkGhQjwAjjDCkWxhY0VKEhk8wzY7F5cA== +decompress-response@^4.2.0: + version "4.2.1" + resolved "https://registry.yarnpkg.com/decompress-response/-/decompress-response-4.2.1.tgz#414023cc7a302da25ce2ec82d0d5238ccafd8986" + integrity sha512-jOSne2qbyE+/r8G1VU+G/82LBs2Fs4LAsTiLSHOCOMZQl2OKZ6i8i4IyHemTe+/yIXOtTcRQMzPcgyhoFlqPkw== + dependencies: + mimic-response "^2.0.0" + decompress-response@^6.0.0: version "6.0.0" resolved "https://registry.yarnpkg.com/decompress-response/-/decompress-response-6.0.0.tgz#ca387612ddb7e104bd16d85aab00d5ecf09c66fc" @@ -1780,6 +1801,11 @@ iconv-lite@0.6.3, iconv-lite@^0.6.3: dependencies: safer-buffer ">= 2.1.2 < 3.0.0" +idb-keyval@^6.2.0: + version "6.2.1" + resolved "https://registry.yarnpkg.com/idb-keyval/-/idb-keyval-6.2.1.tgz#94516d625346d16f56f3b33855da11bfded2db33" + integrity sha512-8Sb3veuYCyrZL+VBt9LJfZjLUPWVvqn8tG28VqYNFCo43KHcKuq+b4EiXGeuaLAQWL2YmyDgMp2aSpH9JHsEQg== + ieee754@^1.1.13, ieee754@^1.2.1: version "1.2.1" resolved "https://registry.yarnpkg.com/ieee754/-/ieee754-1.2.1.tgz#8eb7a10a63fff25d15a57b001586d177d1b0d352" @@ -1903,6 +1929,11 @@ is-stream@^2.0.0: resolved "https://registry.yarnpkg.com/is-stream/-/is-stream-2.0.1.tgz#fac1e3d53b97ad5a9d0ae9cef2389f5810a5c077" integrity sha512-hFoiJiTl63nn+kstHGBtewWSKnQLpyb155KHheA1l39uvtO9nWIop1p3udqPcUd/xbF1VLMO4n7OI6p7RbngDg== +is-url@^1.2.4: + version "1.2.4" + resolved "https://registry.yarnpkg.com/is-url/-/is-url-1.2.4.tgz#04a4df46d28c4cff3d73d01ff06abeb318a1aa52" + integrity sha512-ITvGim8FhRiYe4IQ5uHSkj7pVaPDrCTkNd3yq3cV7iZAcJdHTUMPMEHcqSOy9xZ9qFenQCvi+2wjH9a1nXqHww== + isarray@~1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/isarray/-/isarray-1.0.0.tgz#bb935d48582cba168c06834957a54a3e07124f11" @@ -2276,6 +2307,11 @@ mime@^3.0.0: resolved "https://registry.yarnpkg.com/mime/-/mime-3.0.0.tgz#b374550dca3a0c18443b0c950a6a58f1931cf7a7" integrity sha512-jSCU7/VB1loIWBZe14aEYHU/+1UMEHoaO7qxCOVJOw9GgH72VAWppxNcjU+x9a2k3GSIBXNKxXQFqRvvZ7vr3A== +mimic-response@^2.0.0: + version "2.1.0" + resolved "https://registry.yarnpkg.com/mimic-response/-/mimic-response-2.1.0.tgz#d13763d35f613d09ec37ebb30bac0469c0ee8f43" + integrity sha512-wXqjST+SLt7R009ySCglWBCFpjUygmCIfD790/kVbiGmUgfYGuB14PiTd5DwVxSV4NcYHjzMkoj5LjQZwTQLEA== + mimic-response@^3.1.0: version "3.1.0" resolved "https://registry.yarnpkg.com/mimic-response/-/mimic-response-3.1.0.tgz#2d1d59af9c1b129815accc2c46a022a5ce1fa3c9" @@ -2389,6 +2425,11 @@ mustache@^4.2.0: resolved "https://registry.yarnpkg.com/mustache/-/mustache-4.2.0.tgz#e5892324d60a12ec9c2a73359edca52972bf6f64" integrity sha512-71ippSywq5Yb7/tVYyGbkBggbU8H3u5Rz56fH60jGFgr8uHwxs+aSKeqmluIVzM0m0kB7xQjKS6qPfd0b2ZoqQ== +nan@^2.17.0: + version "2.22.0" + resolved "https://registry.yarnpkg.com/nan/-/nan-2.22.0.tgz#31bc433fc33213c97bad36404bb68063de604de3" + integrity sha512-nbajikzWTMwsW+eSsNm3QwlOs7het9gGJU5dDZzRTQGk03vyBOauxgI4VakDzE0PtsGTmXPsXTbbjVhRwR5mpw== + napi-build-utils@^1.0.1: version "1.0.2" resolved "https://registry.yarnpkg.com/napi-build-utils/-/napi-build-utils-1.0.2.tgz#b1fddc0b2c46e380a0b7a76f984dd47c41a13806" @@ -2431,7 +2472,7 @@ node-ensure@^0.0.0: resolved "https://registry.yarnpkg.com/node-ensure/-/node-ensure-0.0.0.tgz#ecae764150de99861ec5c810fd5d096b183932a7" integrity sha512-DRI60hzo2oKN1ma0ckc6nQWlHU69RH6xN0sjQTjMpChPfTYvKZdcQFfdYK2RWbJcKyUizSIy/l8OTGxMAM1QDw== -node-fetch@^2.6.12, node-fetch@^2.6.7: +node-fetch@^2.6.12, node-fetch@^2.6.7, node-fetch@^2.6.9: version "2.7.0" resolved "https://registry.yarnpkg.com/node-fetch/-/node-fetch-2.7.0.tgz#d0f0fa6e3e2dc1d27efcd8ad99d550bda94d187d" integrity sha512-c4FRfUm/dbcWZ7U+1Wq0AwCyFL+3nt2bEw05wfxSz+DWpWsitgmSgYmy2dQdWyKC1694ELPqMs/YzUSNozLt8A== @@ -2634,6 +2675,11 @@ openapi-types@^12.1.3: resolved "https://registry.yarnpkg.com/openapi-types/-/openapi-types-12.1.3.tgz#471995eb26c4b97b7bd356aacf7b91b73e777dd3" integrity sha512-N4YtSYJqghVu4iek2ZUvcN/0aqH1kRDuNqzcycDxhOUpg7GdvLa2F3DgS6yBNhInhv2r/6I0Flkn7CqL8+nIcw== +opencollective-postinstall@^2.0.3: + version "2.0.3" + resolved "https://registry.yarnpkg.com/opencollective-postinstall/-/opencollective-postinstall-2.0.3.tgz#7a0fff978f6dbfa4d006238fbac98ed4198c3259" + integrity sha512-8AV/sCtuzUeTo8gQK5qDZzARrulB3egtLzFgteqB2tcT4Mw7B8Kt7JcDHmltjz6FOAHsvTevk70gZEbhM4ZS9Q== + option@~0.2.1: version "0.2.4" resolved "https://registry.yarnpkg.com/option/-/option-0.2.4.tgz#fd475cdf98dcabb3cb397a3ba5284feb45edbfe4" @@ -2990,6 +3036,11 @@ readdirp@~3.6.0: dependencies: picomatch "^2.2.1" +regenerator-runtime@^0.13.3: + version "0.13.11" + resolved "https://registry.yarnpkg.com/regenerator-runtime/-/regenerator-runtime-0.13.11.tgz#f6dca3e7ceec20590d07ada785636a90cdca17f9" + integrity sha512-kY1AZVr2Ra+t+piVaJ4gxaFaReZVH40AKNo7UCX6W+dEwBo/2oZJzqfuN1qLq1oL45o56cPaTXELwrTh8Fpggg== + require-directory@^2.1.1: version "2.1.1" resolved "https://registry.yarnpkg.com/require-directory/-/require-directory-2.1.1.tgz#8c64ad5fd30dab1c976e2344ffe7f792a6a6df42" @@ -3204,6 +3255,15 @@ simple-concat@^1.0.0: resolved "https://registry.yarnpkg.com/simple-concat/-/simple-concat-1.0.1.tgz#f46976082ba35c2263f1c8ab5edfe26c41c9552f" integrity sha512-cSFtAPtRhljv69IK0hTVZQ+OfE9nePi/rtJmw5UjHeVyVroEqJXP1sFztKUy1qU+xvz3u/sfYJLa947b7nAN2Q== +simple-get@^3.0.3: + version "3.1.1" + resolved "https://registry.yarnpkg.com/simple-get/-/simple-get-3.1.1.tgz#cc7ba77cfbe761036fbfce3d021af25fc5584d55" + integrity sha512-CQ5LTKGfCpvE1K0n2us+kuMPbk/q0EKl82s4aheV9oXjFEz6W/Y7oQFVJuU6QG77hRT4Ghb5RURteF5vnWjupA== + dependencies: + decompress-response "^4.2.0" + once "^1.3.1" + simple-concat "^1.0.0" + simple-get@^4.0.0, simple-get@^4.0.1: version "4.0.1" resolved "https://registry.yarnpkg.com/simple-get/-/simple-get-4.0.1.tgz#4a39db549287c979d352112fa03fd99fd6bc3543" @@ -3421,6 +3481,26 @@ tar@^6.1.11: mkdirp "^1.0.3" yallist "^4.0.0" +tesseract.js-core@^6.0.0: + version "6.0.0" + resolved "https://registry.yarnpkg.com/tesseract.js-core/-/tesseract.js-core-6.0.0.tgz#6f25da94f70f8e8f02aff47a43be61d49e6f67c3" + integrity sha512-1Qncm/9oKM7xgrQXZXNB+NRh19qiXGhxlrR8EwFbK5SaUbPZnS5OMtP/ghtqfd23hsr1ZvZbZjeuAGcMxd/ooA== + +tesseract.js@^6.0.0: + version "6.0.0" + resolved "https://registry.yarnpkg.com/tesseract.js/-/tesseract.js-6.0.0.tgz#62ff7fffc8833b5810430a4067785e49d5ca8e7f" + integrity sha512-tqYCod1HwJzkeZw1l6XWx+ly2hhisGcBtak9MArhYwDAxL0NgeVhLJcUjqPxZMQtpgtVUzWcpZPryi+hnaQGVw== + dependencies: + bmp-js "^0.1.0" + idb-keyval "^6.2.0" + is-url "^1.2.4" + node-fetch "^2.6.9" + opencollective-postinstall "^2.0.3" + regenerator-runtime "^0.13.3" + tesseract.js-core "^6.0.0" + wasm-feature-detect "^1.2.11" + zlibjs "^0.3.1" + text-hex@1.0.x: version "1.0.0" resolved "https://registry.yarnpkg.com/text-hex/-/text-hex-1.0.0.tgz#69dc9c1b17446ee79a92bf5b884bb4b9127506f5" @@ -3602,6 +3682,11 @@ vary@^1, vary@~1.1.2: resolved "https://registry.yarnpkg.com/vary/-/vary-1.1.2.tgz#2299f02c6ded30d4a5961b0b9f74524a18f634fc" integrity sha512-BNGbWLfd0eUPabhkXUVm0j8uuvREyTh5ovRa/dyow/BqAbZJyC+5fU+IzQOzmAKzYqYRAISoRhdQr3eIZ/PXqg== +wasm-feature-detect@^1.2.11: + version "1.8.0" + resolved "https://registry.yarnpkg.com/wasm-feature-detect/-/wasm-feature-detect-1.8.0.tgz#4e9f55b0a64d801f372fbb0324ed11ad3abd0c78" + integrity sha512-zksaLKM2fVlnB5jQQDqKXXwYHLQUVH9es+5TOOHwGOVJOCeRBCiPjwSg+3tN2AdTCzjgli4jijCH290kXb/zWQ== + wavefile@^11.0.0: version "11.0.0" resolved "https://registry.yarnpkg.com/wavefile/-/wavefile-11.0.0.tgz#9302165874327ff63a704d00b154c753eaa1b8e7" @@ -3766,6 +3851,11 @@ youtubei.js@^9.1.0: tslib "^2.5.0" undici "^5.19.1" +zlibjs@^0.3.1: + version "0.3.1" + resolved "https://registry.yarnpkg.com/zlibjs/-/zlibjs-0.3.1.tgz#50197edb28a1c42ca659cc8b4e6a9ddd6d444554" + integrity sha512-+J9RrgTKOmlxFSDHo0pI1xM6BLVUv+o0ZT9ANtCxGkjIVCCUdx9alUF8Gm+dGLKbkkkidWIHFDZHDMpfITt4+w== + zod-to-json-schema@^3.22.3, zod-to-json-schema@^3.22.5: version "3.23.0" resolved "https://registry.yarnpkg.com/zod-to-json-schema/-/zod-to-json-schema-3.23.0.tgz#4fc60e88d3c709eedbfaae3f92f8a7bf786469f2" diff --git a/server/storage/models/.gitignore b/server/storage/models/.gitignore index 0481af71a8f1d8e08720a74d70160851dd8d7428..037663a35aa5c1d701bb3a5b315eb2c5005c8bcd 100644 --- a/server/storage/models/.gitignore +++ b/server/storage/models/.gitignore @@ -6,4 +6,5 @@ apipie novita mixedbread-ai* gemini -togetherAi \ No newline at end of file +togetherAi +tesseract \ No newline at end of file