Update bgworker to use `fork` instead of `worker_thread` (#1808)

server/jobs/helpers/index.js

@@ -7,8 +7,8 @@ const documentsPath =
     : path.resolve(process.env.STORAGE_DIR, `documents`);
 
 function log(stringContent = '') {
-  if (parentPort) parentPort.postMessage(stringContent);
-  else console.log(`parentPort is undefined. Debug: ${stringContent}`)
+  if (parentPort) parentPort.postMessage(`\x1b[33m[${process.pid}]\x1b[0m: ${stringContent}`); // running as worker
+  else process.send(`\x1b[33m[${process.ppid}:${process.pid}]\x1b[0m: ${stringContent}`); // running as child_process
 }
 
 function conclude() {

server/package.json

@@ -32,6 +32,7 @@
     "@langchain/core": "0.1.61",
     "@langchain/openai": "0.0.28",
     "@langchain/textsplitters": "0.0.0",
+    "@mintplex-labs/bree": "^9.2.5",
     "@pinecone-database/pinecone": "^2.0.1",
     "@prisma/client": "5.3.1",
     "@qdrant/js-client-rest": "^1.9.0",
@@ -40,7 +41,6 @@
     "archiver": "^5.3.1",
     "bcrypt": "^5.1.0",
     "body-parser": "^1.20.2",
-    "bree": "^9.2.3",
     "chalk": "^4",
     "check-disk-space": "^3.4.0",
     "chromadb": "^1.5.2",

server/utils/BackgroundWorkers/index.js

 const path = require("path");
 const Graceful = require("@ladjs/graceful");
-const Bree = require("bree");
+const Bree = require("@mintplex-labs/bree");
 
 class BackgroundService {
   name = "BackgroundWorkerService";
@@ -35,6 +35,7 @@ class BackgroundService {
       jobs: this.jobs(),
       errorHandler: this.onError,
       workerMessageHandler: this.onWorkerMessageHandler,
+      runJobsAs: "process",
     });
     this.graceful = new Graceful({ brees: [this.bree], logger: this.logger });
     this.graceful.listen();
@@ -50,6 +51,7 @@ class BackgroundService {
     this.#log("Service stopped");
   }
 
+  /** @returns {import("@mintplex-labs/bree").Job[]} */
   jobs() {
     return [
       // Job for auto-sync of documents

server/yarn.lock

@@ -688,6 +688,22 @@
     semver "^7.3.5"
     tar "^6.1.11"
 
+"@mintplex-labs/bree@^9.2.5":
+  version "9.2.5"
+  resolved "https://registry.yarnpkg.com/@mintplex-labs/bree/-/bree-9.2.5.tgz#2e4c4120e1dcc680ee1822a788fa010929275b4c"
+  integrity sha512-AdwR1bd3O3bucazeh8Rfdmgauy0CXwqzUKURVwBx+MJkWAUqxiMc6qOaIqg7qcWEqXC4Bm0IzOlDPNLL0EFeRQ==
+  dependencies:
+    "@breejs/later" "^4.2.0"
+    boolean "^3.2.0"
+    combine-errors "^3.0.3"
+    cron-validate "^1.4.5"
+    human-interval "^2.0.1"
+    is-string-and-not-blank "^0.0.2"
+    is-valid-path "^0.1.1"
+    ms "^2.1.3"
+    p-wait-for "3"
+    safe-timers "^1.1.0"
+
 "@nodelib/fs.scandir@2.1.5":
   version "2.1.5"
   resolved "https://registry.yarnpkg.com/@nodelib/fs.scandir/-/fs.scandir-2.1.5.tgz#7619c2eb21b25483f6d167548b4cfd5a7488c3d5"
@@ -1136,9 +1152,9 @@
     "@types/node" "*"
 
 "@types/lodash@^4.14.165":
-  version "4.17.5"
-  resolved "https://registry.yarnpkg.com/@types/lodash/-/lodash-4.17.5.tgz#e6c29b58e66995d57cd170ce3e2a61926d55ee04"
-  integrity sha512-MBIOHVZqVqgfro1euRDWX7OO0fBVUUMrN6Pwm8LQsz8cWhEpihlvR70ENj3f40j58TNxZaWv2ndSkInykNBBJw==
+  version "4.17.6"
+  resolved "https://registry.yarnpkg.com/@types/lodash/-/lodash-4.17.6.tgz#193ced6a40c8006cfc1ca3f4553444fb38f0e543"
+  integrity sha512-OpXEVoCKSS3lQqjx9GGGOapBeuW5eUboYHRlHP9urXPX25IKZ6AnP5ZRxtVf63iieUbsHxLn8NQ5Nlftc6yzAA==
 
 "@types/long@^4.0.1":
   version "4.0.2"
@@ -1762,22 +1778,6 @@ braces@~3.0.2:
   dependencies:
     fill-range "^7.0.1"
 
-bree@^9.2.3:
-  version "9.2.3"
-  resolved "https://registry.yarnpkg.com/bree/-/bree-9.2.3.tgz#8c47402efcc79ed6da31637f84092ef59743d395"
-  integrity sha512-iCVyLVcqql8rFogVX5gzkofdo6OZu8mxe5dUSkAZyaR43UdNfP0DOj3jJk31yogy6lfnRMhGvO5Gj1ypLeInuA==
-  dependencies:
-    "@breejs/later" "^4.2.0"
-    boolean "^3.2.0"
-    combine-errors "^3.0.3"
-    cron-validate "^1.4.5"
-    human-interval "^2.0.1"
-    is-string-and-not-blank "^0.0.2"
-    is-valid-path "^0.1.1"
-    ms "^2.1.3"
-    p-wait-for "3"
-    safe-timers "^1.1.0"
-
 bson@^6.2.0:
   version "6.6.0"
   resolved "https://registry.yarnpkg.com/bson/-/bson-6.6.0.tgz#f225137eb49fe19bee4d87949a0515c05176e2ad"