diff --git a/MongoReader.ts b/MongoReader.ts
deleted file mode 100644
index 2e63d33cc53ca5d0f36ec0574570cd1f9bc9f5e3..0000000000000000000000000000000000000000
--- a/MongoReader.ts
+++ /dev/null
@@ -1,44 +0,0 @@
-import {Document} from "../Node";
-import {BaseReader} from "./base";
-import { MongoClient } from "mongodb";
-
-/**
- * Read in from MongoDB
- */
-export class MongoReader implements BaseReader {
- private client:MongoClient;
-
- constructor(client: MongoClient)
- {
- this.client = client;
- }
- async loadData(db_name: string,
- collection_name: string,
- max_docs?: number,
- //TODO: Think about whether we want to pass generic objects in...
- field_names?: object,
- query_dict?: object,
- query_options?: object,
- projection?: object
- ): Promise<Document[]> {
-
- //Make cursor
- const query: object = (query_dict) ? query_dict: {};
- const options: object = (query_options) ? query_options: {};
- const projections: object = (projection) ? projection: {};
- const limit: number = (max_docs) ? max_docs: Infinity;
-
- //Get items from collection using built-in functions
- const cursor: Partial<Document>[] = await this.client.db(db_name).collection(collection_name)
- .find(query, options)
- .limit(limit)
- .project(projections)
- .toArray();
-
- //Aggregate results and return
- const documents: Document[] = [];
- cursor.forEach((element: Partial<Document>)=> {documents.push(new Document(element))});
- return documents;
- }
-}
-
diff --git a/packages/core/package.json b/packages/core/package.json
index 04d39635d74a1956c359807751973d0cae2b7a34..37272fc521ce825582c85404bf68787dddd15143 100644
--- a/packages/core/package.json
+++ b/packages/core/package.json
@@ -6,6 +6,8 @@
"@notionhq/client": "^2.2.12",
"lodash": "^4.17.21",
"notion-md-crawler": "^0.0.2",
+ "md-utils-ts": "^2.0.0",
+ "mongodb": "^6.0.0",
"openai": "^4.3.1",
"papaparse": "^5.4.1",
"pdf-parse": "^1.1.1",
diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml
index 529d71a62dcb367706e86cc9ef521dd51137fdbd..b5ae91f801e2f36e968d517234d4de3aac81b776 100644
--- a/pnpm-lock.yaml
+++ b/pnpm-lock.yaml
@@ -128,6 +128,12 @@ importers:
lodash:
specifier: ^4.17.21
version: 4.17.21
+ md-utils-ts:
+ specifier: ^2.0.0
+ version: 2.0.0
+ mongodb:
+ specifier: ^6.0.0
+ version: 6.0.0
notion-md-crawler:
specifier: ^0.0.2
version: 0.0.2
@@ -3527,6 +3533,12 @@ packages:
resolution: {integrity: sha512-H1rQc1ZOHANWBvPcW+JpGwr+juXSxM8Q8YCkm3GhZd8REu1fHR3z99CErO1p9pkcfcxZnMdIZdIsXkOHY0NilA==}
dev: false
+ /@mongodb-js/saslprep@1.1.0:
+ resolution: {integrity: sha512-Xfijy7HvfzzqiOAhAepF4SGN5e9leLkMvg/OPOF97XemjfVCYN/oWa75wnkc6mltMSTwY+XlbhWgUOJmkFspSw==}
+ dependencies:
+ sparse-bitfield: 3.0.3
+ dev: false
+
/@next/env@13.4.10:
resolution: {integrity: sha512-3G1yD/XKTSLdihyDSa8JEsaWOELY+OWe08o0LUYzfuHp1zHDA8SObQlzKt+v+wrkkPcnPweoLH1ImZeUa0A1NQ==}
dev: true
@@ -4298,6 +4310,17 @@ packages:
resolution: {integrity: sha512-taHQQH/3ZyI3zP8M/puluDEIEvtQHVYcC6y3N8ijFtAd28+Ey/G4sg1u2gB01S8MwybLOKAp9/yCMu/uR5l3Ug==}
dev: true
+ /@types/webidl-conversions@7.0.0:
+ resolution: {integrity: sha512-xTE1E+YF4aWPJJeUzaZI5DRntlkY3+BCVJi0axFptnjGmAoWxkyREIh/XMrfxVLejwQxMCfDXdICo0VLxThrog==}
+ dev: false
+
+ /@types/whatwg-url@8.2.2:
+ resolution: {integrity: sha512-FtQu10RWgn3D9U4aazdwIE2yzphmTJREDqNdODHrbrZmmMqI0vMheC/6NE/J1Yveaj8H+ela+YwWTjq5PGmuhA==}
+ dependencies:
+ '@types/node': 18.17.12
+ '@types/webidl-conversions': 7.0.0
+ dev: false
+
/@types/ws@8.5.5:
resolution: {integrity: sha512-lwhs8hktwxSjf9UaZ9tG5M03PGogvFaH8gUgLNbN9HKIg0dvv6q+gkSuJ8HN4/VbyxkuLzCjlN7GquQ0gUJfIg==}
dependencies:
@@ -5242,6 +5265,11 @@ packages:
node-int64: 0.4.0
dev: true
+ /bson@6.0.0:
+ resolution: {integrity: sha512-FoWvdELfF2wQaUo8S/a1Rh2BDwJEUancDDnzdTpYymJTZjmvRpLWoqRPelKn+XSeh5D4YddWDG66cLtEhGGvcg==}
+ engines: {node: '>=16.20.1'}
+ dev: false
+
/buffer-from@1.1.2:
resolution: {integrity: sha512-E+XQCRwSbaaiChtv6k6Dwgc+bx+Bs6vuKJHHl5kox/BaKbhiXzqQOwK4cO22yElGp2OCmjwVhT3HmxgyPGnJfQ==}
@@ -9827,6 +9855,10 @@ packages:
fs-monkey: 1.0.4
dev: false
+ /memory-pager@1.5.0:
+ resolution: {integrity: sha512-ZS4Bp4r/Zoeq6+NLJpP+0Zzm0pR8whtGPf1XExKLJBAczGMnSi3It14OiNCStjQjM6NU1okjQGSxgEZN8eBYKg==}
+ dev: false
+
/meow@6.1.1:
resolution: {integrity: sha512-3YffViIt2QWgTy6Pale5QpopX/IvU3LPL03jOTqp6pGj3VjesdO/U8CuHMKpnQr4shCNCM5fd5XFFvIIl6JBHg==}
engines: {node: '>=8'}
@@ -9975,6 +10007,45 @@ packages:
minimist: 1.2.8
dev: true
+ /mongodb-connection-string-url@2.6.0:
+ resolution: {integrity: sha512-WvTZlI9ab0QYtTYnuMLgobULWhokRjtC7db9LtcVfJ+Hsnyr5eo6ZtNAt3Ly24XZScGMelOcGtm7lSn0332tPQ==}
+ dependencies:
+ '@types/whatwg-url': 8.2.2
+ whatwg-url: 11.0.0
+ dev: false
+
+ /mongodb@6.0.0:
+ resolution: {integrity: sha512-wUIYesF4DTyDccm0noE5TwGi9ISdXUAi9T2cQ4xPc+EUBZG44bfMVt2ecOG5Ypca7eCz3oRpJm6YI6c7jAnuNw==}
+ engines: {node: '>=16.20.1'}
+ peerDependencies:
+ '@aws-sdk/credential-providers': ^3.188.0
+ '@mongodb-js/zstd': ^1.1.0
+ gcp-metadata: ^5.2.0
+ kerberos: ^2.0.1
+ mongodb-client-encryption: '>=6.0.0 <7'
+ snappy: ^7.2.2
+ socks: ^2.7.1
+ peerDependenciesMeta:
+ '@aws-sdk/credential-providers':
+ optional: true
+ '@mongodb-js/zstd':
+ optional: true
+ gcp-metadata:
+ optional: true
+ kerberos:
+ optional: true
+ mongodb-client-encryption:
+ optional: true
+ snappy:
+ optional: true
+ socks:
+ optional: true
+ dependencies:
+ '@mongodb-js/saslprep': 1.1.0
+ bson: 6.0.0
+ mongodb-connection-string-url: 2.6.0
+ dev: false
+
/mrmime@1.0.1:
resolution: {integrity: sha512-hzzEagAgDyoU1Q6yg5uI+AorQgdvMCur3FcKf7NhMKWsaYg+RnbTyHRa/9IlLF9rf455MOCtcqqrQQ83pPP7Uw==}
engines: {node: '>=10'}
@@ -12477,6 +12548,12 @@ packages:
resolution: {integrity: sha512-q/JSVd1Lptzhf5bkYm4ob4iWPjx0KiRe3sRFBNrVqbJkFaBm5vbbowy1mymoPNLRa52+oadOhJ+K49wsSeSjTA==}
dev: false
+ /sparse-bitfield@3.0.3:
+ resolution: {integrity: sha512-kvzhi7vqKTfkh0PZU+2D2PIllw2ymqJKujUcyPMd9Y75Nv4nPbGJZXNhxsgdQab2BmlDct1YnfQCguEvHr7VsQ==}
+ dependencies:
+ memory-pager: 1.5.0
+ dev: false
+
/spawndamnit@2.0.0:
resolution: {integrity: sha512-j4JKEcncSjFlqIwU5L/rp2N5SIPsdxaRsIv678+TZxZ0SRDJTm8JrxJMjE/XuiEZNEir3S8l0Fa3Ke339WI4qA==}
dependencies:
@@ -13014,6 +13091,13 @@ packages:
punycode: 2.3.0
dev: true
+ /tr46@3.0.0:
+ resolution: {integrity: sha512-l7FvfAHlcmulp8kr+flpQZmVwtu7nfRV7NZujtN0OqES8EL4O4e0qqzL0DC5gAvx/ZC/9lk6rhcUwYvkBnBnYA==}
+ engines: {node: '>=12'}
+ dependencies:
+ punycode: 2.3.0
+ dev: false
+
/tree-kill@1.2.2:
resolution: {integrity: sha512-L0Orpi8qGpRG//Nd+H90vFB+3iHnue1zSSGmNOOCh1GLJ7rUKVwV2HvijphGQS2UmhUZewS9VgvxYIdgr+fG1A==}
hasBin: true
@@ -13774,6 +13858,11 @@ packages:
resolution: {integrity: sha512-YQ+BmxuTgd6UXZW3+ICGfyqRyHXVlD5GtQr5+qjiNW7bF0cqrzX500HVXPBOvgXb5YnzDd+h0zqyv61KUD7+Sg==}
dev: true
+ /webidl-conversions@7.0.0:
+ resolution: {integrity: sha512-VwddBukDzu71offAQR975unBIGqfKZpM+8ZX6ySk8nYhVoo5CYaZyzt3YBvYtRtO+aoGlqxPg/B87NGVZ/fu6g==}
+ engines: {node: '>=12'}
+ dev: false
+
/webpack-bundle-analyzer@4.9.0:
resolution: {integrity: sha512-+bXGmO1LyiNx0i9enBu3H8mv42sj/BJWhZNFwjz92tVnBa9J3JMGo2an2IXlEleoDOPn/Hofl5hr/xCpObUDtw==}
engines: {node: '>= 10.13.0'}
@@ -13936,6 +14025,14 @@ packages:
engines: {node: '>=0.8.0'}
dev: false
+ /whatwg-url@11.0.0:
+ resolution: {integrity: sha512-RKT8HExMpoYx4igMiVMY83lN6UeITKJlBQ+vR/8ZJ8OCdSiN3RwCq+9gH0+Xzj0+5IrM6i4j/6LuvzbZIQgEcQ==}
+ engines: {node: '>=12'}
+ dependencies:
+ tr46: 3.0.0
+ webidl-conversions: 7.0.0
+ dev: false
+
/whatwg-url@5.0.0:
resolution: {integrity: sha512-saE57nupxk6v3HY35+jzBwYa0rKSy0XR8JSxZPwgLr7ys0IBzhGviA1/TUGJLmSVqs8pb9AnvICXEuOHLprYTw==}
dependencies: