diff --git a/.npmrc b/.npmrc
index ded82e2f63f337df34ae03e82d7daabe1852845e..5dc4d03908f43e1ba7168540bf7ca8fbbe0eb45b 100644
--- a/.npmrc
+++ b/.npmrc
@@ -1 +1,2 @@
auto-install-peers = true
+enable-pre-post-scripts = true
diff --git a/packages/core/README.md b/packages/core/README.md
deleted file mode 100644
index bf3d7c1f1cd124f063080e1f60a4d5bbed0a1427..0000000000000000000000000000000000000000
--- a/packages/core/README.md
+++ /dev/null
@@ -1,130 +0,0 @@
-# LlamaIndex.TS
-
-[](https://www.npmjs.com/package/llamaindex)
-[](https://www.npmjs.com/package/llamaindex)
-[](https://www.npmjs.com/package/llamaindex)
-[](https://discord.com/invite/eN6D2HQ4aX)
-
-LlamaIndex is a data framework for your LLM application.
-
-Use your own data with large language models (LLMs, OpenAI ChatGPT and others) in Typescript and Javascript.
-
-Documentation: https://ts.llamaindex.ai/
-
-## What is LlamaIndex.TS?
-
-LlamaIndex.TS aims to be a lightweight, easy to use set of libraries to help you integrate large language models into your applications with your own data.
-
-## Getting started with an example:
-
-LlamaIndex.TS requires Node v18 or higher. You can download it from https://nodejs.org or use https://nvm.sh (our preferred option).
-
-In a new folder:
-
-```bash
-export OPENAI_API_KEY="sk-......" # Replace with your key from https://platform.openai.com/account/api-keys
-pnpm init
-pnpm install typescript
-pnpm exec tsc --init # if needed
-pnpm install llamaindex
-pnpm install @types/node
-```
-
-Create the file example.ts
-
-```ts
-// example.ts
-import fs from "fs/promises";
-import { Document, VectorStoreIndex } from "llamaindex";
-
-async function main() {
- // Load essay from abramov.txt in Node
- const essay = await fs.readFile(
- "node_modules/llamaindex/examples/abramov.txt",
- "utf-8",
- );
-
- // Create Document object with essay
- const document = new Document({ text: essay });
-
- // Split text and create embeddings. Store them in a VectorStoreIndex
- const index = await VectorStoreIndex.fromDocuments([document]);
-
- // Query the index
- const queryEngine = index.asQueryEngine();
- const response = await queryEngine.query(
- "What did the author do in college?",
- );
-
- // Output response
- console.log(response.toString());
-}
-
-main();
-```
-
-Then you can run it using
-
-```bash
-pnpx ts-node example.ts
-```
-
-## Playground
-
-Check out our NextJS playground at https://llama-playground.vercel.app/. The source is available at https://github.com/run-llama/ts-playground
-
-## Core concepts for getting started:
-
-- [Document](/packages/core/src/Node.ts): A document represents a text file, PDF file or other contiguous piece of data.
-
-- [Node](/packages/core/src/Node.ts): The basic data building block. Most commonly, these are parts of the document split into manageable pieces that are small enough to be fed into an embedding model and LLM.
-
-- [Embedding](/packages/core/src/Embedding.ts): Embeddings are sets of floating point numbers which represent the data in a Node. By comparing the similarity of embeddings, we can derive an understanding of the similarity of two pieces of data. One use case is to compare the embedding of a question with the embeddings of our Nodes to see which Nodes may contain the data needed to answer that quesiton.
-
-- [Indices](/packages/core/src/indices/): Indices store the Nodes and the embeddings of those nodes. QueryEngines retrieve Nodes from these Indices using embedding similarity.
-
-- [QueryEngine](/packages/core/src/QueryEngine.ts): Query engines are what generate the query you put in and give you back the result. Query engines generally combine a pre-built prompt with selected Nodes from your Index to give the LLM the context it needs to answer your query.
-
-- [ChatEngine](/packages/core/src/ChatEngine.ts): A ChatEngine helps you build a chatbot that will interact with your Indices.
-
-- [SimplePrompt](/packages/core/src/Prompt.ts): A simple standardized function call definition that takes in inputs and formats them in a template literal. SimplePrompts can be specialized using currying and combined using other SimplePrompt functions.
-
-## Note: NextJS:
-
-If you're using NextJS App Router, you'll need to use the NodeJS runtime (default) and add the following config to your next.config.js to have it use imports/exports in the same way Node does.
-
-```js
-export const runtime = "nodejs"; // default
-```
-
-```js
-// next.config.js
-/** @type {import('next').NextConfig} */
-const nextConfig = {
- webpack: (config) => {
- config.resolve.alias = {
- ...config.resolve.alias,
- sharp$: false,
- "onnxruntime-node$": false,
- };
- return config;
- },
-};
-
-module.exports = nextConfig;
-```
-
-## Supported LLMs:
-
-- OpenAI GPT-3.5-turbo and GPT-4
-- Anthropic Claude Instant and Claude 2
-- Llama2 Chat LLMs (70B, 13B, and 7B parameters)
-- MistralAI Chat LLMs
-
-## Contributing:
-
-We are in the very early days of LlamaIndex.TS. If you’re interested in hacking on it with us check out our [contributing guide](/CONTRIBUTING.md)
-
-## Bugs? Questions?
-
-Please join our Discord! https://discord.com/invite/eN6D2HQ4aX
diff --git a/packages/core/package.json b/packages/core/package.json
index 209fcf2b87e8079f2b7a68a9441df3120ac5d264..6409117adacaf40f8603b769b28eff4f83683605 100644
--- a/packages/core/package.json
+++ b/packages/core/package.json
@@ -1,5 +1,6 @@
{
"name": "llamaindex",
+ "private": true,
"version": "0.1.0",
"license": "MIT",
"dependencies": {
@@ -33,12 +34,14 @@
},
"devDependencies": {
"@aws-crypto/sha256-js": "^5.2.0",
+ "@types/edit-json-file": "^1.7.3",
"@types/jest": "^29.5.11",
"@types/lodash": "^4.14.202",
"@types/node": "^18.19.6",
"@types/papaparse": "^5.3.14",
"@types/pg": "^8.10.9",
"bunchee": "^4.4.1",
+ "edit-json-file": "^1.8.0",
"madge": "^6.1.0",
"typescript": "^5.3.3"
},
@@ -47,12 +50,22 @@
},
"types": "./dist/index.d.ts",
"main": "./dist/index.js",
+ "exports": {
+ ".": {
+ "types": "./dist/index.d.mts",
+ "import": "./dist/index.mjs",
+ "edge-light": "./dist/index.mjs",
+ "require": "./dist/index.js"
+ },
+ "./env": {
+ "types": "./dist/env.d.mts",
+ "import": "./dist/env.mjs",
+ "edge-light": "./dist/env.mjs",
+ "require": "./dist/env.js"
+ }
+ },
"files": [
- "dist",
- "examples",
- "src",
- "types",
- "CHANGELOG.md"
+ "**"
],
"repository": {
"type": "git",
@@ -63,6 +76,10 @@
"lint": "eslint .",
"test": "jest",
"build": "bunchee",
+ "postbuild": "pnpm run copy && pnpm run modify-package-json",
+ "copy": "cp -r package.json CHANGELOG.md ../../README.md ../../LICENSE examples src dist/",
+ "modify-package-json": "node ./scripts/modify-package-json.mjs",
+ "prepublish": "echo \"please cd ./dist and run pnpm publish\" && exit 1",
"dev": "bunchee -w",
"circular-check": "madge --circular ./src/*.ts"
}
diff --git a/packages/core/scripts/modify-package-json.mjs b/packages/core/scripts/modify-package-json.mjs
new file mode 100755
index 0000000000000000000000000000000000000000..88a4ad238056d41aeeaa0a43e0a2cdedbcc358ad
--- /dev/null
+++ b/packages/core/scripts/modify-package-json.mjs
@@ -0,0 +1,13 @@
+#!/usr/bin/env node
+/**
+ * This script is used to modify the package.json file in the dist folder
+ * so that it can be published to npm.
+ */
+import editJsonFile from "edit-json-file";
+import fs from "node:fs/promises";
+await fs.copyFile("./package.json", "./dist/package.json");
+const file = editJsonFile("./dist/package.json");
+
+file.unset("scripts");
+file.unset("private");
+await new Promise((resolve) => file.save(resolve));
diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml
index 0b47ca72608eee3f2b0ea92e108e6cf90398d4d5..e95236846238d16a6f28908872086585d237a602 100644
--- a/pnpm-lock.yaml
+++ b/pnpm-lock.yaml
@@ -17,7 +17,7 @@ importers:
version: 2.27.1
'@turbo/gen':
specifier: ^1.11.3
- version: 1.11.3(@types/node@20.11.6)(typescript@5.3.3)
+ version: 1.11.3(@types/node@18.19.6)(typescript@5.3.3)
'@types/jest':
specifier: ^29.5.11
version: 29.5.11
@@ -32,7 +32,7 @@ importers:
version: 8.0.3
jest:
specifier: ^29.7.0
- version: 29.7.0(@types/node@20.11.6)
+ version: 29.7.0(@types/node@18.19.6)
lint-staged:
specifier: ^15.2.0
version: 15.2.0
@@ -234,6 +234,9 @@ importers:
'@aws-crypto/sha256-js':
specifier: ^5.2.0
version: 5.2.0
+ '@types/edit-json-file':
+ specifier: ^1.7.3
+ version: 1.7.3
'@types/jest':
specifier: ^29.5.11
version: 29.5.11
@@ -252,6 +255,9 @@ importers:
bunchee:
specifier: ^4.4.1
version: 4.4.1(typescript@5.3.3)
+ edit-json-file:
+ specifier: ^1.8.0
+ version: 1.8.0
madge:
specifier: ^6.1.0
version: 6.1.0(typescript@5.3.3)
@@ -4384,7 +4390,7 @@ packages:
resolution: {integrity: sha512-vxhUy4J8lyeyinH7Azl1pdd43GJhZH/tP2weN8TntQblOY+A0XbT8DJk1/oCPuOOyg/Ja757rG0CgHcWC8OfMA==}
dev: true
- /@turbo/gen@1.11.3(@types/node@20.11.6)(typescript@5.3.3):
+ /@turbo/gen@1.11.3(@types/node@18.19.6)(typescript@5.3.3):
resolution: {integrity: sha512-cHGRj7Jn7Hw1cA7NuwWYfYdhEliQX4LuSfEB9L1m8ifGkHalU3bbYXcehzLThmckpGpUQGnXYx0UtVudbQ42HA==}
hasBin: true
dependencies:
@@ -4396,7 +4402,7 @@ packages:
minimatch: 9.0.3
node-plop: 0.26.3
proxy-agent: 6.3.1
- ts-node: 10.9.2(@types/node@20.11.6)(typescript@5.3.3)
+ ts-node: 10.9.2(@types/node@18.19.6)(typescript@5.3.3)
update-check: 1.5.4
validate-npm-package-name: 5.0.0
transitivePeerDependencies:
@@ -4511,6 +4517,13 @@ packages:
dependencies:
'@types/ms': 0.7.34
+ /@types/edit-json-file@1.7.3:
+ resolution: {integrity: sha512-88OYHTiVq7PNN50T+CIm+65Sl0aRweoXvTuTz4JhoQfy1FeK+wTCC9Peu1dljOqslRaqqSFlDee3wgkWMpxKHg==}
+ dependencies:
+ '@types/node': 18.19.6
+ '@types/set-value': 4.0.3
+ dev: true
+
/@types/eslint-scope@3.7.7:
resolution: {integrity: sha512-MzMFlSLBqNF2gcHWO0G1vP/YQyfvrxZ0bF+u7mzUdZ1/xK4A4sru+nraZz5i3iEIk1l1uyicaDVTB4QbbEkAYg==}
dependencies:
@@ -4817,6 +4830,10 @@ packages:
'@types/mime': 3.0.2
'@types/node': 18.19.6
+ /@types/set-value@4.0.3:
+ resolution: {integrity: sha512-tSuUcLl6kMzI+l0gG7FZ04xbIcynxNIYgWFj91LPAvRcn7W3L1EveXNdVjqFDgAZPjY1qCOsm8Sb1C70SxAPHw==}
+ dev: true
+
/@types/sockjs@0.3.34:
resolution: {integrity: sha512-R+n7qBFnm/6jinlteC9DBL5dGiDGjWAvjo4viUanpnc/dG1y7uDoacXPIQ/PQEg1fI912SMHIa014ZjRpvDw4g==}
dependencies:
@@ -6654,7 +6671,7 @@ packages:
path-type: 4.0.0
typescript: 5.3.3
- /create-jest@29.7.0(@types/node@20.11.6):
+ /create-jest@29.7.0(@types/node@18.19.6):
resolution: {integrity: sha512-Adz2bdH0Vq3F53KEMJOoftQFutWCukm6J24wbPWRO4k1kMY7gS7ds/uoJkNuV8wDCtWWnuwGcJwpWcih+zEW1Q==}
engines: {node: ^14.15.0 || ^16.10.0 || >=18.0.0}
hasBin: true
@@ -6663,7 +6680,7 @@ packages:
chalk: 4.1.2
exit: 0.1.2
graceful-fs: 4.2.11
- jest-config: 29.7.0(@types/node@20.11.6)
+ jest-config: 29.7.0(@types/node@18.19.6)
jest-util: 29.7.0
prompts: 2.4.2
transitivePeerDependencies:
@@ -7533,6 +7550,16 @@ packages:
/eastasianwidth@0.2.0:
resolution: {integrity: sha512-I88TYZWc9XiYHRQ4/3c5rjjfgkjhLyW2luGIheGERbNQ6OY7yTybanSpDXZa8y7VUP9YmDcYa+eyq4ca7iLqWA==}
+ /edit-json-file@1.8.0:
+ resolution: {integrity: sha512-IBOpbe2aQufNl5oZ4jsr2AmNVUy5bO7jS5hk0cCyWhOLdH59Xv41B3XQObE/JB89Ae5qDY9hVsq13/hgGhFBZg==}
+ dependencies:
+ find-value: 1.0.12
+ iterate-object: 1.3.4
+ r-json: 1.3.0
+ set-value: 4.1.0
+ w-json: 1.3.10
+ dev: true
+
/ee-first@1.1.1:
resolution: {integrity: sha512-WMwm9LhRUo+WUaRN+vRuETqG89IgZphVSNkdFgeb6sS/E4OrDIN7t48CAewSHXc6C8lefD8KKfr5vY61brQlow==}
@@ -8523,6 +8550,10 @@ packages:
locate-path: 7.2.0
path-exists: 5.0.0
+ /find-value@1.0.12:
+ resolution: {integrity: sha512-OCpo8LTk8eZ2sdDCwbU2Lc3ivYsdM6yod6jP2jHcNEFcjPhkgH0+POzTIol7xx1LZgtbI5rkO5jqxsG5MWtPjQ==}
+ dev: true
+
/find-yarn-workspace-root2@1.2.16:
resolution: {integrity: sha512-hr6hb1w8ePMpPVUK39S4RlwJzi+xPLuVuG8XlwXU3KD5Yn3qgBWVfy3AzNlDhWvE1EORCE65/Qm26rFQt3VLVA==}
dependencies:
@@ -9976,6 +10007,11 @@ packages:
resolution: {integrity: sha512-VRSzKkbMm5jMDoKLbltAkFQ5Qr7VDiTFGXxYFXXowVj387GeGNOCsOH6Msy00SGZ3Fp84b1Naa1psqgcCIEP5Q==}
engines: {node: '>=0.10.0'}
+ /is-primitive@3.0.1:
+ resolution: {integrity: sha512-GljRxhWvlCNRfZyORiH77FwdFwGcMO620o37EOYC0ORWdq+WYNVqW0w2Juzew4M+L81l6/QS3t5gkkihyRqv9w==}
+ engines: {node: '>=0.10.0'}
+ dev: true
+
/is-reference@1.2.1:
resolution: {integrity: sha512-U82MsXXiFIrjCK4otLT+o2NA2Cd2g5MLoOVXUZjIOhLurrRxpEXzI8O0KZHr3IjLvlAH1kTPYSuqer5T9ZVBKQ==}
dependencies:
@@ -10196,6 +10232,10 @@ packages:
istanbul-lib-report: 3.0.1
dev: true
+ /iterate-object@1.3.4:
+ resolution: {integrity: sha512-4dG1D1x/7g8PwHS9aK6QV5V94+ZvyP4+d19qDv43EzImmrndysIl4prmJ1hWWIGCqrZHyaHBm6BSEWHOLnpoNw==}
+ dev: true
+
/jackspeak@2.3.6:
resolution: {integrity: sha512-N3yCS/NegsOBokc8GAdM8UcmfsKiSS8cipheD/nivzr700H+nsMOxJjQnvwOcRYVuFkdH0wGUvW2WbXGmrZGbQ==}
engines: {node: '>=14'}
@@ -10243,7 +10283,7 @@ packages:
- supports-color
dev: true
- /jest-cli@29.7.0(@types/node@20.11.6):
+ /jest-cli@29.7.0(@types/node@18.19.6):
resolution: {integrity: sha512-OVVobw2IubN/GSYsxETi+gOe7Ka59EFMR/twOU3Jb2GnKKeMGJB5SGUUrEz3SFVmJASUdZUzy83sLNNQ2gZslg==}
engines: {node: ^14.15.0 || ^16.10.0 || >=18.0.0}
hasBin: true
@@ -10257,10 +10297,10 @@ packages:
'@jest/test-result': 29.7.0
'@jest/types': 29.6.3
chalk: 4.1.2
- create-jest: 29.7.0(@types/node@20.11.6)
+ create-jest: 29.7.0(@types/node@18.19.6)
exit: 0.1.2
import-local: 3.1.0
- jest-config: 29.7.0(@types/node@20.11.6)
+ jest-config: 29.7.0(@types/node@18.19.6)
jest-util: 29.7.0
jest-validate: 29.7.0
yargs: 17.7.2
@@ -10271,6 +10311,46 @@ packages:
- ts-node
dev: true
+ /jest-config@29.7.0(@types/node@18.19.6):
+ resolution: {integrity: sha512-uXbpfeQ7R6TZBqI3/TxCU4q4ttk3u0PJeC+E0zbfSoSjq6bJ7buBPxzQPL0ifrkY4DNu4JUdk0ImlBUYi840eQ==}
+ engines: {node: ^14.15.0 || ^16.10.0 || >=18.0.0}
+ peerDependencies:
+ '@types/node': '*'
+ ts-node: '>=9.0.0'
+ peerDependenciesMeta:
+ '@types/node':
+ optional: true
+ ts-node:
+ optional: true
+ dependencies:
+ '@babel/core': 7.23.9
+ '@jest/test-sequencer': 29.7.0
+ '@jest/types': 29.6.3
+ '@types/node': 18.19.6
+ babel-jest: 29.7.0(@babel/core@7.23.9)
+ chalk: 4.1.2
+ ci-info: 3.9.0
+ deepmerge: 4.3.1
+ glob: 7.2.3
+ graceful-fs: 4.2.11
+ jest-circus: 29.7.0
+ jest-environment-node: 29.7.0
+ jest-get-type: 29.6.3
+ jest-regex-util: 29.6.3
+ jest-resolve: 29.7.0
+ jest-runner: 29.7.0
+ jest-util: 29.7.0
+ jest-validate: 29.7.0
+ micromatch: 4.0.5
+ parse-json: 5.2.0
+ pretty-format: 29.7.0
+ slash: 3.0.0
+ strip-json-comments: 3.1.1
+ transitivePeerDependencies:
+ - babel-plugin-macros
+ - supports-color
+ dev: true
+
/jest-config@29.7.0(@types/node@20.11.6):
resolution: {integrity: sha512-uXbpfeQ7R6TZBqI3/TxCU4q4ttk3u0PJeC+E0zbfSoSjq6bJ7buBPxzQPL0ifrkY4DNu4JUdk0ImlBUYi840eQ==}
engines: {node: ^14.15.0 || ^16.10.0 || >=18.0.0}
@@ -10600,7 +10680,7 @@ packages:
merge-stream: 2.0.0
supports-color: 8.1.1
- /jest@29.7.0(@types/node@20.11.6):
+ /jest@29.7.0(@types/node@18.19.6):
resolution: {integrity: sha512-NIy3oAFp9shda19hy4HK0HRTWKtPJmGdnvywu01nOqNC2vZg+Z+fvJDxpMQA88eb2I9EcafcdjYgsDthnYTvGw==}
engines: {node: ^14.15.0 || ^16.10.0 || >=18.0.0}
hasBin: true
@@ -10613,7 +10693,7 @@ packages:
'@jest/core': 29.7.0
'@jest/types': 29.6.3
import-local: 3.1.0
- jest-cli: 29.7.0(@types/node@20.11.6)
+ jest-cli: 29.7.0(@types/node@18.19.6)
transitivePeerDependencies:
- '@types/node'
- babel-plugin-macros
@@ -13715,6 +13795,12 @@ packages:
resolution: {integrity: sha512-twwRO/ilhlG/FIgYeKGFqyHhoEhqgnKVkcmqMKi2r524gz3ZbDTcyFt38E9xjJI2vT+KbRNHVbnJ/e0I25Azwg==}
dev: true
+ /r-json@1.3.0:
+ resolution: {integrity: sha512-xesd+RHCpymPCYd9DvDvUr1w1IieSChkqYF1EpuAYrvCfLXji9NP36DvyYZJZZB5soVDvZ0WUtBoZaU1g5Yt9A==}
+ dependencies:
+ w-json: 1.3.10
+ dev: true
+
/rake-modified@1.0.8:
resolution: {integrity: sha512-rj/1t+EyI8Ly52eaCeSy5hoNpdNnDlNQ/+jll2DypR6nkuxotMbaupzwbuMSaXzuSL1I2pYVYy7oPus/Ls49ag==}
dependencies:
@@ -14690,6 +14776,14 @@ packages:
has-property-descriptors: 1.0.1
dev: true
+ /set-value@4.1.0:
+ resolution: {integrity: sha512-zTEg4HL0RwVrqcWs3ztF+x1vkxfm0lP+MQQFPiMJTKVceBwEV0A569Ou8l9IYQG8jOZdMVI1hGsc0tmeD2o/Lw==}
+ engines: {node: '>=11.0'}
+ dependencies:
+ is-plain-object: 2.0.4
+ is-primitive: 3.0.1
+ dev: true
+
/setimmediate@1.0.5:
resolution: {integrity: sha512-MATJdZp8sLqDl/68LfQmbP8zKPLQNV6BIZoIgrscFDQ+RsvK/BxeDQOgyxKKoh0y/8h3BqVFnCqQ/gd+reiIXA==}
dev: false
@@ -15703,7 +15797,7 @@ packages:
'@babel/core': 7.23.9
bs-logger: 0.2.6
fast-json-stable-stringify: 2.1.0
- jest: 29.7.0(@types/node@20.11.6)
+ jest: 29.7.0(@types/node@18.19.6)
jest-util: 29.7.0
json5: 2.2.3
lodash.memoize: 4.1.2
@@ -15744,7 +15838,7 @@ packages:
yn: 3.1.1
dev: true
- /ts-node@10.9.2(@types/node@20.11.6)(typescript@5.3.3):
+ /ts-node@10.9.2(@types/node@18.19.6)(typescript@5.3.3):
resolution: {integrity: sha512-f0FFpIdcHgn8zcPSbf1dRevwt047YMnaiJM3u2w2RewrB+fob/zePZcrOyQoLMMO7aBIddLcQIEK5dYjkLnGrQ==}
hasBin: true
peerDependencies:
@@ -15763,7 +15857,7 @@ packages:
'@tsconfig/node12': 1.0.11
'@tsconfig/node14': 1.0.3
'@tsconfig/node16': 1.0.4
- '@types/node': 20.11.6
+ '@types/node': 18.19.6
acorn: 8.11.3
acorn-walk: 8.3.2
arg: 4.1.3
@@ -16323,6 +16417,10 @@ packages:
resolution: {integrity: sha512-AFbieoL7a5LMqcnOF04ji+rpXadgOXnZsxQr//r83kLPr7biP7am3g9zbaZIaBGwBRWeSvoMD4mgPdX3e4NWBg==}
dev: true
+ /w-json@1.3.10:
+ resolution: {integrity: sha512-XadVyw0xE+oZ5FGApXsdswv96rOhStzKqL53uSe5UaTadABGkWIg1+DTx8kiZ/VqTZTBneoL0l65RcPe4W3ecw==}
+ dev: true
+
/wait-port@1.1.0:
resolution: {integrity: sha512-3e04qkoN3LxTMLakdqeWth8nih8usyg+sf1Bgdf9wwUkp05iuK1eSY/QpLvscT/+F/gA89+LpUmmgBtesbqI2Q==}
engines: {node: '>=10'}