diff --git a/.changeset/cold-beers-try.md b/.changeset/cold-beers-try.md
new file mode 100644
index 0000000000000000000000000000000000000000..4b27e70e3fd5cb4dd45143c22cc913f9e5bddf4c
--- /dev/null
+++ b/.changeset/cold-beers-try.md
@@ -0,0 +1,5 @@
+---
+"@llamaindex/cloud": patch
+---
+
+fix(cloud): allow filename in llama parse
diff --git a/examples/data/pto_policy_employee.docx b/examples/data/pto_policy_employee.docx
new file mode 100644
index 0000000000000000000000000000000000000000..72ce29c35a91a4eee838c688349439edb4e9ad32
Binary files /dev/null and b/examples/data/pto_policy_employee.docx differ
diff --git a/examples/readers/src/llamaparse-docx.ts b/examples/readers/src/llamaparse-docx.ts
new file mode 100644
index 0000000000000000000000000000000000000000..6e6609706415290132934e93aa2a1214b95b263f
--- /dev/null
+++ b/examples/readers/src/llamaparse-docx.ts
@@ -0,0 +1,52 @@
+import { Language, LlamaParseReader } from "llamaindex";
+import fs from "node:fs";
+import path from "node:path";
+
+type LlamaParseReaderParams = Partial<
+ Omit<LlamaParseReader, "language" | "apiKey">
+> & {
+ language?: Language | Language[] | undefined;
+ apiKey?: string | undefined;
+};
+
+async function main() {
+ const filePath = "../data/pto_policy_employee.docx";
+ if (!fs.existsSync(filePath)) {
+ console.error(`File ${filePath} does not exist`);
+ process.exit(1);
+ } else {
+ console.log(`File ${filePath} exists`);
+ }
+
+ const params: LlamaParseReaderParams = {
+ verbose: true,
+ parsingInstruction:
+ "Extract the text from the document a long with any images and tables. This is a document for a course and the contents of the images are important.",
+ fastMode: false,
+ gpt4oMode: true,
+ useVendorMultimodalModel: true,
+ vendorMultimodalModelName: "anthropic-sonnet-3.5",
+ premiumMode: true,
+ resultType: "markdown",
+ apiKey: process.env.LLAMA_CLOUD_API_KEY,
+ doNotCache: true,
+ };
+
+ // set up the llamaparse reader
+ const reader = new LlamaParseReader(params);
+
+ const buffer = fs.readFileSync(filePath);
+ const documents = await reader.loadDataAsContent(
+ new Uint8Array(buffer),
+ path.basename(filePath),
+ );
+
+ let allText = "";
+ documents.forEach((doc) => {
+ allText += doc.text;
+ });
+
+ console.log(allText);
+}
+
+main().catch(console.error);
diff --git a/packages/cloud/openapi.json b/packages/cloud/openapi.json
index e27a423d8000d59be99c3ccc0119646db4780e55..d3f56b253b4dfc0257876570fff3404028548043 100644
--- a/packages/cloud/openapi.json
+++ b/packages/cloud/openapi.json
@@ -264,8 +264,9 @@
},
"/api/v1/validate-integrations/validate-embedding-connection": {
"post": {
+ "tags": ["Validation"],
"summary": "Validate Embedding Connection",
- "description": "Validate an embedding connection.",
+ "description": "Validate an embedding connection.\n\nArgs:\n embedding_config: The embedding configuration to validate.\n pipeline_id: If provided, the embedding connection will be validated for the pipeline.\n user: The user to validate the embedding connection for.\n db: The database session.\n\nReturns:\n A BaseConnectionValidation object indicating the result of the validation.",
"operationId": "validate_embedding_connection_api_v1_validate_integrations_validate_embedding_connection_post",
"security": [
{
@@ -276,6 +277,23 @@
}
],
"parameters": [
+ {
+ "name": "pipeline_id",
+ "in": "query",
+ "required": false,
+ "schema": {
+ "anyOf": [
+ {
+ "type": "string",
+ "format": "uuid"
+ },
+ {
+ "type": "null"
+ }
+ ],
+ "title": "Pipeline Id"
+ }
+ },
{
"name": "session",
"in": "cookie",
@@ -364,6 +382,7 @@
},
"/api/v1/validate-integrations/validate-data-source-connection": {
"post": {
+ "tags": ["Validation"],
"summary": "Validate Data Source Connection",
"description": "Validate a data source connection.",
"operationId": "validate_data_source_connection_api_v1_validate_integrations_validate_data_source_connection_post",
@@ -429,6 +448,7 @@
},
"/api/v1/validate-integrations/validate-data-sink-connection": {
"post": {
+ "tags": ["Validation"],
"summary": "Validate Data Sink Connection",
"description": "Validate a data sink connection.",
"operationId": "validate_data_sink_connection_api_v1_validate_integrations_validate_data_sink_connection_post",
@@ -2247,12 +2267,12 @@
}
}
},
- "/api/v1/projects": {
+ "/api/v1/organizations/{organization_id}/roles": {
"get": {
- "tags": ["Projects"],
- "summary": "List Projects",
- "description": "List projects or get one by name",
- "operationId": "list_projects_api_v1_projects_get",
+ "tags": ["Organizations"],
+ "summary": "List Roles",
+ "description": "List all roles in an organization.",
+ "operationId": "list_roles_api_v1_organizations__organization_id__roles_get",
"security": [
{
"HTTPBearer": []
@@ -2264,37 +2284,14 @@
"parameters": [
{
"name": "organization_id",
- "in": "query",
- "required": false,
+ "in": "path",
+ "required": true,
"schema": {
- "anyOf": [
- {
- "type": "string",
- "format": "uuid"
- },
- {
- "type": "null"
- }
- ],
+ "type": "string",
+ "format": "uuid",
"title": "Organization Id"
}
},
- {
- "name": "project_name",
- "in": "query",
- "required": false,
- "schema": {
- "anyOf": [
- {
- "type": "string"
- },
- {
- "type": "null"
- }
- ],
- "title": "Project Name"
- }
- },
{
"name": "session",
"in": "cookie",
@@ -2320,9 +2317,9 @@
"schema": {
"type": "array",
"items": {
- "$ref": "#/components/schemas/Project"
+ "$ref": "#/components/schemas/Role"
},
- "title": "Response List Projects Api V1 Projects Get"
+ "title": "Response List Roles Api V1 Organizations Organization Id Roles Get"
}
}
}
@@ -2338,12 +2335,14 @@
}
}
}
- },
- "post": {
- "tags": ["Projects"],
- "summary": "Create Project",
- "description": "Create a new project.",
- "operationId": "create_project_api_v1_projects_post",
+ }
+ },
+ "/api/v1/organizations/{organization_id}/users/roles": {
+ "put": {
+ "tags": ["Organizations"],
+ "summary": "Assign Role To User In Organization",
+ "description": "Assign a role to a user in an organization.",
+ "operationId": "assign_role_to_user_in_organization_api_v1_organizations__organization_id__users_roles_put",
"security": [
{
"HTTPBearer": []
@@ -2355,18 +2354,11 @@
"parameters": [
{
"name": "organization_id",
- "in": "query",
- "required": false,
+ "in": "path",
+ "required": true,
"schema": {
- "anyOf": [
- {
- "type": "string",
- "format": "uuid"
- },
- {
- "type": "null"
- }
- ],
+ "type": "string",
+ "format": "uuid",
"title": "Organization Id"
}
},
@@ -2392,7 +2384,7 @@
"content": {
"application/json": {
"schema": {
- "$ref": "#/components/schemas/ProjectCreate"
+ "$ref": "#/components/schemas/UserOrganizationRoleCreate"
}
}
}
@@ -2403,7 +2395,7 @@
"content": {
"application/json": {
"schema": {
- "$ref": "#/components/schemas/Project"
+ "$ref": "#/components/schemas/UserOrganizationRole"
}
}
}
@@ -2420,11 +2412,11 @@
}
}
},
- "put": {
- "tags": ["Projects"],
- "summary": "Upsert Project",
- "description": "Upsert a project.\nUpdates if a project with the same name already exists. Otherwise, creates a new project.",
- "operationId": "upsert_project_api_v1_projects_put",
+ "get": {
+ "tags": ["Organizations"],
+ "summary": "Get User Role",
+ "description": "Get the role of a user in an organization.",
+ "operationId": "get_user_role_api_v1_organizations__organization_id__users_roles_get",
"security": [
{
"HTTPBearer": []
@@ -2436,18 +2428,11 @@
"parameters": [
{
"name": "organization_id",
- "in": "query",
- "required": false,
+ "in": "path",
+ "required": true,
"schema": {
- "anyOf": [
- {
- "type": "string",
- "format": "uuid"
- },
- {
- "type": "null"
- }
- ],
+ "type": "string",
+ "format": "uuid",
"title": "Organization Id"
}
},
@@ -2468,23 +2453,13 @@
}
}
],
- "requestBody": {
- "required": true,
- "content": {
- "application/json": {
- "schema": {
- "$ref": "#/components/schemas/ProjectCreate"
- }
- }
- }
- },
"responses": {
"200": {
"description": "Successful Response",
"content": {
"application/json": {
"schema": {
- "$ref": "#/components/schemas/Project"
+ "$ref": "#/components/schemas/UserOrganizationRole"
}
}
}
@@ -2502,12 +2477,12 @@
}
}
},
- "/api/v1/projects/{project_id}": {
- "delete": {
- "tags": ["Projects"],
- "summary": "Delete Project",
- "description": "Delete a project by ID.",
- "operationId": "delete_project_api_v1_projects__project_id__delete",
+ "/api/v1/organizations/{organization_id}/users/{user_id}/projects": {
+ "get": {
+ "tags": ["Organizations"],
+ "summary": "List Projects By User",
+ "description": "List all projects for a user in an organization.",
+ "operationId": "list_projects_by_user_api_v1_organizations__organization_id__users__user_id__projects_get",
"security": [
{
"HTTPBearer": []
@@ -2518,13 +2493,22 @@
],
"parameters": [
{
- "name": "project_id",
+ "name": "organization_id",
"in": "path",
"required": true,
"schema": {
"type": "string",
"format": "uuid",
- "title": "Project Id"
+ "title": "Organization Id"
+ }
+ },
+ {
+ "name": "user_id",
+ "in": "path",
+ "required": true,
+ "schema": {
+ "type": "string",
+ "title": "User Id"
}
},
{
@@ -2545,8 +2529,19 @@
}
],
"responses": {
- "204": {
- "description": "Successful Response"
+ "200": {
+ "description": "Successful Response",
+ "content": {
+ "application/json": {
+ "schema": {
+ "type": "array",
+ "items": {
+ "$ref": "#/components/schemas/Project"
+ },
+ "title": "Response List Projects By User Api V1 Organizations Organization Id Users User Id Projects Get"
+ }
+ }
+ }
},
"422": {
"description": "Validation Error",
@@ -2560,11 +2555,11 @@
}
}
},
- "get": {
- "tags": ["Projects"],
- "summary": "Get Project",
- "description": "Get a project by ID.",
- "operationId": "get_project_api_v1_projects__project_id__get",
+ "put": {
+ "tags": ["Organizations"],
+ "summary": "Add User To Project",
+ "description": "Add a user to a project.",
+ "operationId": "add_user_to_project_api_v1_organizations__organization_id__users__user_id__projects_put",
"security": [
{
"HTTPBearer": []
@@ -2575,9 +2570,28 @@
],
"parameters": [
{
- "name": "project_id",
+ "name": "organization_id",
+ "in": "path",
+ "required": true,
+ "schema": {
+ "type": "string",
+ "format": "uuid",
+ "title": "Organization Id"
+ }
+ },
+ {
+ "name": "user_id",
"in": "path",
"required": true,
+ "schema": {
+ "type": "string",
+ "title": "User Id"
+ }
+ },
+ {
+ "name": "project_id",
+ "in": "query",
+ "required": true,
"schema": {
"type": "string",
"format": "uuid",
@@ -2606,9 +2620,7 @@
"description": "Successful Response",
"content": {
"application/json": {
- "schema": {
- "$ref": "#/components/schemas/Project"
- }
+ "schema": {}
}
}
},
@@ -2623,12 +2635,14 @@
}
}
}
- },
- "put": {
- "tags": ["Projects"],
- "summary": "Update Existing Project",
- "description": "Update an existing project.",
- "operationId": "update_existing_project_api_v1_projects__project_id__put",
+ }
+ },
+ "/api/v1/organizations/{organization_id}/users/{user_id}/projects/{project_id}": {
+ "delete": {
+ "tags": ["Organizations"],
+ "summary": "Remove User From Project",
+ "description": "Remove a user from a project.",
+ "operationId": "remove_user_from_project_api_v1_organizations__organization_id__users__user_id__projects__project_id__delete",
"security": [
{
"HTTPBearer": []
@@ -2638,6 +2652,25 @@
}
],
"parameters": [
+ {
+ "name": "organization_id",
+ "in": "path",
+ "required": true,
+ "schema": {
+ "type": "string",
+ "format": "uuid",
+ "title": "Organization Id"
+ }
+ },
+ {
+ "name": "user_id",
+ "in": "path",
+ "required": true,
+ "schema": {
+ "type": "string",
+ "title": "User Id"
+ }
+ },
{
"name": "project_id",
"in": "path",
@@ -2665,24 +2698,12 @@
}
}
],
- "requestBody": {
- "required": true,
- "content": {
- "application/json": {
- "schema": {
- "$ref": "#/components/schemas/ProjectUpdate"
- }
- }
- }
- },
"responses": {
"200": {
"description": "Successful Response",
"content": {
"application/json": {
- "schema": {
- "$ref": "#/components/schemas/Project"
- }
+ "schema": {}
}
}
},
@@ -2699,12 +2720,12 @@
}
}
},
- "/api/v1/projects/{project_id}/eval/dataset": {
- "post": {
+ "/api/v1/projects": {
+ "get": {
"tags": ["Projects"],
- "summary": "Create Eval Dataset For Project",
- "description": "Create a new eval dataset for a project.",
- "operationId": "create_eval_dataset_for_project_api_v1_projects__project_id__eval_dataset_post",
+ "summary": "List Projects",
+ "description": "List projects or get one by name",
+ "operationId": "list_projects_api_v1_projects_get",
"security": [
{
"HTTPBearer": []
@@ -2715,18 +2736,25 @@
],
"parameters": [
{
- "name": "project_id",
- "in": "path",
- "required": true,
+ "name": "organization_id",
+ "in": "query",
+ "required": false,
"schema": {
- "type": "string",
- "format": "uuid",
- "title": "Project Id"
+ "anyOf": [
+ {
+ "type": "string",
+ "format": "uuid"
+ },
+ {
+ "type": "null"
+ }
+ ],
+ "title": "Organization Id"
}
},
{
- "name": "session",
- "in": "cookie",
+ "name": "project_name",
+ "in": "query",
"required": false,
"schema": {
"anyOf": [
@@ -2737,9 +2765,454 @@
"type": "null"
}
],
- "title": "Session"
+ "title": "Project Name"
}
- }
+ },
+ {
+ "name": "session",
+ "in": "cookie",
+ "required": false,
+ "schema": {
+ "anyOf": [
+ {
+ "type": "string"
+ },
+ {
+ "type": "null"
+ }
+ ],
+ "title": "Session"
+ }
+ }
+ ],
+ "responses": {
+ "200": {
+ "description": "Successful Response",
+ "content": {
+ "application/json": {
+ "schema": {
+ "type": "array",
+ "items": {
+ "$ref": "#/components/schemas/Project"
+ },
+ "title": "Response List Projects Api V1 Projects Get"
+ }
+ }
+ }
+ },
+ "422": {
+ "description": "Validation Error",
+ "content": {
+ "application/json": {
+ "schema": {
+ "$ref": "#/components/schemas/HTTPValidationError"
+ }
+ }
+ }
+ }
+ }
+ },
+ "post": {
+ "tags": ["Projects"],
+ "summary": "Create Project",
+ "description": "Create a new project.",
+ "operationId": "create_project_api_v1_projects_post",
+ "security": [
+ {
+ "HTTPBearer": []
+ },
+ {
+ "HTTPBearer": []
+ }
+ ],
+ "parameters": [
+ {
+ "name": "organization_id",
+ "in": "query",
+ "required": false,
+ "schema": {
+ "anyOf": [
+ {
+ "type": "string",
+ "format": "uuid"
+ },
+ {
+ "type": "null"
+ }
+ ],
+ "title": "Organization Id"
+ }
+ },
+ {
+ "name": "session",
+ "in": "cookie",
+ "required": false,
+ "schema": {
+ "anyOf": [
+ {
+ "type": "string"
+ },
+ {
+ "type": "null"
+ }
+ ],
+ "title": "Session"
+ }
+ }
+ ],
+ "requestBody": {
+ "required": true,
+ "content": {
+ "application/json": {
+ "schema": {
+ "$ref": "#/components/schemas/ProjectCreate"
+ }
+ }
+ }
+ },
+ "responses": {
+ "200": {
+ "description": "Successful Response",
+ "content": {
+ "application/json": {
+ "schema": {
+ "$ref": "#/components/schemas/Project"
+ }
+ }
+ }
+ },
+ "422": {
+ "description": "Validation Error",
+ "content": {
+ "application/json": {
+ "schema": {
+ "$ref": "#/components/schemas/HTTPValidationError"
+ }
+ }
+ }
+ }
+ }
+ },
+ "put": {
+ "tags": ["Projects"],
+ "summary": "Upsert Project",
+ "description": "Upsert a project.\nUpdates if a project with the same name already exists. Otherwise, creates a new project.",
+ "operationId": "upsert_project_api_v1_projects_put",
+ "security": [
+ {
+ "HTTPBearer": []
+ },
+ {
+ "HTTPBearer": []
+ }
+ ],
+ "parameters": [
+ {
+ "name": "organization_id",
+ "in": "query",
+ "required": false,
+ "schema": {
+ "anyOf": [
+ {
+ "type": "string",
+ "format": "uuid"
+ },
+ {
+ "type": "null"
+ }
+ ],
+ "title": "Organization Id"
+ }
+ },
+ {
+ "name": "session",
+ "in": "cookie",
+ "required": false,
+ "schema": {
+ "anyOf": [
+ {
+ "type": "string"
+ },
+ {
+ "type": "null"
+ }
+ ],
+ "title": "Session"
+ }
+ }
+ ],
+ "requestBody": {
+ "required": true,
+ "content": {
+ "application/json": {
+ "schema": {
+ "$ref": "#/components/schemas/ProjectCreate"
+ }
+ }
+ }
+ },
+ "responses": {
+ "200": {
+ "description": "Successful Response",
+ "content": {
+ "application/json": {
+ "schema": {
+ "$ref": "#/components/schemas/Project"
+ }
+ }
+ }
+ },
+ "422": {
+ "description": "Validation Error",
+ "content": {
+ "application/json": {
+ "schema": {
+ "$ref": "#/components/schemas/HTTPValidationError"
+ }
+ }
+ }
+ }
+ }
+ }
+ },
+ "/api/v1/projects/{project_id}": {
+ "delete": {
+ "tags": ["Projects"],
+ "summary": "Delete Project",
+ "description": "Delete a project by ID.",
+ "operationId": "delete_project_api_v1_projects__project_id__delete",
+ "security": [
+ {
+ "HTTPBearer": []
+ },
+ {
+ "HTTPBearer": []
+ }
+ ],
+ "parameters": [
+ {
+ "name": "project_id",
+ "in": "path",
+ "required": true,
+ "schema": {
+ "type": "string",
+ "format": "uuid",
+ "title": "Project Id"
+ }
+ },
+ {
+ "name": "session",
+ "in": "cookie",
+ "required": false,
+ "schema": {
+ "anyOf": [
+ {
+ "type": "string"
+ },
+ {
+ "type": "null"
+ }
+ ],
+ "title": "Session"
+ }
+ }
+ ],
+ "responses": {
+ "204": {
+ "description": "Successful Response"
+ },
+ "422": {
+ "description": "Validation Error",
+ "content": {
+ "application/json": {
+ "schema": {
+ "$ref": "#/components/schemas/HTTPValidationError"
+ }
+ }
+ }
+ }
+ }
+ },
+ "get": {
+ "tags": ["Projects"],
+ "summary": "Get Project",
+ "description": "Get a project by ID.",
+ "operationId": "get_project_api_v1_projects__project_id__get",
+ "security": [
+ {
+ "HTTPBearer": []
+ },
+ {
+ "HTTPBearer": []
+ }
+ ],
+ "parameters": [
+ {
+ "name": "project_id",
+ "in": "path",
+ "required": true,
+ "schema": {
+ "type": "string",
+ "format": "uuid",
+ "title": "Project Id"
+ }
+ },
+ {
+ "name": "session",
+ "in": "cookie",
+ "required": false,
+ "schema": {
+ "anyOf": [
+ {
+ "type": "string"
+ },
+ {
+ "type": "null"
+ }
+ ],
+ "title": "Session"
+ }
+ }
+ ],
+ "responses": {
+ "200": {
+ "description": "Successful Response",
+ "content": {
+ "application/json": {
+ "schema": {
+ "$ref": "#/components/schemas/Project"
+ }
+ }
+ }
+ },
+ "422": {
+ "description": "Validation Error",
+ "content": {
+ "application/json": {
+ "schema": {
+ "$ref": "#/components/schemas/HTTPValidationError"
+ }
+ }
+ }
+ }
+ }
+ },
+ "put": {
+ "tags": ["Projects"],
+ "summary": "Update Existing Project",
+ "description": "Update an existing project.",
+ "operationId": "update_existing_project_api_v1_projects__project_id__put",
+ "security": [
+ {
+ "HTTPBearer": []
+ },
+ {
+ "HTTPBearer": []
+ }
+ ],
+ "parameters": [
+ {
+ "name": "project_id",
+ "in": "path",
+ "required": true,
+ "schema": {
+ "type": "string",
+ "format": "uuid",
+ "title": "Project Id"
+ }
+ },
+ {
+ "name": "session",
+ "in": "cookie",
+ "required": false,
+ "schema": {
+ "anyOf": [
+ {
+ "type": "string"
+ },
+ {
+ "type": "null"
+ }
+ ],
+ "title": "Session"
+ }
+ }
+ ],
+ "requestBody": {
+ "required": true,
+ "content": {
+ "application/json": {
+ "schema": {
+ "$ref": "#/components/schemas/ProjectUpdate"
+ }
+ }
+ }
+ },
+ "responses": {
+ "200": {
+ "description": "Successful Response",
+ "content": {
+ "application/json": {
+ "schema": {
+ "$ref": "#/components/schemas/Project"
+ }
+ }
+ }
+ },
+ "422": {
+ "description": "Validation Error",
+ "content": {
+ "application/json": {
+ "schema": {
+ "$ref": "#/components/schemas/HTTPValidationError"
+ }
+ }
+ }
+ }
+ }
+ }
+ },
+ "/api/v1/projects/{project_id}/eval/dataset": {
+ "post": {
+ "tags": ["Projects"],
+ "summary": "Create Eval Dataset For Project",
+ "description": "Create a new eval dataset for a project.",
+ "operationId": "create_eval_dataset_for_project_api_v1_projects__project_id__eval_dataset_post",
+ "security": [
+ {
+ "HTTPBearer": []
+ },
+ {
+ "HTTPBearer": []
+ }
+ ],
+ "parameters": [
+ {
+ "name": "project_id",
+ "in": "path",
+ "required": true,
+ "schema": {
+ "type": "string",
+ "format": "uuid",
+ "title": "Project Id"
+ }
+ },
+ {
+ "name": "session",
+ "in": "cookie",
+ "required": false,
+ "schema": {
+ "anyOf": [
+ {
+ "type": "string"
+ },
+ {
+ "type": "null"
+ }
+ ],
+ "title": "Session"
+ }
+ }
],
"requestBody": {
"required": true,
@@ -10643,7 +11116,7 @@
},
"chunk_size": {
"type": "integer",
- "exclusiveMinimum": 0.0,
+ "exclusiveMinimum": 0,
"title": "Chunk Size",
"description": "Chunk size for the transformation.",
"default": 1024
@@ -10664,20 +11137,17 @@
"model_name": {
"type": "string",
"title": "Model Name",
- "description": "The name of the embedding model.",
- "default": "unknown"
+ "description": "The name of the OpenAI embedding model.",
+ "default": "text-embedding-ada-002"
},
"embed_batch_size": {
"type": "integer",
- "maximum": 2048.0,
- "exclusiveMinimum": 0.0,
+ "maximum": 2048,
+ "exclusiveMinimum": 0,
"title": "Embed Batch Size",
"description": "The batch size for embedding calls.",
"default": 10
},
- "callback_manager": {
- "title": "Callback Manager"
- },
"num_workers": {
"anyOf": [
{
@@ -10696,7 +11166,14 @@
"description": "Additional kwargs for the OpenAI API."
},
"api_key": {
- "type": "string",
+ "anyOf": [
+ {
+ "type": "string"
+ },
+ {
+ "type": "null"
+ }
+ ],
"title": "Api Key",
"description": "The OpenAI API key."
},
@@ -10723,7 +11200,7 @@
"type": "number",
"title": "Timeout",
"description": "Timeout for each request.",
- "default": 60.0,
+ "default": 60,
"gte": 0
},
"default_headers": {
@@ -10783,23 +11260,6 @@
"title": "Azure Deployment",
"description": "The Azure deployment to use."
},
- "azure_ad_token_provider": {
- "anyOf": [
- {
- "type": "string"
- },
- {
- "type": "null"
- }
- ],
- "title": "Azure Ad Token Provider",
- "description": "Callback function to provide Azure AD token."
- },
- "use_azure_ad": {
- "type": "boolean",
- "title": "Use Azure Ad",
- "description": "Indicates if Microsoft Entra ID (former Azure AD) is used for token authentication"
- },
"class_name": {
"type": "string",
"title": "Class Name",
@@ -10807,7 +11267,6 @@
}
},
"type": "object",
- "required": ["api_key", "use_azure_ad"],
"title": "AzureOpenAIEmbedding"
},
"AzureOpenAIEmbeddingConfig": {
@@ -10915,19 +11374,17 @@
"model_name": {
"type": "string",
"title": "Model Name",
- "description": "The modelId of the Bedrock model to use."
+ "description": "The modelId of the Bedrock model to use.",
+ "default": "amazon.titan-embed-text-v1"
},
"embed_batch_size": {
"type": "integer",
- "maximum": 2048.0,
- "exclusiveMinimum": 0.0,
+ "maximum": 2048,
+ "exclusiveMinimum": 0,
"title": "Embed Batch Size",
"description": "The batch size for embedding calls.",
"default": 10
},
- "callback_manager": {
- "title": "Callback Manager"
- },
"num_workers": {
"anyOf": [
{
@@ -11000,29 +11457,9 @@
"title": "Region Name",
"description": "AWS region name to use. Uses region configured in AWS CLI if not passed"
},
- "botocore_session": {
- "anyOf": [
- {},
- {
- "type": "null"
- }
- ],
- "title": "Botocore Session",
- "description": "Use this Botocore session instead of creating a new default one."
- },
- "botocore_config": {
- "anyOf": [
- {},
- {
- "type": "null"
- }
- ],
- "title": "Botocore Config",
- "description": "Custom configuration object to use instead of the default generated one."
- },
"max_retries": {
"type": "integer",
- "exclusiveMinimum": 0.0,
+ "exclusiveMinimum": 0,
"title": "Max Retries",
"description": "The maximum number of API retries.",
"default": 10
@@ -11031,7 +11468,7 @@
"type": "number",
"title": "Timeout",
"description": "The timeout for the Bedrock API request in seconds. It will be used for both connect and read timeouts.",
- "default": 60.0
+ "default": 60
},
"additional_kwargs": {
"type": "object",
@@ -11045,7 +11482,6 @@
}
},
"type": "object",
- "required": ["model_name"],
"title": "BedrockEmbedding"
},
"BedrockEmbeddingConfig": {
@@ -11123,6 +11559,11 @@
"title": "Premium Mode",
"default": false
},
+ "continuous_mode": {
+ "type": "boolean",
+ "title": "Continuous Mode",
+ "default": false
+ },
"gpt4o_api_key": {
"type": "string",
"title": "Gpt4O Api Key",
@@ -11181,11 +11622,21 @@
"title": "Take Screenshot",
"default": false
},
+ "is_formatting_instruction": {
+ "type": "boolean",
+ "title": "Is Formatting Instruction",
+ "default": true
+ },
"disable_ocr": {
"type": "boolean",
"title": "Disable Ocr",
"default": false
},
+ "annotate_links": {
+ "type": "boolean",
+ "title": "Annotate Links",
+ "default": false
+ },
"disable_reconstruction": {
"type": "boolean",
"title": "Disable Reconstruction",
@@ -11201,6 +11652,22 @@
"title": "Output S3 Path Prefix",
"default": ""
},
+ "azure_openai_deployment_name": {
+ "type": "string",
+ "title": "Azure Openai Deployment Name"
+ },
+ "azure_openai_endpoint": {
+ "type": "string",
+ "title": "Azure Openai Endpoint"
+ },
+ "azure_openai_api_version": {
+ "type": "string",
+ "title": "Azure Openai Api Version"
+ },
+ "azure_openai_key": {
+ "type": "string",
+ "title": "Azure Openai Key"
+ },
"file": {
"anyOf": [
{
@@ -11274,6 +11741,11 @@
"title": "Premium Mode",
"default": false
},
+ "continuous_mode": {
+ "type": "boolean",
+ "title": "Continuous Mode",
+ "default": false
+ },
"gpt4o_api_key": {
"type": "string",
"title": "Gpt4O Api Key",
@@ -11332,11 +11804,21 @@
"title": "Take Screenshot",
"default": false
},
+ "is_formatting_instruction": {
+ "type": "boolean",
+ "title": "Is Formatting Instruction",
+ "default": true
+ },
"disable_ocr": {
"type": "boolean",
"title": "Disable Ocr",
"default": false
},
+ "annotate_links": {
+ "type": "boolean",
+ "title": "Annotate Links",
+ "default": false
+ },
"disable_reconstruction": {
"type": "boolean",
"title": "Disable Reconstruction",
@@ -11352,6 +11834,22 @@
"title": "Output S3 Path Prefix",
"default": ""
},
+ "azure_openai_deployment_name": {
+ "type": "string",
+ "title": "Azure Openai Deployment Name"
+ },
+ "azure_openai_endpoint": {
+ "type": "string",
+ "title": "Azure Openai Endpoint"
+ },
+ "azure_openai_api_version": {
+ "type": "string",
+ "title": "Azure Openai Api Version"
+ },
+ "azure_openai_key": {
+ "type": "string",
+ "title": "Azure Openai Key"
+ },
"file": {
"anyOf": [
{
@@ -11377,7 +11875,7 @@
"properties": {
"chunk_size": {
"type": "integer",
- "exclusiveMinimum": 0.0,
+ "exclusiveMinimum": 0,
"title": "Chunk Size",
"default": 1024
},
@@ -11429,14 +11927,14 @@
},
"chunk_size": {
"type": "integer",
- "exclusiveMinimum": 0.0,
+ "exclusiveMinimum": 0,
"title": "Chunk Size",
"description": "The token chunk size for each chunk.",
"default": 1024
},
"chunk_overlap": {
"type": "integer",
- "minimum": 0.0,
+ "minimum": 0,
"title": "Chunk Overlap",
"description": "The token overlap of each chunk when splitting.",
"default": 200
@@ -11644,7 +12142,9 @@
"account_key": {
"anyOf": [
{
- "type": "string"
+ "type": "string",
+ "format": "password",
+ "writeOnly": true
},
{
"type": "null"
@@ -11680,7 +12180,9 @@
"client_secret": {
"anyOf": [
{
- "type": "string"
+ "type": "string",
+ "format": "password",
+ "writeOnly": true
},
{
"type": "null"
@@ -11710,7 +12212,9 @@
},
"search_service_api_key": {
"type": "string",
- "title": "Search Service Api Key"
+ "format": "password",
+ "title": "Search Service Api Key",
+ "writeOnly": true
},
"search_service_endpoint": {
"type": "string",
@@ -11774,7 +12278,9 @@
"client_secret": {
"anyOf": [
{
- "type": "string"
+ "type": "string",
+ "format": "password",
+ "writeOnly": true
},
{
"type": "null"
@@ -11825,7 +12331,9 @@
"developer_token": {
"anyOf": [
{
- "type": "string"
+ "type": "string",
+ "format": "password",
+ "writeOnly": true
},
{
"type": "null"
@@ -11849,7 +12357,9 @@
"client_secret": {
"anyOf": [
{
- "type": "string"
+ "type": "string",
+ "format": "password",
+ "writeOnly": true
},
{
"type": "null"
@@ -11892,91 +12402,6 @@
"required": ["authentication_mechanism"],
"title": "CloudBoxDataSource"
},
- "CloudChromaVectorStore": {
- "properties": {
- "supports_nested_metadata_filters": {
- "type": "boolean",
- "enum": [false],
- "const": false,
- "title": "Supports Nested Metadata Filters",
- "default": false
- },
- "collection_name": {
- "anyOf": [
- {
- "type": "string"
- },
- {
- "type": "null"
- }
- ],
- "title": "Collection Name"
- },
- "host": {
- "anyOf": [
- {
- "type": "string"
- },
- {
- "type": "null"
- }
- ],
- "title": "Host"
- },
- "port": {
- "anyOf": [
- {
- "type": "string"
- },
- {
- "type": "null"
- }
- ],
- "title": "Port"
- },
- "ssl": {
- "type": "boolean",
- "title": "Ssl"
- },
- "headers": {
- "anyOf": [
- {
- "additionalProperties": {
- "type": "string"
- },
- "type": "object"
- },
- {
- "type": "null"
- }
- ],
- "title": "Headers"
- },
- "persist_dir": {
- "anyOf": [
- {
- "type": "string"
- },
- {
- "type": "null"
- }
- ],
- "title": "Persist Dir"
- },
- "collection_kwargs": {
- "type": "object",
- "title": "Collection Kwargs"
- },
- "class_name": {
- "type": "string",
- "title": "Class Name",
- "default": "CloudChromaVectorStore"
- }
- },
- "type": "object",
- "required": ["ssl"],
- "title": "CloudChromaVectorStore"
- },
"CloudConfluenceDataSource": {
"properties": {
"server_url": {
@@ -12004,7 +12429,9 @@
"api_token": {
"anyOf": [
{
- "type": "string"
+ "type": "string",
+ "format": "password",
+ "writeOnly": true
},
{
"type": "null"
@@ -12219,7 +12646,9 @@
"api_token": {
"anyOf": [
{
- "type": "string"
+ "type": "string",
+ "format": "password",
+ "writeOnly": true
},
{
"type": "null"
@@ -12300,7 +12729,9 @@
"token": {
"anyOf": [
{
- "type": "string"
+ "type": "string",
+ "format": "password",
+ "writeOnly": true
},
{
"type": "null"
@@ -12341,7 +12772,9 @@
},
"mongodb_uri": {
"type": "string",
- "title": "Mongodb Uri"
+ "format": "password",
+ "title": "Mongodb Uri",
+ "writeOnly": true
},
"db_name": {
"type": "string",
@@ -12388,8 +12821,10 @@
"properties": {
"integration_token": {
"type": "string",
+ "format": "password",
"title": "Integration Token",
- "description": "The integration token to use for authentication."
+ "description": "The integration token to use for authentication.",
+ "writeOnly": true
},
"database_ids": {
"anyOf": [
@@ -12463,8 +12898,10 @@
},
"client_secret": {
"type": "string",
+ "format": "password",
"title": "Client Secret",
- "description": "The client secret to use for authentication."
+ "description": "The client secret to use for authentication.",
+ "writeOnly": true
},
"tenant_id": {
"type": "string",
@@ -12497,7 +12934,10 @@
},
"api_key": {
"type": "string",
- "title": "Api Key"
+ "format": "password",
+ "title": "Api Key",
+ "description": "The API key for authenticating with Pinecone",
+ "writeOnly": true
},
"index_name": {
"type": "string",
@@ -12555,7 +12995,9 @@
},
"password": {
"type": "string",
- "title": "Password"
+ "format": "password",
+ "title": "Password",
+ "writeOnly": true
},
"port": {
"type": "string",
@@ -12627,7 +13069,9 @@
},
"api_key": {
"type": "string",
- "title": "Api Key"
+ "format": "password",
+ "title": "Api Key",
+ "writeOnly": true
},
"max_retries": {
"type": "integer",
@@ -12683,7 +13127,9 @@
"aws_access_secret": {
"anyOf": [
{
- "type": "string"
+ "type": "string",
+ "format": "password",
+ "writeOnly": true
},
{
"type": "null"
@@ -12783,8 +13229,10 @@
},
"client_secret": {
"type": "string",
+ "format": "password",
"title": "Client Secret",
- "description": "The client secret to use for authentication."
+ "description": "The client secret to use for authentication.",
+ "writeOnly": true
},
"tenant_id": {
"type": "string",
@@ -12805,8 +13253,10 @@
"properties": {
"slack_token": {
"type": "string",
+ "format": "password",
"title": "Slack Token",
- "description": "Slack Bot Token."
+ "description": "Slack Bot Token.",
+ "writeOnly": true
},
"channel_ids": {
"anyOf": [
@@ -12890,52 +13340,6 @@
"required": ["slack_token"],
"title": "CloudSlackDataSource"
},
- "CloudWeaviateVectorStore": {
- "properties": {
- "supports_nested_metadata_filters": {
- "type": "boolean",
- "enum": [false],
- "const": false,
- "title": "Supports Nested Metadata Filters",
- "default": false
- },
- "index_name": {
- "type": "string",
- "title": "Index Name"
- },
- "url": {
- "anyOf": [
- {
- "type": "string"
- },
- {
- "type": "null"
- }
- ],
- "title": "Url"
- },
- "text_key": {
- "type": "string",
- "title": "Text Key"
- },
- "auth_config": {
- "type": "object",
- "title": "Auth Config"
- },
- "client_kwargs": {
- "type": "object",
- "title": "Client Kwargs"
- },
- "class_name": {
- "type": "string",
- "title": "Class Name",
- "default": "CloudWeaviateVectorStore"
- }
- },
- "type": "object",
- "required": ["index_name", "text_key"],
- "title": "CloudWeaviateVectorStore"
- },
"CodeSplitter": {
"properties": {
"include_metadata": {
@@ -12972,21 +13376,21 @@
},
"chunk_lines": {
"type": "integer",
- "exclusiveMinimum": 0.0,
+ "exclusiveMinimum": 0,
"title": "Chunk Lines",
"description": "The number of lines to include in each chunk.",
"default": 40
},
"chunk_lines_overlap": {
"type": "integer",
- "exclusiveMinimum": 0.0,
+ "exclusiveMinimum": 0,
"title": "Chunk Lines Overlap",
"description": "How many lines of code each chunk overlaps with.",
"default": 15
},
"max_chars": {
"type": "integer",
- "exclusiveMinimum": 0.0,
+ "exclusiveMinimum": 0,
"title": "Max Chars",
"description": "Maximum number of characters per chunk.",
"default": 1500
@@ -13007,20 +13411,17 @@
"model_name": {
"type": "string",
"title": "Model Name",
- "description": "The name of the embedding model.",
- "default": "unknown"
+ "description": "The modelId of the Cohere model to use.",
+ "default": "embed-english-v3.0"
},
"embed_batch_size": {
"type": "integer",
- "maximum": 2048.0,
- "exclusiveMinimum": 0.0,
+ "maximum": 2048,
+ "exclusiveMinimum": 0,
"title": "Embed Batch Size",
"description": "The batch size for embedding calls.",
"default": 10
},
- "callback_manager": {
- "title": "Callback Manager"
- },
"num_workers": {
"anyOf": [
{
@@ -13034,14 +13435,22 @@
"description": "The number of workers to use for async embedding calls."
},
"api_key": {
- "type": "string",
+ "anyOf": [
+ {
+ "type": "string"
+ },
+ {
+ "type": "null"
+ }
+ ],
"title": "Api Key",
"description": "The Cohere API key."
},
"truncate": {
"type": "string",
"title": "Truncate",
- "description": "Truncation type - START/ END/ NONE"
+ "description": "Truncation type - START/ END/ NONE",
+ "default": "END"
},
"input_type": {
"anyOf": [
@@ -13058,7 +13467,8 @@
"embedding_type": {
"type": "string",
"title": "Embedding Type",
- "description": "Embedding type. If not provided float embedding_type is used when needed."
+ "description": "Embedding type. If not provided float embedding_type is used when needed.",
+ "default": "float"
},
"class_name": {
"type": "string",
@@ -13067,9 +13477,8 @@
}
},
"type": "object",
- "required": ["api_key", "truncate", "embedding_type"],
- "title": "CohereEmbedding",
- "description": "CohereEmbedding uses the Cohere API to generate embeddings for text."
+ "required": ["api_key"],
+ "title": "CohereEmbedding"
},
"CohereEmbeddingConfig": {
"properties": {
@@ -13092,11 +13501,9 @@
"ConfigurableDataSinkNames": {
"type": "string",
"enum": [
- "CHROMA",
"PINECONE",
"POSTGRES",
"QDRANT",
- "WEAVIATE",
"AZUREAI_SEARCH",
"MONGODB_ATLAS",
"MILVUS"
@@ -13159,14 +13566,7 @@
"SENTENCE_AWARE_NODE_PARSER",
"TOKEN_AWARE_NODE_PARSER",
"MARKDOWN_NODE_PARSER",
- "MARKDOWN_ELEMENT_NODE_PARSER",
- "OPENAI_EMBEDDING",
- "AZURE_EMBEDDING",
- "COHERE_EMBEDDING",
- "BEDROCK_EMBEDDING",
- "HUGGINGFACE_API_EMBEDDING",
- "GEMINI_EMBEDDING",
- "VERTEXAI_EMBEDDING"
+ "MARKDOWN_ELEMENT_NODE_PARSER"
],
"title": "ConfigurableTransformationNames"
},
@@ -13206,27 +13606,6 @@
},
{
"$ref": "#/components/schemas/MarkdownElementNodeParser"
- },
- {
- "$ref": "#/components/schemas/OpenAIEmbedding"
- },
- {
- "$ref": "#/components/schemas/AzureOpenAIEmbedding"
- },
- {
- "$ref": "#/components/schemas/CohereEmbedding"
- },
- {
- "$ref": "#/components/schemas/BedrockEmbedding"
- },
- {
- "$ref": "#/components/schemas/HuggingFaceInferenceAPIEmbedding"
- },
- {
- "$ref": "#/components/schemas/GeminiEmbedding"
- },
- {
- "$ref": "#/components/schemas/ExtendVertexTextEmbedding"
}
],
"title": "Component",
@@ -13298,9 +13677,6 @@
{
"type": "object"
},
- {
- "$ref": "#/components/schemas/CloudChromaVectorStore"
- },
{
"$ref": "#/components/schemas/CloudPineconeVectorStore"
},
@@ -13310,9 +13686,6 @@
{
"$ref": "#/components/schemas/CloudQdrantVectorStore"
},
- {
- "$ref": "#/components/schemas/CloudWeaviateVectorStore"
- },
{
"$ref": "#/components/schemas/CloudAzureAISearchVectorStore"
},
@@ -13349,10 +13722,7 @@
"component": {
"anyOf": [
{
- "type": "object"
- },
- {
- "$ref": "#/components/schemas/CloudChromaVectorStore"
+ "type": "object"
},
{
"$ref": "#/components/schemas/CloudPineconeVectorStore"
@@ -13363,9 +13733,6 @@
{
"$ref": "#/components/schemas/CloudQdrantVectorStore"
},
- {
- "$ref": "#/components/schemas/CloudWeaviateVectorStore"
- },
{
"$ref": "#/components/schemas/CloudAzureAISearchVectorStore"
},
@@ -13428,9 +13795,6 @@
{
"type": "object"
},
- {
- "$ref": "#/components/schemas/CloudChromaVectorStore"
- },
{
"$ref": "#/components/schemas/CloudPineconeVectorStore"
},
@@ -13440,9 +13804,6 @@
{
"$ref": "#/components/schemas/CloudQdrantVectorStore"
},
- {
- "$ref": "#/components/schemas/CloudWeaviateVectorStore"
- },
{
"$ref": "#/components/schemas/CloudAzureAISearchVectorStore"
},
@@ -14331,122 +14692,6 @@
"title": "EvalQuestionResult",
"description": "Schema for the result of an eval question job."
},
- "ExtendVertexTextEmbedding": {
- "properties": {
- "model_name": {
- "type": "string",
- "title": "Model Name",
- "description": "The name of the embedding model.",
- "default": "unknown"
- },
- "embed_batch_size": {
- "type": "integer",
- "maximum": 2048.0,
- "exclusiveMinimum": 0.0,
- "title": "Embed Batch Size",
- "description": "The batch size for embedding calls.",
- "default": 10
- },
- "callback_manager": {
- "title": "Callback Manager"
- },
- "num_workers": {
- "anyOf": [
- {
- "type": "integer"
- },
- {
- "type": "null"
- }
- ],
- "title": "Num Workers",
- "description": "The number of workers to use for async embedding calls."
- },
- "embed_mode": {
- "$ref": "#/components/schemas/VertexEmbeddingMode",
- "description": "The embedding mode to use."
- },
- "additional_kwargs": {
- "type": "object",
- "title": "Additional Kwargs",
- "description": "Additional kwargs for the Vertex."
- },
- "client_email": {
- "anyOf": [
- {
- "type": "string"
- },
- {
- "type": "null"
- }
- ],
- "title": "Client Email",
- "description": "The client email to use when making Vertex API calls."
- },
- "token_uri": {
- "anyOf": [
- {
- "type": "string"
- },
- {
- "type": "null"
- }
- ],
- "title": "Token Uri",
- "description": "The token uri to use when making Vertex API calls."
- },
- "private_key_id": {
- "anyOf": [
- {
- "type": "string"
- },
- {
- "type": "null"
- }
- ],
- "title": "Private Key Id",
- "description": "The private key id to use when making Vertex API calls."
- },
- "private_key": {
- "anyOf": [
- {
- "type": "string"
- },
- {
- "type": "null"
- }
- ],
- "title": "Private Key",
- "description": "The private key to use when making Vertex API calls."
- },
- "project": {
- "type": "string",
- "title": "Project",
- "description": "The default GCP project to use when making Vertex API calls."
- },
- "location": {
- "type": "string",
- "title": "Location",
- "description": "The default location to use when making API calls."
- },
- "class_name": {
- "type": "string",
- "title": "Class Name",
- "default": "ExtendVertexTextEmbedding"
- }
- },
- "type": "object",
- "required": [
- "embed_mode",
- "client_email",
- "token_uri",
- "private_key_id",
- "private_key",
- "project",
- "location"
- ],
- "title": "ExtendVertexTextEmbedding"
- },
"ExtractionJob": {
"properties": {
"id": {
@@ -14887,7 +15132,7 @@
"anyOf": [
{
"type": "integer",
- "minimum": 0.0
+ "minimum": 0
},
{
"type": "null"
@@ -14998,7 +15243,7 @@
"anyOf": [
{
"type": "integer",
- "minimum": 0.0
+ "minimum": 0
},
{
"type": "null"
@@ -15086,11 +15331,11 @@
"type": "string",
"enum": [
"==",
- ">",
- "<",
+ "\u003E",
+ "\u003C",
"!=",
- ">=",
- "<=",
+ "\u003E=",
+ "\u003C=",
"in",
"nin",
"any",
@@ -15107,20 +15352,17 @@
"model_name": {
"type": "string",
"title": "Model Name",
- "description": "The name of the embedding model.",
- "default": "unknown"
+ "description": "The modelId of the Gemini model to use.",
+ "default": "models/embedding-001"
},
"embed_batch_size": {
"type": "integer",
- "maximum": 2048.0,
- "exclusiveMinimum": 0.0,
+ "maximum": 2048,
+ "exclusiveMinimum": 0,
"title": "Embed Batch Size",
"description": "The batch size for embedding calls.",
"default": 10
},
- "callback_manager": {
- "title": "Callback Manager"
- },
"num_workers": {
"anyOf": [
{
@@ -15171,6 +15413,30 @@
"title": "Api Key",
"description": "API key to access the model. Defaults to None."
},
+ "api_base": {
+ "anyOf": [
+ {
+ "type": "string"
+ },
+ {
+ "type": "null"
+ }
+ ],
+ "title": "Api Base",
+ "description": "API base to access the model. Defaults to None."
+ },
+ "transport": {
+ "anyOf": [
+ {
+ "type": "string"
+ },
+ {
+ "type": "null"
+ }
+ ],
+ "title": "Transport",
+ "description": "Transport to access the model. Defaults to None."
+ },
"class_name": {
"type": "string",
"title": "Class Name",
@@ -15178,8 +15444,7 @@
}
},
"type": "object",
- "title": "GeminiEmbedding",
- "description": "Google Gemini embeddings.\n\nArgs:\n model_name (str): Model for embedding.\n Defaults to \"models/embedding-001\".\n\n api_key (Optional[str]): API key to access the model. Defaults to None.\n api_base (Optional[str]): API base to access the model. Defaults to Official Base.\n transport (Optional[str]): Transport to access the model."
+ "title": "GeminiEmbedding"
},
"GeminiEmbeddingConfig": {
"properties": {
@@ -15228,15 +15493,12 @@
},
"embed_batch_size": {
"type": "integer",
- "maximum": 2048.0,
- "exclusiveMinimum": 0.0,
+ "maximum": 2048,
+ "exclusiveMinimum": 0,
"title": "Embed Batch Size",
"description": "The batch size for embedding calls.",
"default": 10
},
- "callback_manager": {
- "title": "Callback Manager"
- },
"num_workers": {
"anyOf": [
{
@@ -15361,8 +15623,7 @@
}
},
"type": "object",
- "title": "HuggingFaceInferenceAPIEmbedding",
- "description": "Wrapper on the Hugging Face's Inference API for embeddings.\n\nOverview of the design:\n- Uses the feature extraction task: https://huggingface.co/tasks/feature-extraction"
+ "title": "HuggingFaceInferenceAPIEmbedding"
},
"HuggingFaceInferenceAPIEmbeddingConfig": {
"properties": {
@@ -15596,6 +15857,11 @@
"title": "Disable Ocr",
"default": false
},
+ "annotate_links": {
+ "type": "boolean",
+ "title": "Annotate Links",
+ "default": false
+ },
"disable_reconstruction": {
"type": "boolean",
"title": "Disable Reconstruction",
@@ -15614,7 +15880,7 @@
"fast_mode": {
"type": "boolean",
"title": "Fast Mode",
- "default": true
+ "default": false
},
"skip_diagonal_text": {
"type": "boolean",
@@ -15692,11 +15958,21 @@
"title": "Take Screenshot",
"default": false
},
+ "is_formatting_instruction": {
+ "type": "boolean",
+ "title": "Is Formatting Instruction",
+ "default": true
+ },
"premium_mode": {
"type": "boolean",
"title": "Premium Mode",
"default": false
},
+ "continuous_mode": {
+ "type": "boolean",
+ "title": "Continuous Mode",
+ "default": false
+ },
"s3_input_path": {
"type": "string",
"title": "S3 Input Path",
@@ -15706,6 +15982,50 @@
"type": "string",
"title": "S3 Output Path Prefix",
"default": ""
+ },
+ "azure_openai_deployment_name": {
+ "anyOf": [
+ {
+ "type": "string"
+ },
+ {
+ "type": "null"
+ }
+ ],
+ "title": "Azure Openai Deployment Name"
+ },
+ "azure_openai_endpoint": {
+ "anyOf": [
+ {
+ "type": "string"
+ },
+ {
+ "type": "null"
+ }
+ ],
+ "title": "Azure Openai Endpoint"
+ },
+ "azure_openai_api_version": {
+ "anyOf": [
+ {
+ "type": "string"
+ },
+ {
+ "type": "null"
+ }
+ ],
+ "title": "Azure Openai Api Version"
+ },
+ "azure_openai_key": {
+ "anyOf": [
+ {
+ "type": "string"
+ },
+ {
+ "type": "null"
+ }
+ ],
+ "title": "Azure Openai Key"
}
},
"type": "object",
@@ -16419,20 +16739,17 @@
"model_name": {
"type": "string",
"title": "Model Name",
- "description": "The name of the embedding model.",
- "default": "unknown"
+ "description": "The name of the OpenAI embedding model.",
+ "default": "text-embedding-ada-002"
},
"embed_batch_size": {
"type": "integer",
- "maximum": 2048.0,
- "exclusiveMinimum": 0.0,
+ "maximum": 2048,
+ "exclusiveMinimum": 0,
"title": "Embed Batch Size",
"description": "The batch size for embedding calls.",
"default": 10
},
- "callback_manager": {
- "title": "Callback Manager"
- },
"num_workers": {
"anyOf": [
{
@@ -16451,7 +16768,14 @@
"description": "Additional kwargs for the OpenAI API."
},
"api_key": {
- "type": "string",
+ "anyOf": [
+ {
+ "type": "string"
+ },
+ {
+ "type": "null"
+ }
+ ],
"title": "Api Key",
"description": "The OpenAI API key."
},
@@ -16492,7 +16816,7 @@
"type": "number",
"title": "Timeout",
"description": "Timeout for each request.",
- "default": 60.0,
+ "default": 60,
"gte": 0
},
"default_headers": {
@@ -16535,9 +16859,7 @@
}
},
"type": "object",
- "required": ["api_key"],
- "title": "OpenAIEmbedding",
- "description": "OpenAI class for embeddings.\n\nArgs:\n mode (str): Mode for embedding.\n Defaults to OpenAIEmbeddingMode.TEXT_SEARCH_MODE.\n Options are:\n\n - OpenAIEmbeddingMode.SIMILARITY_MODE\n - OpenAIEmbeddingMode.TEXT_SEARCH_MODE\n\n model (str): Model for embedding.\n Defaults to OpenAIEmbeddingModelType.TEXT_EMBED_ADA_002.\n Options are:\n\n - OpenAIEmbeddingModelType.DAVINCI\n - OpenAIEmbeddingModelType.CURIE\n - OpenAIEmbeddingModelType.BABBAGE\n - OpenAIEmbeddingModelType.ADA\n - OpenAIEmbeddingModelType.TEXT_EMBED_ADA_002"
+ "title": "OpenAIEmbedding"
},
"OpenAIEmbeddingConfig": {
"properties": {
@@ -16644,7 +16966,7 @@
"properties": {
"page_index": {
"type": "integer",
- "minimum": 0.0,
+ "minimum": 0,
"title": "Page Index",
"description": "The index of the page for which the screenshot is taken (0-indexed)"
},
@@ -16656,7 +16978,7 @@
},
"image_size": {
"type": "integer",
- "minimum": 0.0,
+ "minimum": 0,
"title": "Image Size",
"description": "The size of the image in bytes"
}
@@ -17040,6 +17362,70 @@
"title": "PartitionNames",
"description": "Enum for dataset partition names."
},
+ "Permission": {
+ "properties": {
+ "id": {
+ "type": "string",
+ "format": "uuid",
+ "title": "Id",
+ "description": "Unique identifier"
+ },
+ "created_at": {
+ "anyOf": [
+ {
+ "type": "string",
+ "format": "date-time"
+ },
+ {
+ "type": "null"
+ }
+ ],
+ "title": "Created At",
+ "description": "Creation datetime"
+ },
+ "updated_at": {
+ "anyOf": [
+ {
+ "type": "string",
+ "format": "date-time"
+ },
+ {
+ "type": "null"
+ }
+ ],
+ "title": "Updated At",
+ "description": "Update datetime"
+ },
+ "name": {
+ "type": "string",
+ "maxLength": 3000,
+ "minLength": 1,
+ "title": "Name",
+ "description": "A name for the permission."
+ },
+ "description": {
+ "anyOf": [
+ {
+ "type": "string"
+ },
+ {
+ "type": "null"
+ }
+ ],
+ "title": "Description",
+ "description": "A description for the permission."
+ },
+ "access": {
+ "type": "boolean",
+ "title": "Access",
+ "description": "Whether the permission is granted or not."
+ }
+ },
+ "type": "object",
+ "required": ["id", "name", "description", "access"],
+ "title": "Permission",
+ "description": "Schema for a permission."
+ },
"Pipeline": {
"properties": {
"id": {
@@ -17741,7 +18127,7 @@
"anyOf": [
{
"type": "integer",
- "minimum": 0.0
+ "minimum": 0
},
{
"type": "null"
@@ -18295,8 +18681,8 @@
"anyOf": [
{
"type": "integer",
- "maximum": 100.0,
- "minimum": 1.0
+ "maximum": 100,
+ "minimum": 1
},
{
"type": "null"
@@ -18310,8 +18696,8 @@
"anyOf": [
{
"type": "integer",
- "maximum": 100.0,
- "minimum": 1.0
+ "maximum": 100,
+ "minimum": 1
},
{
"type": "null"
@@ -18337,8 +18723,8 @@
"anyOf": [
{
"type": "integer",
- "maximum": 100.0,
- "minimum": 1.0
+ "maximum": 100,
+ "minimum": 1
},
{
"type": "null"
@@ -18352,8 +18738,8 @@
"anyOf": [
{
"type": "number",
- "maximum": 1.0,
- "minimum": 0.0
+ "maximum": 1,
+ "minimum": 0
},
{
"type": "null"
@@ -18377,8 +18763,8 @@
"anyOf": [
{
"type": "integer",
- "maximum": 5.0,
- "minimum": 1.0
+ "maximum": 5,
+ "minimum": 1
},
{
"type": "null"
@@ -18391,7 +18777,7 @@
"retrieval_mode": {
"$ref": "#/components/schemas/RetrievalMode",
"description": "The retrieval mode for the query.",
- "default": "chunks"
+ "default": "auto_routed"
},
"retrieve_image_nodes": {
"type": "boolean",
@@ -18688,7 +19074,12 @@
},
"RetrievalMode": {
"type": "string",
- "enum": ["chunks", "files_via_metadata", "files_via_content"],
+ "enum": [
+ "chunks",
+ "files_via_metadata",
+ "files_via_content",
+ "auto_routed"
+ ],
"title": "RetrievalMode"
},
"RetrievalParams": {
@@ -18697,8 +19088,8 @@
"anyOf": [
{
"type": "integer",
- "maximum": 100.0,
- "minimum": 1.0
+ "maximum": 100,
+ "minimum": 1
},
{
"type": "null"
@@ -18712,8 +19103,8 @@
"anyOf": [
{
"type": "integer",
- "maximum": 100.0,
- "minimum": 1.0
+ "maximum": 100,
+ "minimum": 1
},
{
"type": "null"
@@ -18739,8 +19130,8 @@
"anyOf": [
{
"type": "integer",
- "maximum": 100.0,
- "minimum": 1.0
+ "maximum": 100,
+ "minimum": 1
},
{
"type": "null"
@@ -18754,8 +19145,8 @@
"anyOf": [
{
"type": "number",
- "maximum": 1.0,
- "minimum": 0.0
+ "maximum": 1,
+ "minimum": 0
},
{
"type": "null"
@@ -18779,8 +19170,8 @@
"anyOf": [
{
"type": "integer",
- "maximum": 5.0,
- "minimum": 1.0
+ "maximum": 5,
+ "minimum": 1
},
{
"type": "null"
@@ -18793,7 +19184,7 @@
"retrieval_mode": {
"$ref": "#/components/schemas/RetrievalMode",
"description": "The retrieval mode for the query.",
- "default": "chunks"
+ "default": "auto_routed"
},
"retrieve_image_nodes": {
"type": "boolean",
@@ -18849,6 +19240,14 @@
"title": "Retrieval Latency",
"description": "The end-to-end latency for retrieval and reranking."
},
+ "metadata": {
+ "additionalProperties": {
+ "type": "string"
+ },
+ "type": "object",
+ "title": "Metadata",
+ "description": "Metadata associated with the retrieval execution"
+ },
"class_name": {
"type": "string",
"title": "Class Name",
@@ -18860,6 +19259,74 @@
"title": "RetrieveResults",
"description": "Schema for the result of an retrieval execution."
},
+ "Role": {
+ "properties": {
+ "id": {
+ "type": "string",
+ "format": "uuid",
+ "title": "Id",
+ "description": "Unique identifier"
+ },
+ "created_at": {
+ "anyOf": [
+ {
+ "type": "string",
+ "format": "date-time"
+ },
+ {
+ "type": "null"
+ }
+ ],
+ "title": "Created At",
+ "description": "Creation datetime"
+ },
+ "updated_at": {
+ "anyOf": [
+ {
+ "type": "string",
+ "format": "date-time"
+ },
+ {
+ "type": "null"
+ }
+ ],
+ "title": "Updated At",
+ "description": "Update datetime"
+ },
+ "name": {
+ "type": "string",
+ "maxLength": 3000,
+ "minLength": 1,
+ "title": "Name",
+ "description": "A name for the role."
+ },
+ "organization_id": {
+ "anyOf": [
+ {
+ "type": "string",
+ "format": "uuid"
+ },
+ {
+ "type": "null"
+ }
+ ],
+ "title": "Organization Id",
+ "description": "The organization's ID."
+ },
+ "permissions": {
+ "items": {
+ "$ref": "#/components/schemas/Permission"
+ },
+ "type": "array",
+ "title": "Permissions",
+ "description": "The actual permissions of the role."
+ }
+ },
+ "type": "object",
+ "required": ["id", "name", "organization_id", "permissions"],
+ "title": "Role",
+ "description": "Schema for a role."
+ },
"SemanticChunkingConfig": {
"properties": {
"mode": {
@@ -18887,7 +19354,7 @@
"properties": {
"chunk_size": {
"type": "integer",
- "exclusiveMinimum": 0.0,
+ "exclusiveMinimum": 0,
"title": "Chunk Size",
"default": 1024
},
@@ -18949,14 +19416,14 @@
},
"chunk_size": {
"type": "integer",
- "exclusiveMinimum": 0.0,
+ "exclusiveMinimum": 0,
"title": "Chunk Size",
"description": "The token chunk size for each chunk.",
"default": 1024
},
"chunk_overlap": {
"type": "integer",
- "minimum": 0.0,
+ "minimum": 0,
"title": "Chunk Overlap",
"description": "The token overlap of each chunk when splitting.",
"default": 200
@@ -19179,7 +19646,7 @@
"class_name": {
"type": "string",
"title": "Class Name",
- "default": "NodeWithScore"
+ "default": "TextNodeWithScore"
}
},
"type": "object",
@@ -19191,7 +19658,7 @@
"properties": {
"chunk_size": {
"type": "integer",
- "exclusiveMinimum": 0.0,
+ "exclusiveMinimum": 0,
"title": "Chunk Size",
"default": 1024
},
@@ -19248,14 +19715,14 @@
},
"chunk_size": {
"type": "integer",
- "exclusiveMinimum": 0.0,
+ "exclusiveMinimum": 0,
"title": "Chunk Size",
"description": "The token chunk size for each chunk.",
"default": 1024
},
"chunk_overlap": {
"type": "integer",
- "minimum": 0.0,
+ "minimum": 0,
"title": "Chunk Overlap",
"description": "The token overlap of each chunk when splitting.",
"default": 20
@@ -19375,10 +19842,18 @@
],
"title": "Invited By User Email",
"description": "The email address of the user who added the user to the organization."
+ },
+ "roles": {
+ "items": {
+ "$ref": "#/components/schemas/UserOrganizationRole"
+ },
+ "type": "array",
+ "title": "Roles",
+ "description": "The roles of the user in the organization."
}
},
"type": "object",
- "required": ["id", "email", "organization_id"],
+ "required": ["id", "email", "organization_id", "roles"],
"title": "UserOrganization",
"description": "Schema for a user's membership to an organization."
},
@@ -19408,9 +19883,39 @@
],
"title": "Email",
"description": "The user's email address."
+ },
+ "project_ids": {
+ "anyOf": [
+ {
+ "items": {
+ "type": "string",
+ "format": "uuid"
+ },
+ "type": "array"
+ },
+ {
+ "type": "null"
+ }
+ ],
+ "title": "Project Ids",
+ "description": "The project IDs to add the user to."
+ },
+ "role_id": {
+ "anyOf": [
+ {
+ "type": "string",
+ "format": "uuid"
+ },
+ {
+ "type": "null"
+ }
+ ],
+ "title": "Role Id",
+ "description": "The role ID to assign to the user."
}
},
"type": "object",
+ "required": ["project_ids"],
"title": "UserOrganizationCreate",
"description": "Schema for creating a user's membership to an organization."
},
@@ -19446,6 +19951,92 @@
"title": "UserOrganizationDelete",
"description": "Schema for deleting a user's membership to an organization."
},
+ "UserOrganizationRole": {
+ "properties": {
+ "id": {
+ "type": "string",
+ "format": "uuid",
+ "title": "Id",
+ "description": "Unique identifier"
+ },
+ "created_at": {
+ "anyOf": [
+ {
+ "type": "string",
+ "format": "date-time"
+ },
+ {
+ "type": "null"
+ }
+ ],
+ "title": "Created At",
+ "description": "Creation datetime"
+ },
+ "updated_at": {
+ "anyOf": [
+ {
+ "type": "string",
+ "format": "date-time"
+ },
+ {
+ "type": "null"
+ }
+ ],
+ "title": "Updated At",
+ "description": "Update datetime"
+ },
+ "user_id": {
+ "type": "string",
+ "title": "User Id",
+ "description": "The user's ID."
+ },
+ "organization_id": {
+ "type": "string",
+ "format": "uuid",
+ "title": "Organization Id",
+ "description": "The organization's ID."
+ },
+ "role_id": {
+ "type": "string",
+ "format": "uuid",
+ "title": "Role Id",
+ "description": "The role's ID."
+ },
+ "role": {
+ "$ref": "#/components/schemas/Role",
+ "description": "The role."
+ }
+ },
+ "type": "object",
+ "required": ["id", "user_id", "organization_id", "role_id", "role"],
+ "title": "UserOrganizationRole",
+ "description": "Schema for a user's role in an organization."
+ },
+ "UserOrganizationRoleCreate": {
+ "properties": {
+ "user_id": {
+ "type": "string",
+ "title": "User Id",
+ "description": "The user's ID."
+ },
+ "organization_id": {
+ "type": "string",
+ "format": "uuid",
+ "title": "Organization Id",
+ "description": "The organization's ID."
+ },
+ "role_id": {
+ "type": "string",
+ "format": "uuid",
+ "title": "Role Id",
+ "description": "The role's ID."
+ }
+ },
+ "type": "object",
+ "required": ["user_id", "organization_id", "role_id"],
+ "title": "UserOrganizationRoleCreate",
+ "description": "Schema for creating a user's role in an organization."
+ },
"ValidationError": {
"properties": {
"loc": {
@@ -19486,7 +20077,7 @@
"default": "VERTEXAI_EMBEDDING"
},
"component": {
- "$ref": "#/components/schemas/ExtendVertexTextEmbedding",
+ "$ref": "#/components/schemas/VertexTextEmbedding",
"description": "Configuration for the VertexAI embedding model."
}
},
@@ -19503,7 +20094,120 @@
"retrieval"
],
"title": "VertexEmbeddingMode",
- "description": "VertexAI embedding mode.\n\nAttributes:\n DEFAULT_MODE (str): The default embedding mode, for older models before August 2023,\n that does not support task_type\n CLASSIFICATION_MODE (str): Optimizes embeddings for classification tasks.\n CLUSTERING_MODE (str): Optimizes embeddings for clustering tasks.\n SEMANTIC_SIMILARITY_MODE (str): Optimizes embeddings for tasks that require assessments of semantic similarity.\n RETRIEVAL_MODE (str): Optimizes embeddings for retrieval tasks, including search and document retrieval."
+ "description": "Copied from llama_index.embeddings.vertex.base.VertexEmbeddingMode\nsince importing llama_index.embeddings.vertex.base incurs a lot of memory usage."
+ },
+ "VertexTextEmbedding": {
+ "properties": {
+ "model_name": {
+ "type": "string",
+ "title": "Model Name",
+ "description": "The modelId of the VertexAI model to use.",
+ "default": "textembedding-gecko@003"
+ },
+ "embed_batch_size": {
+ "type": "integer",
+ "maximum": 2048,
+ "exclusiveMinimum": 0,
+ "title": "Embed Batch Size",
+ "description": "The batch size for embedding calls.",
+ "default": 10
+ },
+ "num_workers": {
+ "anyOf": [
+ {
+ "type": "integer"
+ },
+ {
+ "type": "null"
+ }
+ ],
+ "title": "Num Workers",
+ "description": "The number of workers to use for async embedding calls."
+ },
+ "location": {
+ "type": "string",
+ "title": "Location",
+ "description": "The default location to use when making API calls."
+ },
+ "project": {
+ "type": "string",
+ "title": "Project",
+ "description": "The default GCP project to use when making Vertex API calls."
+ },
+ "embed_mode": {
+ "$ref": "#/components/schemas/VertexEmbeddingMode",
+ "description": "The embedding mode to use.",
+ "default": "retrieval"
+ },
+ "additional_kwargs": {
+ "type": "object",
+ "title": "Additional Kwargs",
+ "description": "Additional kwargs for the Vertex."
+ },
+ "client_email": {
+ "anyOf": [
+ {
+ "type": "string"
+ },
+ {
+ "type": "null"
+ }
+ ],
+ "title": "Client Email",
+ "description": "The client email for the VertexAI credentials."
+ },
+ "token_uri": {
+ "anyOf": [
+ {
+ "type": "string"
+ },
+ {
+ "type": "null"
+ }
+ ],
+ "title": "Token Uri",
+ "description": "The token URI for the VertexAI credentials."
+ },
+ "private_key_id": {
+ "anyOf": [
+ {
+ "type": "string"
+ },
+ {
+ "type": "null"
+ }
+ ],
+ "title": "Private Key Id",
+ "description": "The private key ID for the VertexAI credentials."
+ },
+ "private_key": {
+ "anyOf": [
+ {
+ "type": "string"
+ },
+ {
+ "type": "null"
+ }
+ ],
+ "title": "Private Key",
+ "description": "The private key for the VertexAI credentials."
+ },
+ "class_name": {
+ "type": "string",
+ "title": "Class Name",
+ "default": "VertexTextEmbedding"
+ }
+ },
+ "type": "object",
+ "required": [
+ "location",
+ "project",
+ "client_email",
+ "token_uri",
+ "private_key_id",
+ "private_key"
+ ],
+ "title": "VertexTextEmbedding"
}
},
"securitySchemes": {
diff --git a/packages/cloud/package.json b/packages/cloud/package.json
index 38d31e93e4073649ae05dc179ebf2a8364670760..b7cf8b512a0989b7680118c533d5aa2bac7a8189 100644
--- a/packages/cloud/package.json
+++ b/packages/cloud/package.json
@@ -4,8 +4,9 @@
"type": "module",
"license": "MIT",
"scripts": {
- "generate": "pnpx @hey-api/openapi-ts@0.53.0",
- "build": "pnpm run generate && bunchee"
+ "generate": "pnpx @hey-api/openapi-ts@0.53.11",
+ "build": "pnpm run generate && bunchee",
+ "dev": "bunchee --watch"
},
"files": [
"openapi.json",
@@ -49,8 +50,8 @@
"directory": "packages/cloud"
},
"devDependencies": {
- "@hey-api/client-fetch": "^0.2.4",
- "@hey-api/openapi-ts": "^0.53.0",
+ "@hey-api/client-fetch": "^0.4.2",
+ "@hey-api/openapi-ts": "^0.53.11",
"@llamaindex/core": "workspace:*",
"@llamaindex/env": "workspace:*",
"bunchee": "5.5.1"
diff --git a/packages/cloud/src/reader.ts b/packages/cloud/src/reader.ts
index fad945b09b2e614a33defd080493c1a91e30718f..165aacec8a6b0bd9971ed8fb413a52a453cff703 100644
--- a/packages/cloud/src/reader.ts
+++ b/packages/cloud/src/reader.ts
@@ -84,6 +84,13 @@ export class LlamaParseReader extends FileReader {
disableReconstruction?: boolean | undefined;
inputS3Path?: string | undefined;
outputS3PathPrefix?: string | undefined;
+ continuousMode?: boolean | undefined;
+ isFormattingInstruction?: boolean | undefined;
+ annotateLinks?: boolean | undefined;
+ azureOpenaiDeploymentName?: string | undefined;
+ azureOpenaiEndpoint?: string | undefined;
+ azureOpenaiApiVersion?: string | undefined;
+ azureOpenaiKey?: string | undefined;
// numWorkers is implemented in SimpleDirectoryReader
stdout?: WriteStream | undefined;
@@ -144,13 +151,19 @@ export class LlamaParseReader extends FileReader {
}
// Create a job for the LlamaParse API
- private async createJob(data: Uint8Array): Promise<string> {
+ async #createJob(data: Uint8Array, filename?: string): Promise<string> {
if (this.verbose) {
console.log("Started uploading the file");
}
+ // todo: remove Blob usage when we drop Node.js 18 support
+ const file: File | Blob =
+ globalThis.File && filename
+ ? new File([data], filename)
+ : new Blob([data]);
+
const body = {
- file: new Blob([data]),
+ file,
language: this.language,
parsing_instruction: this.parsingInstruction,
skip_diagonal_text: this.skipDiagonalText,
@@ -175,6 +188,13 @@ export class LlamaParseReader extends FileReader {
disable_reconstruction: this.disableReconstruction,
input_s3_path: this.inputS3Path,
output_s3_path_prefix: this.outputS3PathPrefix,
+ continuous_mode: this.continuousMode,
+ is_formatting_instruction: this.isFormattingInstruction,
+ annotate_links: this.annotateLinks,
+ azure_openai_deployment_name: this.azureOpenaiDeploymentName,
+ azure_openai_endpoint: this.azureOpenaiEndpoint,
+ azure_openai_api_version: this.azureOpenaiApiVersion,
+ azure_openai_key: this.azureOpenaiKey,
} satisfies {
[Key in keyof Body_upload_file_api_v1_parsing_upload_post]-?:
| Body_upload_file_api_v1_parsing_upload_post[Key]
@@ -286,10 +306,14 @@ export class LlamaParseReader extends FileReader {
* To be used with resultType = "text" and "markdown"
*
* @param {Uint8Array} fileContent - The content of the file to be loaded.
+ * @param {string} filename - The name of the file to be loaded.
* @return {Promise<Document[]>} A Promise object that resolves to an array of Document objects.
*/
- async loadDataAsContent(fileContent: Uint8Array): Promise<Document[]> {
- return this.createJob(fileContent)
+ async loadDataAsContent(
+ fileContent: Uint8Array,
+ filename?: string,
+ ): Promise<Document[]> {
+ return this.#createJob(fileContent, filename)
.then(async (jobId) => {
if (this.verbose) {
console.log(`Started parsing the file under job id ${jobId}`);
@@ -312,7 +336,9 @@ export class LlamaParseReader extends FileReader {
})
.catch((error) => {
if (this.ignoreErrors) {
- console.warn(`Error while parsing the file: ${error.message}`);
+ console.warn(
+ `Error while parsing the file: ${error.message ?? error.detail}`,
+ );
return [];
} else {
throw error;
@@ -336,7 +362,10 @@ export class LlamaParseReader extends FileReader {
? await fs.readFile(filePathOrContent)
: filePathOrContent;
// Creates a job for the file
- jobId = await this.createJob(data);
+ jobId = await this.#createJob(
+ data,
+ isFilePath ? path.basename(filePathOrContent) : undefined,
+ );
if (this.verbose) {
console.log(`Started parsing the file under job id ${jobId}`);
}
diff --git a/packages/core/src/schema/type.ts b/packages/core/src/schema/type.ts
index 4c7271d90b3682e4fd96358a1a29eee90840c6ea..3df194fb5e6dcff30c77c1bb20b18a5761ab77f4 100644
--- a/packages/core/src/schema/type.ts
+++ b/packages/core/src/schema/type.ts
@@ -42,13 +42,13 @@ export interface BaseReader {
export abstract class FileReader implements BaseReader {
abstract loadDataAsContent(
fileContent: Uint8Array,
- fileName?: string,
+ filename?: string,
): Promise<Document[]>;
async loadData(filePath: string): Promise<Document[]> {
const fileContent = await fs.readFile(filePath);
- const fileName = path.basename(filePath);
- const docs = await this.loadDataAsContent(fileContent, fileName);
+ const filename = path.basename(filePath);
+ const docs = await this.loadDataAsContent(fileContent, filename);
docs.forEach(FileReader.addMetaData(filePath));
return docs;
}
diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml
index 7dbff0c97dcc02dce70ea5818ec3a7a01a0b8fbb..a6fc3fe832f7155855517ae3873f3c52a0e988a3 100644
--- a/pnpm-lock.yaml
+++ b/pnpm-lock.yaml
@@ -345,11 +345,11 @@ importers:
packages/cloud:
devDependencies:
'@hey-api/client-fetch':
- specifier: ^0.2.4
- version: 0.2.4
+ specifier: ^0.4.2
+ version: 0.4.2
'@hey-api/openapi-ts':
- specifier: ^0.53.0
- version: 0.53.0(typescript@5.6.3)
+ specifier: ^0.53.11
+ version: 0.53.11(typescript@5.6.3)
'@llamaindex/core':
specifier: workspace:*
version: link:../core
@@ -3314,11 +3314,11 @@ packages:
'@hapi/topo@5.1.0':
resolution: {integrity: sha512-foQZKJig7Ob0BMAYBfcJk8d77QtOe7Wo4ox7ff1lQYoNNAb6jwcY1ncdoy2e9wQZzvNy7ODZCYJkK8kzmcAnAg==}
- '@hey-api/client-fetch@0.2.4':
- resolution: {integrity: sha512-SGTVAVw3PlKDLw+IyhNhb/jCH3P1P2xJzLxA8Kyz1g95HrkYOJdRpl9F5I7LLwo9aCIB7nwR2NrSeX7QaQD7vQ==}
+ '@hey-api/client-fetch@0.4.2':
+ resolution: {integrity: sha512-9BqcLTjsM3rWbads3afJkELS86vK7EqJvYgT429EVS9IO/kN75HEka3Ay/k142xCHSfXOuOShMdDam3nbG8wVA==}
- '@hey-api/openapi-ts@0.53.0':
- resolution: {integrity: sha512-5pDd/s0yHJniruYyKYmEsAMbY10Nh/EwhHlgIrdpQ1KZWQdyTbH/tn8rVHT5Mopr1dMuYX0kq0TzpjcNlvrROQ==}
+ '@hey-api/openapi-ts@0.53.11':
+ resolution: {integrity: sha512-PaO+o0jDhfHVS5SjtonP5CzP/NYoW8dVZUn8WthSgzpgPts8AiWYXplOyk5uEnM4ZxbkZbeTiREwaNLnJmXlTQ==}
engines: {node: ^18.0.0 || >=20.0.0}
hasBin: true
peerDependencies:
@@ -5941,10 +5941,10 @@ packages:
resolution: {integrity: sha512-/Nf7TyzTx6S3yRJObOAV7956r8cr2+Oj8AC5dt8wSP3BQAoeX58NoHyCU8P8zGkNXStjTSi6fzO6F0pBdcYbEg==}
engines: {node: '>= 0.8'}
- c12@1.11.1:
- resolution: {integrity: sha512-KDU0TvSvVdaYcQKQ6iPHATGz/7p/KiVjPg4vQrB6Jg/wX9R0yl5RZxWm9IoZqaIHD2+6PZd81+KMGwRr/lRIUg==}
+ c12@2.0.1:
+ resolution: {integrity: sha512-Z4JgsKXHG37C6PYUtIxCfLJZvo6FyhHJoClwwb9ftUkLpPSkuYqn6Tr+vnaN8hymm0kIbcg6Ey3kv/Q71k5w/A==}
peerDependencies:
- magicast: ^0.3.4
+ magicast: ^0.3.5
peerDependenciesMeta:
magicast:
optional: true
@@ -6077,6 +6077,10 @@ packages:
resolution: {integrity: sha512-7VT13fmjotKpGipCW9JEQAusEPE+Ei8nl6/g4FBAmIm0GOOLMua9NDDo/DWp0ZAxCr3cPq5ZpBqmPAQgDda2Pw==}
engines: {node: '>= 8.10.0'}
+ chokidar@4.0.1:
+ resolution: {integrity: sha512-n8enUVCED/KVRQlab1hr3MVpcVMvxtZjmEa956u+4YijlmQED223XMSYj2tLuKvr4jcCTzNNMpQDUer72MMmzA==}
+ engines: {node: '>= 14.16.0'}
+
chownr@1.1.4:
resolution: {integrity: sha512-jJ0bqzaylmJtVnNgzTeSOs8DPavpbYgEr/b0YL8/2GO3xJEhInFmhKMUnEJQjZumK7KXGFhUy89PrsJWlakBVg==}
@@ -8389,6 +8393,10 @@ packages:
resolution: {integrity: sha512-2yTgeWTWzMWkHu6Jp9NKgePDaYHbntiwvYuuJLbbN9vl7DC9DvXKOB2BC3ZZ92D3cvV/aflH0osDfwpHepQ53w==}
hasBin: true
+ jiti@2.3.3:
+ resolution: {integrity: sha512-EX4oNDwcXSivPrw2qKH2LB5PoFxEvgtv2JgwW0bU858HoLQ+kutSvjLMUqBd0PeJYEinLWhoI9Ol0eYMqj/wNQ==}
+ hasBin: true
+
joi@17.13.3:
resolution: {integrity: sha512-otDA4ldcIx+ZXsKHWmp0YizCweVRZG96J10b0FevjfuncLO1oX59THoAmHkNubYJ+9gWsYsp5k8v4ib6oDv1fA==}
@@ -9462,9 +9470,6 @@ packages:
obuf@1.1.2:
resolution: {integrity: sha512-PX1wu0AmAdPqOL1mWhqmlOd8kOIZQwGZw6rh7uby9fTc5lhaOWFLX3I6R1hrF9k3zUY40e6igsLGkDXK92LJNg==}
- ohash@1.1.3:
- resolution: {integrity: sha512-zuHHiGTYTA1sYJ/wZN+t5HKZaH23i4yI1HMwbuXm24Nid7Dv0KcuRlKoNKS9UNfAVSBlnGLcuQrnOKWOZoEGaw==}
-
ohash@1.1.4:
resolution: {integrity: sha512-FlDryZAahJmEF3VR3w1KogSEdWX3WhA5GPakFx4J81kEAiHyLMpdLLElS8n8dfNadMgAne/MywcvmogzscVt4g==}
@@ -10522,6 +10527,10 @@ packages:
resolution: {integrity: sha512-hOS089on8RduqdbhvQ5Z37A0ESjsqz6qnRcffsMU3495FuTdqSm+7bhJ29JvIOsBDEEnan5DPu9t3To9VRlMzA==}
engines: {node: '>=8.10.0'}
+ readdirp@4.0.2:
+ resolution: {integrity: sha512-yDMz9g+VaZkqBYS/ozoBJwaBhTbZo3UNYQHNRw1D3UFQB8oHB4uS/tAODO+ZLjGWmUbKnIlOWO+aaIiAxrUWHA==}
+ engines: {node: '>= 14.16.0'}
+
reading-time@1.5.0:
resolution: {integrity: sha512-onYyVhBNr4CmAxFsKS7bz+uTLRakypIe4R+5A824vBSkQy/hB3fZepoVEf8OVAxzLvK+H/jm9TzpI3ETSm64Kg==}
@@ -16694,12 +16703,12 @@ snapshots:
dependencies:
'@hapi/hoek': 9.3.0
- '@hey-api/client-fetch@0.2.4': {}
+ '@hey-api/client-fetch@0.4.2': {}
- '@hey-api/openapi-ts@0.53.0(typescript@5.6.3)':
+ '@hey-api/openapi-ts@0.53.11(typescript@5.6.3)':
dependencies:
'@apidevtools/json-schema-ref-parser': 11.7.0
- c12: 1.11.1
+ c12: 2.0.1
commander: 12.1.0
handlebars: 4.7.8
typescript: 5.6.3
@@ -19886,16 +19895,16 @@ snapshots:
bytes@3.1.2: {}
- c12@1.11.1:
+ c12@2.0.1:
dependencies:
- chokidar: 3.6.0
+ chokidar: 4.0.1
confbox: 0.1.7
defu: 6.1.4
dotenv: 16.4.5
giget: 1.2.3
- jiti: 1.21.6
+ jiti: 2.3.3
mlly: 1.7.1
- ohash: 1.1.3
+ ohash: 1.1.4
pathe: 1.1.2
perfect-debounce: 1.0.0
pkg-types: 1.2.0
@@ -20054,6 +20063,10 @@ snapshots:
optionalDependencies:
fsevents: 2.3.3
+ chokidar@4.0.1:
+ dependencies:
+ readdirp: 4.0.2
+
chownr@1.1.4: {}
chownr@2.0.0: {}
@@ -21121,7 +21134,7 @@ snapshots:
'@typescript-eslint/parser': 7.2.0(eslint@8.57.0)(typescript@5.6.2)
eslint: 8.57.0
eslint-import-resolver-node: 0.3.9
- eslint-import-resolver-typescript: 3.6.3(@typescript-eslint/parser@7.2.0(eslint@8.57.0)(typescript@5.6.2))(eslint-import-resolver-node@0.3.9)(eslint-plugin-import@2.29.1(@typescript-eslint/parser@7.2.0(eslint@8.57.0)(typescript@5.6.2))(eslint@8.57.0))(eslint@8.57.0)
+ eslint-import-resolver-typescript: 3.6.3(@typescript-eslint/parser@7.2.0(eslint@8.57.0)(typescript@5.6.2))(eslint-import-resolver-node@0.3.9)(eslint-plugin-import@2.29.1)(eslint@8.57.0)
eslint-plugin-import: 2.29.1(@typescript-eslint/parser@7.2.0(eslint@8.57.0)(typescript@5.6.2))(eslint-import-resolver-typescript@3.6.3)(eslint@8.57.0)
eslint-plugin-jsx-a11y: 6.9.0(eslint@8.57.0)
eslint-plugin-react: 7.35.0(eslint@8.57.0)
@@ -21169,25 +21182,6 @@ snapshots:
transitivePeerDependencies:
- supports-color
- eslint-import-resolver-typescript@3.6.3(@typescript-eslint/parser@7.2.0(eslint@8.57.0)(typescript@5.6.2))(eslint-import-resolver-node@0.3.9)(eslint-plugin-import@2.29.1(@typescript-eslint/parser@7.2.0(eslint@8.57.0)(typescript@5.6.2))(eslint@8.57.0))(eslint@8.57.0):
- dependencies:
- '@nolyfill/is-core-module': 1.0.39
- debug: 4.3.7
- enhanced-resolve: 5.17.1
- eslint: 8.57.0
- eslint-module-utils: 2.8.2(@typescript-eslint/parser@7.2.0(eslint@8.57.0)(typescript@5.6.2))(eslint-import-resolver-node@0.3.9)(eslint-import-resolver-typescript@3.6.3(@typescript-eslint/parser@7.2.0(eslint@8.57.0)(typescript@5.6.2))(eslint-import-resolver-node@0.3.9)(eslint-plugin-import@2.29.1(@typescript-eslint/parser@7.2.0(eslint@8.57.0)(typescript@5.6.2))(eslint@8.57.0))(eslint@8.57.0))(eslint@8.57.0)
- fast-glob: 3.3.2
- get-tsconfig: 4.8.0
- is-bun-module: 1.1.0
- is-glob: 4.0.3
- optionalDependencies:
- eslint-plugin-import: 2.29.1(@typescript-eslint/parser@7.2.0(eslint@8.57.0)(typescript@5.6.2))(eslint-import-resolver-typescript@3.6.3)(eslint@8.57.0)
- transitivePeerDependencies:
- - '@typescript-eslint/parser'
- - eslint-import-resolver-node
- - eslint-import-resolver-webpack
- - supports-color
-
eslint-import-resolver-typescript@3.6.3(@typescript-eslint/parser@7.2.0(eslint@8.57.0)(typescript@5.6.2))(eslint-import-resolver-node@0.3.9)(eslint-plugin-import@2.29.1)(eslint@8.57.0):
dependencies:
'@nolyfill/is-core-module': 1.0.39
@@ -21207,17 +21201,6 @@ snapshots:
- eslint-import-resolver-webpack
- supports-color
- eslint-module-utils@2.8.2(@typescript-eslint/parser@7.2.0(eslint@8.57.0)(typescript@5.6.2))(eslint-import-resolver-node@0.3.9)(eslint-import-resolver-typescript@3.6.3(@typescript-eslint/parser@7.2.0(eslint@8.57.0)(typescript@5.6.2))(eslint-import-resolver-node@0.3.9)(eslint-plugin-import@2.29.1(@typescript-eslint/parser@7.2.0(eslint@8.57.0)(typescript@5.6.2))(eslint@8.57.0))(eslint@8.57.0))(eslint@8.57.0):
- dependencies:
- debug: 3.2.7
- optionalDependencies:
- '@typescript-eslint/parser': 7.2.0(eslint@8.57.0)(typescript@5.6.2)
- eslint: 8.57.0
- eslint-import-resolver-node: 0.3.9
- eslint-import-resolver-typescript: 3.6.3(@typescript-eslint/parser@7.2.0(eslint@8.57.0)(typescript@5.6.2))(eslint-import-resolver-node@0.3.9)(eslint-plugin-import@2.29.1(@typescript-eslint/parser@7.2.0(eslint@8.57.0)(typescript@5.6.2))(eslint@8.57.0))(eslint@8.57.0)
- transitivePeerDependencies:
- - supports-color
-
eslint-module-utils@2.8.2(@typescript-eslint/parser@7.2.0(eslint@8.57.0)(typescript@5.6.2))(eslint-import-resolver-node@0.3.9)(eslint-import-resolver-typescript@3.6.3(@typescript-eslint/parser@7.2.0(eslint@8.57.0)(typescript@5.6.2))(eslint-import-resolver-node@0.3.9)(eslint-plugin-import@2.29.1)(eslint@8.57.0))(eslint@8.57.0):
dependencies:
debug: 3.2.7
@@ -21239,7 +21222,7 @@ snapshots:
doctrine: 2.1.0
eslint: 8.57.0
eslint-import-resolver-node: 0.3.9
- eslint-module-utils: 2.8.2(@typescript-eslint/parser@7.2.0(eslint@8.57.0)(typescript@5.6.2))(eslint-import-resolver-node@0.3.9)(eslint-import-resolver-typescript@3.6.3(@typescript-eslint/parser@7.2.0(eslint@8.57.0)(typescript@5.6.2))(eslint-import-resolver-node@0.3.9)(eslint-plugin-import@2.29.1(@typescript-eslint/parser@7.2.0(eslint@8.57.0)(typescript@5.6.2))(eslint@8.57.0))(eslint@8.57.0))(eslint@8.57.0)
+ eslint-module-utils: 2.8.2(@typescript-eslint/parser@7.2.0(eslint@8.57.0)(typescript@5.6.2))(eslint-import-resolver-node@0.3.9)(eslint-import-resolver-typescript@3.6.3(@typescript-eslint/parser@7.2.0(eslint@8.57.0)(typescript@5.6.2))(eslint-import-resolver-node@0.3.9)(eslint-plugin-import@2.29.1)(eslint@8.57.0))(eslint@8.57.0)
hasown: 2.0.2
is-core-module: 2.15.1
is-glob: 4.0.3
@@ -22924,6 +22907,8 @@ snapshots:
jiti@1.21.6: {}
+ jiti@2.3.3: {}
+
joi@17.13.3:
dependencies:
'@hapi/hoek': 9.3.0
@@ -24408,8 +24393,6 @@ snapshots:
obuf@1.1.2: {}
- ohash@1.1.3: {}
-
ohash@1.1.4: {}
ollama@0.5.9:
@@ -25552,6 +25535,8 @@ snapshots:
dependencies:
picomatch: 2.3.1
+ readdirp@4.0.2: {}
+
reading-time@1.5.0: {}
rechoir@0.6.2: