Skip to content
Snippets Groups Projects
Unverified Commit 5189b446 authored by George He's avatar George He Committed by GitHub
Browse files

fix: add retry handling logic to parser reader and fix lint issues (#1757) 


Co-authored-by: default avatarAlex Yang <himself65@outlook.com>
parent c7ff3233
No related branches found
No related tags found
No related merge requests found
Expand all Hide whitespace changes
Inline Side-by-side
Showing
with 269 additions and 137 deletions
.changeset/thirty-hats-act.md 0 → 100644
+ 8
− 0
View file @ 5189b446
---
"@llamaindex/cloud": patch
"@llamaindex/community": patch
"@llamaindex/core": patch
"@llamaindex/readers": patch
---
fix: add retry handling logic to parser reader and fix lint issues
eslint.config.mjs
+ 3
− 0
View file @ 5189b446
......@@ -50,6 +50,9 @@ export default tseslint.config(
"**/lib/*",
"**/deps/**",
"**/.next/**",
"**/.source/**", // Ignore .source directories
"!.git", // Don't ignore .git directory
"**/.*", // Ignore all dot files and directories
"**/node_modules/**",
"**/build/**",
"**/.docusaurus/**",
......
package.json
+ 3
− 0
View file @ 5189b446
......@@ -46,5 +46,8 @@
"*.{json,md,yml}": [
"prettier --write"
]
},
"dependencies": {
"p-retry": "^6.2.1"
}
}
packages/cloud/src/reader.ts
+ 203
− 118
View file @ 5189b446
This diff is collapsed.
packages/community/src/llm/bedrock/amazon/provider.ts
+ 2
− 1
View file @ 5189b446
......@@ -24,6 +24,7 @@ import {
} from "./utils";
export class AmazonProvider extends Provider<ConverseStreamOutput> {
// eslint-disable-next-line @typescript-eslint/no-explicit-any
getResultFromResponse(response: Record<string, any>): ConverseResponse {
return JSON.parse(toUtf8(response.body));
}
......@@ -52,7 +53,7 @@ export class AmazonProvider extends Provider<ConverseStreamOutput> {
}
getTextFromStreamResponse(response: ResponseStream): string {
let event: ConverseStreamOutput | undefined =
const event: ConverseStreamOutput | undefined =
this.getStreamingEventResponse(response);
if (!event || !event.contentBlockDelta) return "";
const delta: ContentBlockDelta | undefined = event.contentBlockDelta.delta;
......
packages/community/src/llm/bedrock/amazon/utils.ts
+ 1
− 1
View file @ 5189b446
......@@ -56,7 +56,7 @@ export const mapImageContent = (imageUrl: string): ImageBlock => {
mimeType as keyof typeof ACCEPTED_IMAGE_MIME_TYPE_FORMAT_MAP
],
// @ts-ignore: there's a mistake in the "@aws-sdk/client-bedrock-runtime" compared to the actual api
// @ts-expect-error: there's a mistake in the "@aws-sdk/client-bedrock-runtime" compared to the actual api
source: { bytes: data },
};
};
......
packages/core/src/global/constants.ts
+ 1
− 0
View file @ 5189b446
......@@ -20,4 +20,5 @@ export const DEFAULT_NAMESPACE = "docstore";
//#region llama cloud
export const DEFAULT_PROJECT_NAME = "Default";
export const DEFAULT_BASE_URL = "https://api.cloud.llamaindex.ai";
export const DEFAULT_EU_BASE_URL = "https://api.cloud.eu.llamaindex.ai";
//#endregion
packages/readers/src/cosmosdb.ts
+ 1
− 1
View file @ 5189b446
......@@ -60,7 +60,7 @@ export class SimpleCosmosDBReader implements BaseReader {
const metadataFields = config.metadataFields;
try {
let res = await container.items.query(query).fetchAll();
const res = await container.items.query(query).fetchAll();
const documents: Document[] = [];
for (const item of res.resources) {
......
pnpm-lock.yaml
+ 47
− 16
View file @ 5189b446
......@@ -7,6 +7,10 @@ settings:
importers:
 
.:
dependencies:
p-retry:
specifier: ^6.2.1
version: 6.2.1
devDependencies:
'@changesets/cli':
specifier: ^2.27.5
......@@ -5528,6 +5532,9 @@ packages:
'@types/resolve@1.20.2':
resolution: {integrity: sha512-60BCwRFOZCQhDncwQdxxeOEEkbc5dIMccYLwbxsS4TUNeVECQ/pBJ0j09mrHOl/JJvpRPGwO9SvE4nR2Nb/a4Q==}
 
'@types/retry@0.12.2':
resolution: {integrity: sha512-XISRgDJ2Tc5q4TRqvgJtzsRkFYNJzZrhTdtMoGVBttwzzQJkPnS3WWTFc7kuDRoPtPakl+T+OfdEUjYJj7Jbow==}
'@types/statuses@2.0.5':
resolution: {integrity: sha512-jmIUGWrAiwu3dZpxntxieC+1n/5c3mjrImkmOSQ2NC5uP6cYO4aAZDdSmRcI5C1oiTmqlZGHC+/NmJrKogbP5A==}
 
......@@ -8261,6 +8268,10 @@ packages:
is-module@1.0.0:
resolution: {integrity: sha512-51ypPSPCoTEIN9dy5Oy+h4pShgJmPCygKfyRCISBI+JoWT/2oJvK8QPxmwv7b/p239jXrm9M1mlQbyKJ5A152g==}
 
is-network-error@1.1.0:
resolution: {integrity: sha512-tUdRRAnhT+OtCZR/LxZelH/C7QtjtFrTu5tXCA8pl55eTUElUHT+GPYV8MBMBvea/j+NxQqVt3LbWMRir7Gx9g==}
engines: {node: '>=16'}
is-node-process@1.2.0:
resolution: {integrity: sha512-Vg4o6/fqPxIjtxgUH5QLJhwZ7gW5diGCVlXpuUfELC62CuxM1iHcRe51f2W1FDy04Ai4KJkagKjx3XaqyfRKXw==}
 
......@@ -9826,6 +9837,10 @@ packages:
resolution: {integrity: sha512-y3b8Kpd8OAN444hxfBbFfj1FY/RjtTd8tzYwhUqNYXx0fXx2iX4maP4Qr6qhIKbQXI02wTLAda4fYUbDagTUFw==}
engines: {node: '>=6'}
 
p-retry@6.2.1:
resolution: {integrity: sha512-hEt02O4hUct5wtwg4H4KcWgDdm+l1bOaEy/hWzd8xtXB9BqxTWBBhb+2ImAtH4Cv4rPjV76xN3Zumqk3k3AhhQ==}
engines: {node: '>=16.17'}
p-try@2.2.0:
resolution: {integrity: sha512-R4nPAVTAU0B9D35/Gk3uJf/7XYbQcyohSKdvAxIRSNghFl4e71hVoGnBNQz9cWaXxO2I10KTC+3jMdvvoKw6dQ==}
engines: {node: '>=6'}
......@@ -10676,6 +10691,10 @@ packages:
resolution: {integrity: sha512-dUOvLMJ0/JJYEn8NrpOaGNE7X3vpI5XlZS/u0ANjqtcZVKnIxP7IgCFwrKTxENw29emmwug53awKtaMm4i9g5w==}
engines: {node: '>=14'}
 
retry@0.13.1:
resolution: {integrity: sha512-XQBQ3I8W1Cge0Seh+6gjj03LbmRFWuoszgK9ooCpwYIrhhoO80pfq4cUkU5DkknwfOfFteRwlZ56PYOGYyFWdg==}
engines: {node: '>= 4'}
reusify@1.0.4:
resolution: {integrity: sha512-U9nH88a3fc/ekCF1l0/UP1IosiuIjyTh7hBvXVMHYgVcfGvt897Xguj2UOLDeI5BG2m7/uwyaLVT6fbtCwTyzw==}
engines: {iojs: '>=1.0.0', node: '>=0.10.0'}
......@@ -16604,6 +16623,8 @@ snapshots:
 
'@types/resolve@1.20.2': {}
 
'@types/retry@0.12.2': {}
'@types/statuses@2.0.5': {}
 
'@types/tough-cookie@4.0.5': {}
......@@ -16629,10 +16650,10 @@ snapshots:
'@types/node': 22.9.0
optional: true
 
'@typescript-eslint/eslint-plugin@8.24.0(@typescript-eslint/parser@8.24.0(eslint@9.16.0(jiti@2.4.2))(typescript@5.7.3))(eslint@9.16.0(jiti@2.4.2))(typescript@5.7.3)':
'@typescript-eslint/eslint-plugin@8.24.0(@typescript-eslint/parser@8.24.0(eslint@9.22.0(jiti@2.4.2))(typescript@5.7.3))(eslint@9.16.0(jiti@2.4.2))(typescript@5.7.3)':
dependencies:
'@eslint-community/regexpp': 4.12.1
'@typescript-eslint/parser': 8.24.0(eslint@9.16.0(jiti@2.4.2))(typescript@5.7.3)
'@typescript-eslint/parser': 8.24.0(eslint@9.22.0(jiti@2.4.2))(typescript@5.7.3)
'@typescript-eslint/scope-manager': 8.24.0
'@typescript-eslint/type-utils': 8.24.0(eslint@9.16.0(jiti@2.4.2))(typescript@5.7.3)
'@typescript-eslint/utils': 8.24.0(eslint@9.16.0(jiti@2.4.2))(typescript@5.7.3)
......@@ -18550,12 +18571,12 @@ snapshots:
dependencies:
'@next/eslint-plugin-next': 15.1.0
'@rushstack/eslint-patch': 1.10.5
'@typescript-eslint/eslint-plugin': 8.24.0(@typescript-eslint/parser@8.24.0(eslint@9.16.0(jiti@2.4.2))(typescript@5.7.3))(eslint@9.16.0(jiti@2.4.2))(typescript@5.7.3)
'@typescript-eslint/eslint-plugin': 8.24.0(@typescript-eslint/parser@8.24.0(eslint@9.22.0(jiti@2.4.2))(typescript@5.7.3))(eslint@9.16.0(jiti@2.4.2))(typescript@5.7.3)
'@typescript-eslint/parser': 8.24.0(eslint@9.16.0(jiti@2.4.2))(typescript@5.7.3)
eslint: 9.16.0(jiti@2.4.2)
eslint-import-resolver-node: 0.3.9
eslint-import-resolver-typescript: 3.7.0(eslint-plugin-import@2.31.0)(eslint@9.16.0(jiti@2.4.2))
eslint-plugin-import: 2.31.0(@typescript-eslint/parser@8.24.0(eslint@9.16.0(jiti@2.4.2))(typescript@5.7.3))(eslint@9.16.0(jiti@2.4.2))
eslint-import-resolver-typescript: 3.7.0(eslint-plugin-import@2.31.0(@typescript-eslint/parser@8.24.0(eslint@9.22.0(jiti@2.4.2))(typescript@5.7.3))(eslint@9.22.0(jiti@2.4.2)))(eslint@9.16.0(jiti@2.4.2))
eslint-plugin-import: 2.31.0(@typescript-eslint/parser@8.24.0(eslint@9.22.0(jiti@2.4.2))(typescript@5.7.3))(eslint-import-resolver-typescript@3.7.0(eslint-plugin-import@2.31.0(@typescript-eslint/parser@8.24.0(eslint@9.22.0(jiti@2.4.2))(typescript@5.7.3))(eslint@9.22.0(jiti@2.4.2)))(eslint@9.16.0(jiti@2.4.2)))(eslint@9.16.0(jiti@2.4.2))
eslint-plugin-jsx-a11y: 6.10.2(eslint@9.16.0(jiti@2.4.2))
eslint-plugin-react: 7.37.2(eslint@9.16.0(jiti@2.4.2))
eslint-plugin-react-hooks: 5.1.0(eslint@9.16.0(jiti@2.4.2))
......@@ -18575,7 +18596,7 @@ snapshots:
eslint: 9.22.0(jiti@2.4.2)
eslint-import-resolver-node: 0.3.9
eslint-import-resolver-typescript: 3.7.0(eslint-plugin-import@2.31.0)(eslint@9.22.0(jiti@2.4.2))
eslint-plugin-import: 2.31.0(@typescript-eslint/parser@8.24.0(eslint@9.22.0(jiti@2.4.2))(typescript@5.7.3))(eslint-import-resolver-typescript@3.7.0)(eslint@9.22.0(jiti@2.4.2))
eslint-plugin-import: 2.31.0(@typescript-eslint/parser@8.24.0(eslint@9.22.0(jiti@2.4.2))(typescript@5.7.3))(eslint@9.22.0(jiti@2.4.2))
eslint-plugin-jsx-a11y: 6.10.2(eslint@9.22.0(jiti@2.4.2))
eslint-plugin-react: 7.37.2(eslint@9.22.0(jiti@2.4.2))
eslint-plugin-react-hooks: 5.1.0(eslint@9.22.0(jiti@2.4.2))
......@@ -18604,7 +18625,7 @@ snapshots:
transitivePeerDependencies:
- supports-color
 
eslint-import-resolver-typescript@3.7.0(eslint-plugin-import@2.31.0)(eslint@9.16.0(jiti@2.4.2)):
eslint-import-resolver-typescript@3.7.0(eslint-plugin-import@2.31.0(@typescript-eslint/parser@8.24.0(eslint@9.22.0(jiti@2.4.2))(typescript@5.7.3))(eslint@9.22.0(jiti@2.4.2)))(eslint@9.16.0(jiti@2.4.2)):
dependencies:
'@nolyfill/is-core-module': 1.0.39
debug: 4.4.0
......@@ -18616,7 +18637,7 @@ snapshots:
is-glob: 4.0.3
stable-hash: 0.0.4
optionalDependencies:
eslint-plugin-import: 2.31.0(@typescript-eslint/parser@8.24.0(eslint@9.16.0(jiti@2.4.2))(typescript@5.7.3))(eslint@9.16.0(jiti@2.4.2))
eslint-plugin-import: 2.31.0(@typescript-eslint/parser@8.24.0(eslint@9.22.0(jiti@2.4.2))(typescript@5.7.3))(eslint@9.22.0(jiti@2.4.2))
transitivePeerDependencies:
- supports-color
 
......@@ -18632,18 +18653,18 @@ snapshots:
is-glob: 4.0.3
stable-hash: 0.0.4
optionalDependencies:
eslint-plugin-import: 2.31.0(@typescript-eslint/parser@8.24.0(eslint@9.22.0(jiti@2.4.2))(typescript@5.7.3))(eslint-import-resolver-typescript@3.7.0)(eslint@9.22.0(jiti@2.4.2))
eslint-plugin-import: 2.31.0(@typescript-eslint/parser@8.24.0(eslint@9.22.0(jiti@2.4.2))(typescript@5.7.3))(eslint@9.22.0(jiti@2.4.2))
transitivePeerDependencies:
- supports-color
 
eslint-module-utils@2.12.0(@typescript-eslint/parser@8.24.0(eslint@9.16.0(jiti@2.4.2))(typescript@5.7.3))(eslint-import-resolver-node@0.3.9)(eslint-import-resolver-typescript@3.7.0)(eslint@9.16.0(jiti@2.4.2)):
eslint-module-utils@2.12.0(@typescript-eslint/parser@8.24.0(eslint@9.22.0(jiti@2.4.2))(typescript@5.7.3))(eslint-import-resolver-node@0.3.9)(eslint-import-resolver-typescript@3.7.0(eslint-plugin-import@2.31.0(@typescript-eslint/parser@8.24.0(eslint@9.22.0(jiti@2.4.2))(typescript@5.7.3))(eslint@9.22.0(jiti@2.4.2)))(eslint@9.16.0(jiti@2.4.2)))(eslint@9.16.0(jiti@2.4.2)):
dependencies:
debug: 3.2.7
optionalDependencies:
'@typescript-eslint/parser': 8.24.0(eslint@9.16.0(jiti@2.4.2))(typescript@5.7.3)
'@typescript-eslint/parser': 8.24.0(eslint@9.22.0(jiti@2.4.2))(typescript@5.7.3)
eslint: 9.16.0(jiti@2.4.2)
eslint-import-resolver-node: 0.3.9
eslint-import-resolver-typescript: 3.7.0(eslint-plugin-import@2.31.0)(eslint@9.16.0(jiti@2.4.2))
eslint-import-resolver-typescript: 3.7.0(eslint-plugin-import@2.31.0(@typescript-eslint/parser@8.24.0(eslint@9.22.0(jiti@2.4.2))(typescript@5.7.3))(eslint@9.22.0(jiti@2.4.2)))(eslint@9.16.0(jiti@2.4.2))
transitivePeerDependencies:
- supports-color
 
......@@ -18658,7 +18679,7 @@ snapshots:
transitivePeerDependencies:
- supports-color
 
eslint-plugin-import@2.31.0(@typescript-eslint/parser@8.24.0(eslint@9.16.0(jiti@2.4.2))(typescript@5.7.3))(eslint@9.16.0(jiti@2.4.2)):
eslint-plugin-import@2.31.0(@typescript-eslint/parser@8.24.0(eslint@9.22.0(jiti@2.4.2))(typescript@5.7.3))(eslint-import-resolver-typescript@3.7.0(eslint-plugin-import@2.31.0(@typescript-eslint/parser@8.24.0(eslint@9.22.0(jiti@2.4.2))(typescript@5.7.3))(eslint@9.22.0(jiti@2.4.2)))(eslint@9.16.0(jiti@2.4.2)))(eslint@9.16.0(jiti@2.4.2)):
dependencies:
'@rtsao/scc': 1.1.0
array-includes: 3.1.8
......@@ -18669,7 +18690,7 @@ snapshots:
doctrine: 2.1.0
eslint: 9.16.0(jiti@2.4.2)
eslint-import-resolver-node: 0.3.9
eslint-module-utils: 2.12.0(@typescript-eslint/parser@8.24.0(eslint@9.16.0(jiti@2.4.2))(typescript@5.7.3))(eslint-import-resolver-node@0.3.9)(eslint-import-resolver-typescript@3.7.0)(eslint@9.16.0(jiti@2.4.2))
eslint-module-utils: 2.12.0(@typescript-eslint/parser@8.24.0(eslint@9.22.0(jiti@2.4.2))(typescript@5.7.3))(eslint-import-resolver-node@0.3.9)(eslint-import-resolver-typescript@3.7.0(eslint-plugin-import@2.31.0(@typescript-eslint/parser@8.24.0(eslint@9.22.0(jiti@2.4.2))(typescript@5.7.3))(eslint@9.22.0(jiti@2.4.2)))(eslint@9.16.0(jiti@2.4.2)))(eslint@9.16.0(jiti@2.4.2))
hasown: 2.0.2
is-core-module: 2.16.1
is-glob: 4.0.3
......@@ -18681,13 +18702,13 @@ snapshots:
string.prototype.trimend: 1.0.9
tsconfig-paths: 3.15.0
optionalDependencies:
'@typescript-eslint/parser': 8.24.0(eslint@9.16.0(jiti@2.4.2))(typescript@5.7.3)
'@typescript-eslint/parser': 8.24.0(eslint@9.22.0(jiti@2.4.2))(typescript@5.7.3)
transitivePeerDependencies:
- eslint-import-resolver-typescript
- eslint-import-resolver-webpack
- supports-color
 
eslint-plugin-import@2.31.0(@typescript-eslint/parser@8.24.0(eslint@9.22.0(jiti@2.4.2))(typescript@5.7.3))(eslint-import-resolver-typescript@3.7.0)(eslint@9.22.0(jiti@2.4.2)):
eslint-plugin-import@2.31.0(@typescript-eslint/parser@8.24.0(eslint@9.22.0(jiti@2.4.2))(typescript@5.7.3))(eslint@9.22.0(jiti@2.4.2)):
dependencies:
'@rtsao/scc': 1.1.0
array-includes: 3.1.8
......@@ -20303,6 +20324,8 @@ snapshots:
 
is-module@1.0.0: {}
 
is-network-error@1.1.0: {}
is-node-process@1.2.0: {}
 
is-number-object@1.1.1:
......@@ -22395,6 +22418,12 @@ snapshots:
 
p-map@2.1.0: {}
 
p-retry@6.2.1:
dependencies:
'@types/retry': 0.12.2
is-network-error: 1.1.0
retry: 0.13.1
p-try@2.2.0: {}
 
pac-proxy-agent@7.1.0:
......@@ -23393,6 +23422,8 @@ snapshots:
- encoding
- supports-color
 
retry@0.13.1: {}
reusify@1.0.4: {}
 
rfdc@1.4.1: {}
......
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment

:feet::heart: Strive to be the person your dogs believe you are :heart::feet: