From 18171bc7f47805e6f1166e8ef3d0010b7b13029d Mon Sep 17 00:00:00 2001
From: Timothy Carambat <rambat1010@gmail.com>
Date: Mon, 22 Jul 2024 12:56:30 -0700
Subject: [PATCH] patch text.substring bug from compressor (#1918)
---
server/utils/helpers/tiktoken.js | 9 +++++++--
1 file changed, 7 insertions(+), 2 deletions(-)
diff --git a/server/utils/helpers/tiktoken.js b/server/utils/helpers/tiktoken.js
index c5852892f..a3fa3b639 100644
--- a/server/utils/helpers/tiktoken.js
+++ b/server/utils/helpers/tiktoken.js
@@ -19,8 +19,13 @@ class TokenManager {
// https://github.com/openai/tiktoken/blob/9e79899bc248d5313c7dd73562b5e211d728723d/tiktoken/core.py#L91C20-L91C38
// Returns number[]
tokensFromString(input = "") {
- const tokens = this.encoder.encode(input, undefined, []);
- return tokens;
+ try {
+ const tokens = this.encoder.encode(String(input), undefined, []);
+ return tokens;
+ } catch (e) {
+ console.error(e);
+ return [];
+ }
}
bytesFromTokens(tokens = []) {
--
GitLab