Skip to content
Snippets Groups Projects
Unverified Commit 58791d4b authored by Alex Yang's avatar Alex Yang Committed by GitHub
Browse files

fix: tokenizer type (#934)

parent d3b635b1
No related branches found
No related tags found
No related merge requests found
...@@ -25,7 +25,7 @@ class TokenizerSingleton { ...@@ -25,7 +25,7 @@ class TokenizerSingleton {
}; };
} }
tokenizer(encoding?: Tokenizers) { tokenizer(encoding?: Tokenizers): Tokenizer {
if (encoding && encoding !== Tokenizers.CL100K_BASE) { if (encoding && encoding !== Tokenizers.CL100K_BASE) {
throw new Error(`Tokenizer encoding ${encoding} not yet supported`); throw new Error(`Tokenizer encoding ${encoding} not yet supported`);
} }
...@@ -34,5 +34,5 @@ class TokenizerSingleton { ...@@ -34,5 +34,5 @@ class TokenizerSingleton {
} }
} }
export const tokenizers = new TokenizerSingleton(); export const tokenizers: TokenizerSingleton = new TokenizerSingleton();
export { Tokenizers, type Tokenizer }; export { Tokenizers, type Tokenizer };
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment