feat: truncate embedding tokens (#918)

.changeset/giant-buses-breathe.md

.github/workflows/test.yml

packages/core/e2e/fixtures/embeddings/OpenAIEmbedding.ts

packages/core/package.json

packages/core/src/ChatHistory.ts

packages/core/src/PromptHelper.ts

packages/core/src/TextSplitter.ts

packages/core/src/embeddings/OpenAIEmbedding.ts

packages/core/src/embeddings/tokenizer.ts

packages/core/src/embeddings/types.ts

packages/core/src/index.edge.ts

packages/core/src/llm/openai.ts

packages/core/src/llm/types.ts

packages/core/tests/embeddings/tokenizer.test.ts

packages/env/package.json

packages/env/src/index.edge-light.ts

packages/env/src/index.ts

packages/env/src/index.workerd.ts

packages/env/src/tokenizers/js.ts

packages/env/src/tokenizers/node.ts