diff --git a/server/utils/EmbeddingEngines/azureOpenAi/index.js b/server/utils/EmbeddingEngines/azureOpenAi/index.js
index 8cde1fc7cac4898f975a82f20279d522ba938a38..e80b4b734b3a852a6485d9c1e28454c453d498c3 100644
--- a/server/utils/EmbeddingEngines/azureOpenAi/index.js
+++ b/server/utils/EmbeddingEngines/azureOpenAi/index.js
@@ -13,7 +13,6 @@ class AzureOpenAiEmbedder {
new AzureKeyCredential(process.env.AZURE_OPENAI_KEY)
);
this.openai = openai;
- this.dimensions = 1536;
// Limit of how many strings we can process in a single pass to stay with resource or network limits
// https://learn.microsoft.com/en-us/azure/ai-services/openai/faq#i-am-trying-to-use-embeddings-and-received-the-error--invalidrequesterror--too-many-inputs--the-max-number-of-inputs-is-1---how-do-i-fix-this-:~:text=consisting%20of%20up%20to%2016%20inputs%20per%20API%20request
diff --git a/server/utils/EmbeddingEngines/localAi/index.js b/server/utils/EmbeddingEngines/localAi/index.js
index 6f9d721b9bc0bc8f557e3917debda6743cb35599..1480755d76ab0b7c385cbcf9b37859d73fa4863b 100644
--- a/server/utils/EmbeddingEngines/localAi/index.js
+++ b/server/utils/EmbeddingEngines/localAi/index.js
@@ -16,10 +16,6 @@ class LocalAiEmbedder {
: {}),
});
this.openai = new OpenAIApi(config);
- // We don't know this for user's set model so for vectorDB integrations that requires dimensionality
- // in schema, we will throw an error.
- // Applies to QDrant and Milvus.
- this.dimensions = null;
// Limit of how many strings we can process in a single pass to stay with resource or network limits
this.maxConcurrentChunks = 50;
diff --git a/server/utils/EmbeddingEngines/native/index.js b/server/utils/EmbeddingEngines/native/index.js
index 789e51fe9e8280997ed855a679d9bfac1fd9faf8..fc933e1b8426af5639f7ab50657aad4d793e44b6 100644
--- a/server/utils/EmbeddingEngines/native/index.js
+++ b/server/utils/EmbeddingEngines/native/index.js
@@ -13,7 +13,6 @@ class NativeEmbedder {
: path.resolve(__dirname, `../../../storage/models`)
);
this.modelPath = path.resolve(this.cacheDir, "Xenova", "all-MiniLM-L6-v2");
- this.dimensions = 384;
// Limit of how many strings we can process in a single pass to stay with resource or network limits
this.maxConcurrentChunks = 25;
diff --git a/server/utils/EmbeddingEngines/openAi/index.js b/server/utils/EmbeddingEngines/openAi/index.js
index 31f556e8997deaf0447f9eb2fa01f15acfdec0ba..105be9d73ab58976ab9afb9bb4aaebce9343c35e 100644
--- a/server/utils/EmbeddingEngines/openAi/index.js
+++ b/server/utils/EmbeddingEngines/openAi/index.js
@@ -9,7 +9,6 @@ class OpenAiEmbedder {
});
const openai = new OpenAIApi(config);
this.openai = openai;
- this.dimensions = 1536;
// Limit of how many strings we can process in a single pass to stay with resource or network limits
this.maxConcurrentChunks = 500;
diff --git a/server/utils/vectorDbProviders/milvus/index.js b/server/utils/vectorDbProviders/milvus/index.js
index a9104784bb8afb6cd56dc8434748be7e9f007da6..cc934a9a2b129436f0ba8f55375416097904e3ae 100644
--- a/server/utils/vectorDbProviders/milvus/index.js
+++ b/server/utils/vectorDbProviders/milvus/index.js
@@ -81,13 +81,15 @@ const Milvus = {
await client.dropCollection({ collection_name: namespace });
return true;
},
- getOrCreateCollection: async function (client, namespace) {
+ // Milvus requires a dimension aspect for collection creation
+ // we pass this in from the first chunk to infer the dimensions like other
+ // providers do.
+ getOrCreateCollection: async function (client, namespace, dimensions = null) {
const isExists = await this.namespaceExists(client, namespace);
if (!isExists) {
- const embedder = getEmbeddingEngineSelection();
- if (!embedder.dimensions)
+ if (!dimensions)
throw new Error(
- `Your embedder selection has unknown dimensions output. It should be defined when using ${this.name}. Open an issue on Github for support.`
+ `Milvus:getOrCreateCollection Unable to infer vector dimension from input. Open an issue on Github for support.`
);
await client.createCollection({
@@ -104,7 +106,7 @@ const Milvus = {
name: "vector",
description: "vector",
data_type: DataType.FloatVector,
- dim: embedder.dimensions,
+ dim: dimensions,
},
{
name: "metadata",
@@ -131,6 +133,7 @@ const Milvus = {
) {
const { DocumentVectors } = require("../../../models/vectors");
try {
+ let vectorDimension = null;
const { pageContent, docId, ...metadata } = documentData;
if (!pageContent || pageContent.length == 0) return false;
@@ -138,11 +141,11 @@ const Milvus = {
const cacheResult = await cachedVectorInformation(fullFilePath);
if (cacheResult.exists) {
const { client } = await this.connect();
- await this.getOrCreateCollection(client, namespace);
-
const { chunks } = cacheResult;
const documentVectors = [];
+ vectorDimension = chunks[0][0].values.length || null;
+ await this.getOrCreateCollection(client, namespace, vectorDimension);
for (const chunk of chunks) {
// Before sending to Pinecone and saving the records to our db
// we need to assign the id of each chunk that is stored in the cached file.
@@ -182,6 +185,7 @@ const Milvus = {
if (!!vectorValues && vectorValues.length > 0) {
for (const [i, vector] of vectorValues.entries()) {
+ if (!vectorDimension) vectorDimension = vector.length;
const vectorRecord = {
id: uuidv4(),
values: vector,
@@ -202,7 +206,7 @@ const Milvus = {
if (vectors.length > 0) {
const chunks = [];
const { client } = await this.connect();
- await this.getOrCreateCollection(client, namespace);
+ await this.getOrCreateCollection(client, namespace, vectorDimension);
console.log("Inserting vectorized chunks into Milvus.");
for (const chunk of toChunks(vectors, 100)) {
diff --git a/server/utils/vectorDbProviders/qdrant/index.js b/server/utils/vectorDbProviders/qdrant/index.js
index ddc3408da8b324612d6199046ac698ec7cfaec7a..2783cde932f3e66b43ce129ca74b270b7178dc42 100644
--- a/server/utils/vectorDbProviders/qdrant/index.js
+++ b/server/utils/vectorDbProviders/qdrant/index.js
@@ -108,19 +108,20 @@ const QDrant = {
await client.deleteCollection(namespace);
return true;
},
- getOrCreateCollection: async function (client, namespace) {
+ // QDrant requires a dimension aspect for collection creation
+ // we pass this in from the first chunk to infer the dimensions like other
+ // providers do.
+ getOrCreateCollection: async function (client, namespace, dimensions = null) {
if (await this.namespaceExists(client, namespace)) {
return await client.getCollection(namespace);
}
-
- const embedder = getEmbeddingEngineSelection();
- if (!embedder.dimensions)
+ if (!dimensions)
throw new Error(
- `Your embedder selection has unknown dimensions output. It should be defined when using ${this.name}. Open an issue on Github for support.`
+ `Qdrant:getOrCreateCollection Unable to infer vector dimension from input. Open an issue on Github for support.`
);
await client.createCollection(namespace, {
vectors: {
- size: embedder.dimensions,
+ size: dimensions,
distance: "Cosine",
},
});
@@ -133,6 +134,7 @@ const QDrant = {
) {
const { DocumentVectors } = require("../../../models/vectors");
try {
+ let vectorDimension = null;
const { pageContent, docId, ...metadata } = documentData;
if (!pageContent || pageContent.length == 0) return false;
@@ -140,15 +142,20 @@ const QDrant = {
const cacheResult = await cachedVectorInformation(fullFilePath);
if (cacheResult.exists) {
const { client } = await this.connect();
- const collection = await this.getOrCreateCollection(client, namespace);
+ const { chunks } = cacheResult;
+ const documentVectors = [];
+ vectorDimension = chunks[0][0].vector.length || null;
+
+ const collection = await this.getOrCreateCollection(
+ client,
+ namespace,
+ vectorDimension
+ );
if (!collection)
throw new Error("Failed to create new QDrant collection!", {
namespace,
});
- const { chunks } = cacheResult;
- const documentVectors = [];
-
for (const chunk of chunks) {
const submission = {
ids: [],
@@ -210,6 +217,7 @@ const QDrant = {
if (!!vectorValues && vectorValues.length > 0) {
for (const [i, vector] of vectorValues.entries()) {
+ if (!vectorDimension) vectorDimension = vector.length;
const vectorRecord = {
id: uuidv4(),
vector: vector,
@@ -233,7 +241,11 @@ const QDrant = {
}
const { client } = await this.connect();
- const collection = await this.getOrCreateCollection(client, namespace);
+ const collection = await this.getOrCreateCollection(
+ client,
+ namespace,
+ vectorDimension
+ );
if (!collection)
throw new Error("Failed to create new QDrant collection!", {
namespace,