From ce2f24d73f30cbabe2b4ef5e2ee8ed2f2a927c03 Mon Sep 17 00:00:00 2001
From: "Huu Le (Lee)" <39040748+leehuwuj@users.noreply.github.com>
Date: Thu, 28 Mar 2024 10:00:03 +0700
Subject: [PATCH] feat: Use yaml format instead of json for loaders and tools
config (#26)
---
.changeset/curvy-candles-mix.md | 5 +++
create-app.ts | 4 +--
helpers/python.ts | 31 ++++++++++---------
package.json | 3 +-
pnpm-lock.yaml | 9 ++++++
.../components/engines/python/agent/tools.py | 6 ++--
.../components/loaders/python/__init__.py | 6 ++--
7 files changed, 41 insertions(+), 23 deletions(-)
create mode 100644 .changeset/curvy-candles-mix.md
diff --git a/.changeset/curvy-candles-mix.md b/.changeset/curvy-candles-mix.md
new file mode 100644
index 00000000..204b65f8
--- /dev/null
+++ b/.changeset/curvy-candles-mix.md
@@ -0,0 +1,5 @@
+---
+"create-llama": patch
+---
+
+Update loaders and tools config to yaml format
diff --git a/create-app.ts b/create-app.ts
index d44fb6e4..a4d26ae7 100644
--- a/create-app.ts
+++ b/create-app.ts
@@ -128,8 +128,8 @@ export async function createApp({
console.log(
yellow(
`You have selected tools that require configuration. Please configure them in the ${terminalLink(
- "config/tools.json",
- `file://${root}/config/tools.json`,
+ "config/tools.yaml",
+ `file://${root}/config/tools.yaml`,
)} file.`,
),
);
diff --git a/helpers/python.ts b/helpers/python.ts
index 90c5c140..cdc2be48 100644
--- a/helpers/python.ts
+++ b/helpers/python.ts
@@ -3,6 +3,7 @@ import path from "path";
import { cyan, red } from "picocolors";
import { parse, stringify } from "smol-toml";
import terminalLink from "terminal-link";
+import yaml, { Document } from "yaml";
import { copy } from "./copy";
import { templatesDir } from "./dir";
import { isPoetryAvailable, tryPoetryInstall } from "./poetry";
@@ -242,12 +243,9 @@ export const installPythonTemplate = async ({
tools.forEach((tool) => {
configContent[tool.name] = tool.config ?? {};
});
- const configFilePath = path.join(root, "config/tools.json");
+ const configFilePath = path.join(root, "config/tools.yaml");
await fs.mkdir(path.join(root, "config"), { recursive: true });
- await fs.writeFile(
- configFilePath,
- JSON.stringify(configContent, null, 2),
- );
+ await fs.writeFile(configFilePath, yaml.stringify(configContent));
} else {
await copy("**", enginePath, {
parents: true,
@@ -255,7 +253,7 @@ export const installPythonTemplate = async ({
});
}
- const loaderConfigs: Record<string, any> = {};
+ const loaderConfigs = new Document({});
const loaderPath = path.join(enginePath, "loaders");
// Copy loaders to enginePath
@@ -277,22 +275,27 @@ export const installPythonTemplate = async ({
depth: dsConfig.depth,
};
});
- loaderConfigs["web"] = webLoaderConfig;
+ // Create YamlNode from array of YAMLMap
+ const node = loaderConfigs.createNode(webLoaderConfig);
+ node.commentBefore = `
+ Config for web loader
+- base_url: The url to start crawling with
+- prefix: the prefix of next URLs to crawl
+- depth: the maximum depth in DFS
+ You can add more web loaders by adding more config below`;
+ loaderConfigs.set("web", node);
}
// File loader config
if (dataSources.some((ds) => ds.type === "file")) {
- loaderConfigs["file"] = {
+ loaderConfigs.set("file", {
use_llama_parse: useLlamaParse,
- };
+ });
}
// Write loaders config
if (Object.keys(loaderConfigs).length > 0) {
- const loaderConfigPath = path.join(root, "config/loaders.json");
+ const loaderConfigPath = path.join(root, "config/loaders.yaml");
await fs.mkdir(path.join(root, "config"), { recursive: true });
- await fs.writeFile(
- loaderConfigPath,
- JSON.stringify(loaderConfigs, null, 2),
- );
+ await fs.writeFile(loaderConfigPath, yaml.stringify(loaderConfigs));
}
}
diff --git a/package.json b/package.json
index e4a93ffc..1b20d746 100644
--- a/package.json
+++ b/package.json
@@ -71,7 +71,8 @@
"typescript": "^5.3.3",
"eslint-config-prettier": "^8.10.0",
"ora": "^8.0.1",
- "fs-extra": "11.2.0"
+ "fs-extra": "11.2.0",
+ "yaml": "2.4.1"
},
"engines": {
"node": ">=16.14.0"
diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml
index 4dbb3836..35a5c036 100644
--- a/pnpm-lock.yaml
+++ b/pnpm-lock.yaml
@@ -113,6 +113,9 @@ devDependencies:
wait-port:
specifier: ^1.1.0
version: 1.1.0
+ yaml:
+ specifier: 2.4.1
+ version: 2.4.1
packages:
@@ -3366,6 +3369,12 @@ packages:
resolution: {integrity: sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==}
dev: true
+ /yaml@2.4.1:
+ resolution: {integrity: sha512-pIXzoImaqmfOrL7teGUBt/T7ZDnyeGBWyXQBvOVhLkWLN37GXv8NMLK406UY6dS51JfcQHsmcW5cJ441bHg6Lg==}
+ engines: {node: '>= 14'}
+ hasBin: true
+ dev: true
+
/yargs-parser@18.1.3:
resolution: {integrity: sha512-o50j0JeToy/4K6OZcaQmW6lyXXKhq7csREXcDwk2omFPJEwUNOVtJKvmDr9EI1fAJZUyZcRF7kxGBWmRXudrCQ==}
engines: {node: '>=6'}
diff --git a/templates/components/engines/python/agent/tools.py b/templates/components/engines/python/agent/tools.py
index a2ad3b96..93f63c44 100644
--- a/templates/components/engines/python/agent/tools.py
+++ b/templates/components/engines/python/agent/tools.py
@@ -1,4 +1,4 @@
-import json
+import yaml
import importlib
from llama_index.core.tools.tool_spec.base import BaseToolSpec
@@ -26,8 +26,8 @@ class ToolFactory:
@staticmethod
def from_env() -> list[FunctionTool]:
tools = []
- with open("config/tools.json", "r") as f:
- tool_configs = json.load(f)
+ with open("config/tools.yaml", "r") as f:
+ tool_configs = yaml.safe_load(f)
for name, config in tool_configs.items():
tools += ToolFactory.create_tool(name, **config)
return tools
diff --git a/templates/components/loaders/python/__init__.py b/templates/components/loaders/python/__init__.py
index b8c0f5a1..710144fb 100644
--- a/templates/components/loaders/python/__init__.py
+++ b/templates/components/loaders/python/__init__.py
@@ -1,5 +1,5 @@
import os
-import json
+import yaml
import importlib
import logging
from typing import Dict
@@ -10,8 +10,8 @@ logger = logging.getLogger(__name__)
def load_configs():
- with open("config/loaders.json") as f:
- configs = json.load(f)
+ with open("config/loaders.yaml") as f:
+ configs = yaml.safe_load(f)
return configs
--
GitLab