diff --git a/docs/examples/callbacks/LangfuseCallbackHandler.ipynb b/docs/examples/callbacks/LangfuseCallbackHandler.ipynb new file mode 100644 index 0000000000000000000000000000000000000000..2b14f7f7e3d8db2d8128974e8df4124fd5b73f09 --- /dev/null +++ b/docs/examples/callbacks/LangfuseCallbackHandler.ipynb @@ -0,0 +1,288 @@ +{ + "cells": [ + { + "attachments": {}, + "cell_type": "markdown", + "id": "d6509c3a", + "metadata": {}, + "source": [ + "<a href=\"https://colab.research.google.com/github/run-llama/llama_index/blob/main/docs/examples/callbacks/LangfuseCallbackHandler.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>" + ] + }, + { + "cell_type": "markdown", + "id": "c0d8b66c", + "metadata": {}, + "source": [ + "# Langfuse Callback Handler\n", + "\n", + "[Langfuse](https://langfuse.com/docs) is an open source LLM engineering platform to help teams collaboratively debug, analyze and iterate on their LLM Applications.\n", + "\n", + "The `LangfuseCallbackHandler` is integrated with Langfuse and empowers you to seamlessly track and monitor performance, traces, and metrics of your LlamaIndex application. Detailed traces of the LlamaIndex context augmentation and the LLM querying processes are captured and can be inspected directly in the Langfuse UI." + ] + }, + { + "cell_type": "markdown", + "id": "4a59a00e", + "metadata": {}, + "source": [ + "" + ] + }, + { + "cell_type": "markdown", + "id": "3b9057da", + "metadata": {}, + "source": [ + "## Setup" + ] + }, + { + "cell_type": "markdown", + "id": "5d9dfc7f", + "metadata": {}, + "source": [ + "### Install packages" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "49c3527e", + "metadata": {}, + "outputs": [], + "source": [ + "%pip install llama-index llama-index-callbacks-langfuse" + ] + }, + { + "cell_type": "markdown", + "id": "bc10630b", + "metadata": {}, + "source": [ + "### Configure environment" + ] + }, + { + "cell_type": "markdown", + "id": "4c256817", + "metadata": {}, + "source": [ + "If you haven't done yet, [sign up on Langfuse](https://cloud.langfuse.com/auth/sign-up) and obtain your API keys from the project settings." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "787e836d", + "metadata": {}, + "outputs": [], + "source": [ + "import os\n", + "\n", + "# Langfuse\n", + "os.environ[\"LANGFUSE_SECRET_KEY\"] = \"sk-lf-...\"\n", + "os.environ[\"LANGFUSE_PUBLIC_KEY\"] = \"pk-lf-...\"\n", + "os.environ[\n", + " \"LANGFUSE_HOST\"\n", + "] = \"https://cloud.langfuse.com\" # πͺπΊ EU region, πΊπΈ US region: \"https://us.cloud.langfuse.com\"\n", + "\n", + "# OpenAI\n", + "os.environ[\"OPENAI_API_KEY\"] = \"sk-...\"" + ] + }, + { + "cell_type": "markdown", + "id": "1fe2ba01", + "metadata": {}, + "source": [ + "### Register the Langfuse callback handler" + ] + }, + { + "cell_type": "markdown", + "id": "cfef9ddc", + "metadata": {}, + "source": [ + "#### Option 1: Set global LlamaIndex handler" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "72afb2b9", + "metadata": {}, + "outputs": [], + "source": [ + "from llama_index.core import set_global_handler\n", + "\n", + "set_global_handler(\"langfuse\")\n", + "langfuse_callback_handler = llama_index.core.global_handler" + ] + }, + { + "cell_type": "markdown", + "id": "0e6557d2", + "metadata": {}, + "source": [ + "#### Option 2: Use Langfuse callback directly" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "4bdd95bf", + "metadata": {}, + "outputs": [], + "source": [ + "from llama_index.core import Settings\n", + "from llama_index.core.callbacks import CallbackManager\n", + "from langfuse.llama_index import LlamaIndexCallbackHandler\n", + "\n", + "langfuse_callback_handler = LlamaIndexCallbackHandler()\n", + "Settings.callback_manager = CallbackManager([langfuse_callback_handler])" + ] + }, + { + "cell_type": "markdown", + "id": "e3e03ce7", + "metadata": {}, + "source": [ + "### Flush events to Langfuse" + ] + }, + { + "cell_type": "markdown", + "id": "e2c811ec", + "metadata": {}, + "source": [ + "The Langfuse SDKs queue and batches events in the background to reduce the number of network requests and improve overall performance. Before exiting your application, make sure all queued events have been flushed to Langfuse servers." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "4e28876c", + "metadata": {}, + "outputs": [], + "source": [ + "# ... your LlamaIndex calls here ...\n", + "\n", + "langfuse_callback_handler.flush()" + ] + }, + { + "cell_type": "markdown", + "id": "6b86f1b5", + "metadata": {}, + "source": [ + "Done!β¨ Traces and metrics from your LlamaIndex application are now automatically tracked in Langfuse. If you construct a new index or query an LLM with your documents in context, your traces and metrics are immediately visible in the Langfuse UI. Next, let's take a look at how traces will look in Langfuse." + ] + }, + { + "cell_type": "markdown", + "id": "1f0d4465", + "metadata": {}, + "source": [ + "## Example" + ] + }, + { + "cell_type": "markdown", + "id": "8a9f3428", + "metadata": {}, + "source": [ + "Fetch and save example data." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "aa303ae3", + "metadata": {}, + "outputs": [], + "source": [ + "!mkdir -p 'data/'\n", + "!wget 'https://raw.githubusercontent.com/run-llama/llama_index/main/docs/examples/data/paul_graham/paul_graham_essay.txt' -O 'data/paul_graham_essay.txt'" + ] + }, + { + "cell_type": "markdown", + "id": "9f053996", + "metadata": {}, + "source": [ + "Run an example index construction, query, and chat." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "983cbedd", + "metadata": {}, + "outputs": [], + "source": [ + "from llama_index.core import SimpleDirectoryReader, VectorStoreIndex\n", + "\n", + "# Create index\n", + "documents = SimpleDirectoryReader(\"data\").load_data()\n", + "index = VectorStoreIndex.from_documents(documents)\n", + "\n", + "# Execute query\n", + "query_engine = index.as_query_engine()\n", + "query_response = query_engine.query(\"What did the author do growing up?\")\n", + "print(query_response)\n", + "\n", + "# Execute chat query\n", + "chat_engine = index.as_chat_engine()\n", + "chat_response = chat_engine.chat(\"What did the author do growing up?\")\n", + "print(chat_response)\n", + "\n", + "# As we want to immediately see result in Langfuse, we need to flush the callback handler\n", + "langfuse_callback_handler.flush()" + ] + }, + { + "cell_type": "markdown", + "id": "d5cdd88f", + "metadata": {}, + "source": [ + "Done!β¨ You will now see traces of your index and query in your Langfuse project.\n", + "\n", + "Example traces (public links):\n", + "1. [Index construction](https://cloud.langfuse.com/project/clsuh9o2y0000mbztvdptt1mh/traces/1294ed01-8193-40a5-bb4e-2f0723d2c827)\n", + "2. [Query Engine](https://cloud.langfuse.com/project/clsuh9o2y0000mbztvdptt1mh/traces/eaa4ea74-78e0-42ef-ace0-7aa02c6fbbc6)\n", + "3. [Chat Engine](https://cloud.langfuse.com/project/clsuh9o2y0000mbztvdptt1mh/traces/d95914f5-66eb-4520-b996-49e84fd7f323)" + ] + }, + { + "cell_type": "markdown", + "id": "0b50845f", + "metadata": {}, + "source": [ + "### π More details\n", + "\n", + "Check out the full [Langfuse documentation](https://langfuse.com/docs) for more details on Langfuse's tracing and analytics capabilities and how to make most of this integration." + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/docs/module_guides/observability/observability.md b/docs/module_guides/observability/observability.md index 48ac5f98bd897be0f6f24c255604a1c8733dd972..d19d2e67b1b321ec20f2de10e2bdaa2e5666ab6b 100644 --- a/docs/module_guides/observability/observability.md +++ b/docs/module_guides/observability/observability.md @@ -50,6 +50,27 @@ llama_index.core.set_global_handler("simple") We offer a rich set of integrations with our partners. A short description + usage pattern, and guide is provided for each partner. +### Langfuse + +[Langfuse](https://langfuse.com/docs) is an open source LLM engineering platform to help teams collaboratively debug, analyze and iterate on their LLM Applications. With the Langfuse integration, you can seamlessly track and monitor performance, traces, and metrics of your LlamaIndex application. Detailed traces of the LlamaIndex context augmentation and the LLM querying processes are captured and can be inspected directly in the Langfuse UI. + +#### Usage Pattern + +```python +from llama_index.core import set_global_handler + +# Make sure you've installed the 'llama-index-callbacks-langfuse' integration package. + +# NOTE: Set your environment variables 'LANGFUSE_SECRET_KEY', 'LANGFUSE_PUBLIC_KEY' and 'LANGFUSE_HOST' +# as shown in your langfuse.com project settings. + +set_global_handler("langfuse") +``` + +Check out the [full documentation](/examples/callbacks/LangfuseCallbackHandler.html) for more details. + + + ### DeepEval [DeepEval (by Confident AI)](https://github.com/confident-ai/deepeval) is an open-source evaluation framework for LLM applications. As you "unit test" your LLM app using DeepEval's 14+ default metrics it currently offers (summarization, hallucination, answer relevancy, faithfulness, RAGAS, etc.), you can debug failing test cases through this tracing integration with LlamaIndex, or debug unsatisfactory evaluations in **production** through DeepEval's hosted evaluation platform, [Confident AI](https://confident-ai.com), that runs referenceless evaluations in production. diff --git a/llama-index-cli/llama_index/cli/upgrade/mappings.json b/llama-index-cli/llama_index/cli/upgrade/mappings.json index 86b7b3ae5417fc08fbe40e145aa430d82b8bbf6e..9ad04974d42bb3317362c84dd9b4967b85b70ce0 100644 --- a/llama-index-cli/llama_index/cli/upgrade/mappings.json +++ b/llama-index-cli/llama_index/cli/upgrade/mappings.json @@ -519,6 +519,7 @@ "TextEmbeddingsInference": "llama_index.embeddings.text_embeddings_inference", "UpTrainCallbackHandler": "llama_index.callbacks.uptrain", "deepeval_callback_handler": "llama_index.callbacks.deepeval", + "langfuse_callback_handler": "llama_index.callbacks.langfuse", "OpenInferenceCallbackHandler": "llama_index.callbacks.openinference", "WandbCallbackHandler": "llama_index.callbacks.wandb", "argilla_callback_handler": "llama_index.callbacks.argilla", diff --git a/llama-index-core/llama_index/core/callbacks/global_handlers.py b/llama-index-core/llama_index/core/callbacks/global_handlers.py index d915eb16451d776ead2f6966f686a6a75214b3a7..5a49d193cd7b0cf0d73f1a9556089844e1ea7428 100644 --- a/llama-index-core/llama_index/core/callbacks/global_handlers.py +++ b/llama-index-core/llama_index/core/callbacks/global_handlers.py @@ -95,6 +95,17 @@ def create_global_handler(eval_mode: str, **eval_params: Any) -> BaseCallbackHan "Please install it using `pip install llama-index-callbacks-argilla`" ) handler = argilla_callback_handler(**eval_params) + elif eval_mode == "langfuse": + try: + from llama_index.callbacks.langfuse import ( + langfuse_callback_handler, + ) # pants: no-infer-dep + except ImportError: + raise ImportError( + "LangfuseCallbackHandler is not installed. " + "Please install it using `pip install llama-index-callbacks-langfuse`" + ) + handler = langfuse_callback_handler(**eval_params) else: raise ValueError(f"Eval mode {eval_mode} not supported.") diff --git a/llama-index-core/llama_index/core/command_line/mappings.json b/llama-index-core/llama_index/core/command_line/mappings.json index 86b7b3ae5417fc08fbe40e145aa430d82b8bbf6e..9ad04974d42bb3317362c84dd9b4967b85b70ce0 100644 --- a/llama-index-core/llama_index/core/command_line/mappings.json +++ b/llama-index-core/llama_index/core/command_line/mappings.json @@ -519,6 +519,7 @@ "TextEmbeddingsInference": "llama_index.embeddings.text_embeddings_inference", "UpTrainCallbackHandler": "llama_index.callbacks.uptrain", "deepeval_callback_handler": "llama_index.callbacks.deepeval", + "langfuse_callback_handler": "llama_index.callbacks.langfuse", "OpenInferenceCallbackHandler": "llama_index.callbacks.openinference", "WandbCallbackHandler": "llama_index.callbacks.wandb", "argilla_callback_handler": "llama_index.callbacks.argilla", diff --git a/llama-index-integrations/callbacks/llama-index-callbacks-langfuse/BUILD b/llama-index-integrations/callbacks/llama-index-callbacks-langfuse/BUILD new file mode 100644 index 0000000000000000000000000000000000000000..0896ca890d8bffd60a44fa824f8d57fecd73ee53 --- /dev/null +++ b/llama-index-integrations/callbacks/llama-index-callbacks-langfuse/BUILD @@ -0,0 +1,3 @@ +poetry_requirements( + name="poetry", +) diff --git a/llama-index-integrations/callbacks/llama-index-callbacks-langfuse/Makefile b/llama-index-integrations/callbacks/llama-index-callbacks-langfuse/Makefile new file mode 100644 index 0000000000000000000000000000000000000000..b9eab05aa370629a4a3de75df3ff64cd53887b68 --- /dev/null +++ b/llama-index-integrations/callbacks/llama-index-callbacks-langfuse/Makefile @@ -0,0 +1,17 @@ +GIT_ROOT ?= $(shell git rev-parse --show-toplevel) + +help: ## Show all Makefile targets. + @grep -E '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[33m%-30s\033[0m %s\n", $$1, $$2}' + +format: ## Run code autoformatters (black). + pre-commit install + git ls-files | xargs pre-commit run black --files + +lint: ## Run linters: pre-commit (black, ruff, codespell) and mypy + pre-commit install && git ls-files | xargs pre-commit run --show-diff-on-failure --files + +test: ## Run tests via pytest. + pytest tests + +watch-docs: ## Build and watch documentation. + sphinx-autobuild docs/ docs/_build/html --open-browser --watch $(GIT_ROOT)/llama_index/ diff --git a/llama-index-integrations/callbacks/llama-index-callbacks-langfuse/README.md b/llama-index-integrations/callbacks/llama-index-callbacks-langfuse/README.md new file mode 100644 index 0000000000000000000000000000000000000000..4048b2454006569d1791a1aa870c24bee3f3a0d5 --- /dev/null +++ b/llama-index-integrations/callbacks/llama-index-callbacks-langfuse/README.md @@ -0,0 +1,18 @@ +# LlamaIndex Callbacks Integration: Langfuse + +[Langfuse](https://langfuse.com/docs) is an open source LLM engineering platform to help teams collaboratively debug, analyze and iterate on their LLM Applications. With the Langfuse integration, you can seamlessly track and monitor performance, traces, and metrics of your LlamaIndex application. Detailed traces of the LlamaIndex context augmentation and the LLM querying processes are captured and can be inspected directly in the Langfuse UI. + +#### Usage Pattern + +```python +from llama_index.core import set_global_handler + +# Make sure you've installed the 'llama-index-callbacks-langfuse' integration package. + +# NOTE: Set your environment variables 'LANGFUSE_SECRET_KEY', 'LANGFUSE_PUBLIC_KEY' and 'LANGFUSE_HOST' +# as shown in your langfuse.com project settings. + +set_global_handler("langfuse") +``` + + diff --git a/llama-index-integrations/callbacks/llama-index-callbacks-langfuse/llama_index/callbacks/langfuse/BUILD b/llama-index-integrations/callbacks/llama-index-callbacks-langfuse/llama_index/callbacks/langfuse/BUILD new file mode 100644 index 0000000000000000000000000000000000000000..db46e8d6c978c67e301dd6c47bee08c1b3fd141c --- /dev/null +++ b/llama-index-integrations/callbacks/llama-index-callbacks-langfuse/llama_index/callbacks/langfuse/BUILD @@ -0,0 +1 @@ +python_sources() diff --git a/llama-index-integrations/callbacks/llama-index-callbacks-langfuse/llama_index/callbacks/langfuse/__init__.py b/llama-index-integrations/callbacks/llama-index-callbacks-langfuse/llama_index/callbacks/langfuse/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..b060aaf4c84468d69beacef97868137d194c401c --- /dev/null +++ b/llama-index-integrations/callbacks/llama-index-callbacks-langfuse/llama_index/callbacks/langfuse/__init__.py @@ -0,0 +1,3 @@ +from llama_index.callbacks.langfuse.base import langfuse_callback_handler + +__all__ = ["langfuse_callback_handler"] diff --git a/llama-index-integrations/callbacks/llama-index-callbacks-langfuse/llama_index/callbacks/langfuse/base.py b/llama-index-integrations/callbacks/llama-index-callbacks-langfuse/llama_index/callbacks/langfuse/base.py new file mode 100644 index 0000000000000000000000000000000000000000..381b77ccc69a9174b012d6d559a61287cc32e9f1 --- /dev/null +++ b/llama-index-integrations/callbacks/llama-index-callbacks-langfuse/llama_index/callbacks/langfuse/base.py @@ -0,0 +1,11 @@ +from typing import Any + +from llama_index.core.callbacks.base_handler import BaseCallbackHandler + +from langfuse.llama_index import LlamaIndexCallbackHandler + + +def langfuse_callback_handler(**eval_params: Any) -> BaseCallbackHandler: + return LlamaIndexCallbackHandler( + **eval_params, sdk_integration="llama-index_set-global-handler" + ) diff --git a/llama-index-integrations/callbacks/llama-index-callbacks-langfuse/pyproject.toml b/llama-index-integrations/callbacks/llama-index-callbacks-langfuse/pyproject.toml new file mode 100644 index 0000000000000000000000000000000000000000..132031548ecf0db5d7dd7403dcbc6da5467079fb --- /dev/null +++ b/llama-index-integrations/callbacks/llama-index-callbacks-langfuse/pyproject.toml @@ -0,0 +1,64 @@ +[build-system] +build-backend = "poetry.core.masonry.api" +requires = ["poetry-core"] + +[tool.codespell] +check-filenames = true +check-hidden = true +ignore-words-list = "Gere" +skip = "*.csv,*.html,*.json,*.jsonl,*.pdf,*.txt,*.ipynb" + +[tool.llamahub] +contains_example = false +import_path = "llama_index.callbacks.langfuse" + +[tool.llamahub.class_authors] +LangfuseCallbackHandler = "llama-index" + +[tool.mypy] +disallow_untyped_defs = true +exclude = ["_static", "build", "examples", "notebooks", "venv"] +ignore_missing_imports = true +python_version = "3.8" + +[tool.poetry] +authors = ["Your Name <you@example.com>"] +description = "llama-index callbacks langfuse integration" +exclude = ["**/BUILD"] +license = "MIT" +name = "llama-index-callbacks-langfuse" +readme = "README.md" +version = "0.1.2" + +[tool.poetry.dependencies] +python = ">=3.8.1,<4.0" +llama-index-core = "^0.10.8" +langfuse = "^2.18.0" + +[tool.poetry.group.dev.dependencies] +ipython = "8.10.0" +jupyter = "^1.0.0" +mypy = "0.991" +pre-commit = "3.2.0" +pylint = "2.15.10" +pytest = "7.2.1" +pytest-mock = "3.11.1" +ruff = "0.0.292" +tree-sitter-languages = "^1.8.0" +types-Deprecated = ">=0.1.0" +types-PyYAML = "^6.0.12.12" +types-protobuf = "^4.24.0.4" +types-redis = "4.5.5.0" +types-requests = "2.28.11.8" +types-setuptools = "67.1.0.0" + +[tool.poetry.group.dev.dependencies.black] +extras = ["jupyter"] +version = "<=23.9.1,>=23.7.0" + +[tool.poetry.group.dev.dependencies.codespell] +extras = ["toml"] +version = ">=v2.2.6" + +[[tool.poetry.packages]] +include = "llama_index/"