diff --git a/docs/docs/examples/llm/octoai.ipynb b/docs/docs/examples/llm/octoai.ipynb
new file mode 100644
index 0000000000000000000000000000000000000000..bb84b5a823b687dc4da3b3c5a1b4a295fbd1cf4b
--- /dev/null
+++ b/docs/docs/examples/llm/octoai.ipynb
@@ -0,0 +1,375 @@
+{
+ "cells": [
+ {
+ "attachments": {},
+ "cell_type": "markdown",
+ "id": "6453d3d5",
+ "metadata": {},
+ "source": [
+ "<a href=\"https://colab.research.google.com/github/jerryjliu/llama_index/blob/main/docs/examples/llm/octoai.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
+ ]
+ },
+ {
+ "attachments": {},
+ "cell_type": "markdown",
+ "id": "72ed6f61-28a7-4f90-8a45-e3f452f95dbd",
+ "metadata": {},
+ "source": [
+ "# OctoAI "
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "c78b172f",
+ "metadata": {},
+ "source": [
+ "If you're opening this Notebook on colab, you will probably need to install LlamaIndex 🦙."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "3b4d1be6",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "%pip install llama-index-llms-octoai"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "50fc1a30",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "!pip install llama-index"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "81cf396f",
+ "metadata": {},
+ "source": [
+ "You'll also want to install the OctoAI SDK"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "b75dae79",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "!pip install octoai-sdk"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "f87c3024",
+ "metadata": {},
+ "source": [
+ "#### Initialize the Integration with the default model"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "e4919848",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# Be sure to run this first; the 'octoai' variable is referenced in\n",
+ "# the chat and completion examples in this notebook\n",
+ "\n",
+ "from llama_index.llms.octoai import OctoAI\n",
+ "\n",
+ "# To customize your OctoAI API token, do this\n",
+ "# otherwise it will lookup OCTOAI_TOKEN from your env variable\n",
+ "# octoai = OctoAI(token=\"<token>\")\n",
+ "\n",
+ "octoai = OctoAI()"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "b81a3ef6-2ee5-460d-9aa4-f73708774014",
+ "metadata": {},
+ "source": [
+ "#### Call `complete` with a prompt"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "910b50ad-c55e-487e-8808-5905dfaa78b4",
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ " Paul Graham is a British computer scientist, entrepreneur, and venture capitalist. He is known for his role in co-founding several companies including Viaweb (which was later acquired by Yahoo and became Yahoo Store), and the startup accelerator, Y Combinator. He is also known for writing essays on topics such as startups, programming, and entrepreneurship, which have been published on his website, PaulGraham.com.\n"
+ ]
+ }
+ ],
+ "source": [
+ "response = octoai.complete(\"Paul Graham is \")\n",
+ "print(response)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "d6ad0a98-92dd-48fd-9823-175d701c1ab2",
+ "metadata": {},
+ "source": [
+ "#### Call `chat` with a list of messages"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "5fd3137b-05ce-40a5-bdb0-5ce048f5ca25",
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "assistant: Title: Discovering Seattle: A City of Innovation and Natural Beauty\n",
+ "\n",
+ "Seattle, often referred to as the “Emerald City,†is a vibrant and dynamic destination that offers a unique blend of urban sophistication and breathtaking natural beauty. Nestled in the Pacific Northwest, this city has something for everyone, from iconic landmarks and world-class museums to bustling markets and delicious seafood. In this blog, we will explore the best that Seattle has to offer.\n",
+ "\n",
+ "Getting to Know Seattle\n",
+ "\n",
+ "Seattle is the largest city in the Pacific Northwest and the cultural hub of the region. It is home to a diverse population of over 750,000 people, with a metropolitan area of nearly 4 million. The city is known for its rainy climate, with an average annual rainfall of 38 inches, but don't let that deter you. Seattle also boasts an impressive number of sunny days, especially during the summer months.\n",
+ "\n",
+ "The city's rich history dates back to the early 19th century when European settlers arrived in the area. The city's growth was fueled by the Klondike Gold Rush of the late 1890s, which brought a flood of new residents seeking their fortunes. Today, Seattle is a thriving center of innovation, with a strong economy driven by the technology, healthcare, and maritime industries.\n",
+ "\n",
+ "Iconic Landmarks and Attractions\n",
+ "\n",
+ "No visit to Seattle would be complete without experiencing some of its most iconic landmarks and attractions. Here are a few must-see destinations:\n",
+ "\n",
+ "1. Space Needle: The Space Needle is an iconic symbol of Seattle and a must-visit attraction. Completed in 1962 for the World's Fair, this 605-foot-tall tower offers stunning 360-degree views of the city and its surroundings.\n",
+ "2. Pike Place Market: Pike Place Market is a bustling public market overlooking Elliott Bay. Established in 1907, it is one of the oldest continuously operated public farmers' markets in the United States. Visitors can enjoy fresh produce, seafood, and artisanal goods, as well as street performers and live music.\n",
+ "3. Gum Wall: Located in an alleyway under Pike Place Market, the Gum Wall is a unique and quirky attraction. Since the early 1990s, visitors have been sticking their chewed gum to the brick walls, creating a colorful and ever-evolving work of art.\n",
+ "4. Chihuly Garden and Glass: This stunning museum showcases the breathtaking glass artwork of renowned artist Dale Chihuly. The exhibits include a variety of installations, both indoors and outdoors, that showcase the artist's mastery of color, form, and light.\n",
+ "5. Seattle Art Museum: The Seattle Art Museum (SAM) is a world-class institution that features an impressive collection of modern and contemporary art, as well as rotating exhibitions from around the globe.\n",
+ "\n",
+ "Natural Beauty and Outdoor Activities\n",
+ "\n",
+ "Seattle's stunning natural beauty is one of its most appealing features. The city is surrounded by water, mountains, and forests, providing ample opportunities for outdoor enthusiasts to explore and enjoy the great outdoors. Here are a few must-do activities:\n",
+ "\n",
+ "1. Visit the Olympic Sculpture Park: This nine-acre park, operated by the Seattle Art Museum, features a variety of sculptures and installations set amidst stunning natural landscapes. The park offers panoramic views of Puget Sound and the Olympic Mountains.\n",
+ "2. Explore Discovery Park: Discovery Park is a 534-acre urban wilderness located in the heart of Seattle. The park offers miles of hiking trails, stunning views of Puget Sound, and a historic lighthouse.\n",
+ "3. Take a Ferry Ride: Seattle's waterfront is home to several ferry terminals, offering daily service to nearby islands and communities. A ferry ride is a great way to experience the city's maritime heritage and enjoy stunning views of the Olympic Mountains and Puget Sound.\n",
+ "4. Hike Mount Si: Located just 30 miles east of Seattle, Mount Si is a popular hiking destination that offers stunning views of the Snoqualmie Valley and the surrounding mountains. The hike to the summit is 4 miles one-way, with an elevation gain of 3,100 feet.\n",
+ "\n",
+ "Culinary Delights\n",
+ "\n",
+ "Seattle is a food lover's paradise, with a diverse and vibrant culinary scene that reflects the city's multicultural population. Here are a few must-try dishes:\n",
+ "\n",
+ "1. Seafood: Seattle is renowned for its fresh and delicious seafood, with an abundance of options ranging from fish and chips to Dungeness crab and geoduck.\n",
+ "2. Coffee: Seattle is the birthplace of Starbucks, and the city takes its coffee seriously. There are countless independent coffee shops and roasteries throughout the city, each with its unique style and atmosphere.\n",
+ "3. Asian Cuisine: Seattle has a large Asian population, and the city's culinary scene reflects this diversity. From Vietnamese pho and Japanese sushi to Chinese dumplings and Korean BBQ, there is no shortage of delicious Asian cuisine to explore.\n",
+ "\n",
+ "In Conclusion\n",
+ "\n",
+ "Seattle is a city that offers something for everyone, with its rich history, iconic landmarks, stunning natural beauty, and diverse culinary scene. Whether you're a first-time visitor or a seasoned local, there is always something new to discover and experience in this vibrant and dynamic city. So pack your bags, grab your raincoat, and get ready to explore all that Seattle has to offer!\n"
+ ]
+ }
+ ],
+ "source": [
+ "from llama_index.core.llms import ChatMessage\n",
+ "\n",
+ "messages = [\n",
+ " ChatMessage(\n",
+ " role=\"system\",\n",
+ " content=\"Below is an instruction that describes a task. Write a response that appropriately completes the request.\",\n",
+ " ),\n",
+ " ChatMessage(role=\"user\", content=\"Write a blog about Seattle\"),\n",
+ "]\n",
+ "response = octoai.chat(messages)\n",
+ "print(response)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "56a55ce6-08e3-4534-9bae-345686308b3e",
+ "metadata": {},
+ "source": [
+ "## Streaming"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "57901d1c-d1d4-442e-bb91-cd8f054ae2fd",
+ "metadata": {},
+ "source": [
+ "Using `stream_complete` endpoint "
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "cd9e2b22-7e62-4f50-a9af-84453aeda071",
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ " Paul Graham is a British computer scientist, entrepreneur, and venture capitalist. He is known for his role in co-founding several companies including Viaweb (which was later acquired by Yahoo and became Yahoo Store), and the startup accelerator, Y Combinator. He is also known for writing essays on topics such as startups, programming, and entrepreneurship, which have been published on his website, PaulGraham.com."
+ ]
+ }
+ ],
+ "source": [
+ "response = octoai.stream_complete(\"Paul Graham is \")\n",
+ "for r in response:\n",
+ " print(r.delta, end=\"\")"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "19b226da",
+ "metadata": {},
+ "source": [
+ "Using `stream_chat` with a list of messages"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "10b63238-8d01-48f7-b2ec-a56d23fec172",
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ " Title: Discovering Seattle: A City of Innovation and Natural Beauty\n",
+ "\n",
+ "Seattle, often referred to as the “Emerald City,†is a vibrant and dynamic destination that offers a unique blend of urban sophistication and breathtaking natural beauty. Nestled in the Pacific Northwest, this city has something for everyone, from iconic landmarks and world-class museums to bustling markets and delicious seafood. In this blog, we will explore the best that Seattle has to offer.\n",
+ "\n",
+ "Getting to Know Seattle\n",
+ "\n",
+ "Seattle is the largest city in the Pacific Northwest and is home to over 750,000 residents. The city is known for its rich history, thriving arts scene, and innovative spirit. It is also famous for its coffee culture, with Starbucks and Seattle's Best Coffee both calling Seattle home.\n",
+ "\n",
+ "Exploring Seattle's Landmarks\n",
+ "\n",
+ "No trip to Seattle would be complete without visiting the Space Needle, an iconic symbol of the city. Built for the 1962 World's Fair, the Space Needle offers stunning 360-degree views of the city and the surrounding mountains and waterways. Visitors can enjoy a meal at the Space Needle's revolving restaurant, SkyCity, or simply take in the views from the observation deck.\n",
+ "\n",
+ "Another must-see landmark is Pike Place Market, a bustling public market that has been a Seattle institution since 1907. Here, visitors can sample fresh seafood, watch fishmongers toss fish, and browse stalls filled with local produce, crafts, and souvenirs. The market is also home to the original Starbucks store, which opened in 1971.\n",
+ "\n",
+ "For history buffs, the Museum of History & Industry (MOHAI) is a must-visit. The museum offers a fascinating look at Seattle's history, from its early days as a logging and fishing town to its role as a center of innovation and technology.\n",
+ "\n",
+ "Experiencing Seattle's Natural Beauty\n",
+ "\n",
+ "Seattle is surrounded by natural beauty, and there are plenty of opportunities to experience it firsthand. The city is home to several beautiful parks, including Discovery Park, which offers stunning views of Puget Sound and the Olympic Mountains. Visitors can hike through the park's forests, explore its beaches, and even visit a historic lighthouse.\n",
+ "\n",
+ "For a more urban experience, the Seattle Japanese Garden is a tranquil oasis in the heart of the city. The garden features a variety of plants, waterfalls, and ponds, and is a popular spot for picnics and weddings.\n",
+ "\n",
+ "Just a short ferry ride from Seattle is Bainbridge Island, which offers beautiful views of the city skyline and the Olympic Mountains. Visitors can explore the island's charming downtown area, visit its museums and galleries, or simply enjoy a leisurely stroll along the waterfront.\n",
+ "\n",
+ "Indulging in Seattle's Cuisine\n",
+ "\n",
+ "Seattle is known for its seafood, and there are plenty of opportunities to sample the city's freshest catches. The Crab Pot, located on the waterfront, offers all-you-can-eat seafood boils, while Ivar's Acres of Clams is famous for its clam chowder and fish and chips.\n",
+ "\n",
+ "For a more upscale dining experience, Canlis offers stunning views of the city and a menu filled with Pacific Northwest ingredients. The restaurant has been a Seattle institution since 1950 and has won numerous awards for its food and service.\n",
+ "\n",
+ "In Conclusion\n",
+ "\n",
+ "Seattle is a city that offers something for everyone. From its iconic landmarks and world-class museums to its bustling markets and delicious seafood, there is no shortage of things to see and do in this vibrant and dynamic destination. Whether you're a history buff, a nature lover, or a foodie, Seattle is sure to delight and inspire."
+ ]
+ }
+ ],
+ "source": [
+ "from llama_index.core.llms import ChatMessage\n",
+ "\n",
+ "messages = [\n",
+ " ChatMessage(\n",
+ " role=\"system\",\n",
+ " content=\"Below is an instruction that describes a task. Write a response that appropriately completes the request.\",\n",
+ " ),\n",
+ " ChatMessage(role=\"user\", content=\"Write a blog about Seattle\"),\n",
+ "]\n",
+ "response = octoai.stream_chat(messages)\n",
+ "for r in response:\n",
+ " print(r.delta, end=\"\")"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "d4b6ea50-d777-4174-a326-6e4e57b9ea8b",
+ "metadata": {},
+ "source": [
+ "## Configure Model"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "1ce3de8d-287e-402d-936f-64a106c8fac2",
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ " Paul Graham is a well-known computer scientist, entrepreneur, and essayist. He is best known for his work as a co-founder of Viaweb, a pioneering e-commerce platform, and later as a co-founder of Y Combinator, a successful startup accelerator.\n",
+ "\n",
+ "Graham has written extensively on a wide range of topics, including entrepreneurship, technology, and social issues. His essays, which are often published on his personal website, have been widely read and discussed in the tech community. Some of his most popular essays include:\n",
+ "\n",
+ "1. \"How to Get Your Startup Funded\" - This essay provides advice for entrepreneurs on how to raise funding for their startups.\n",
+ "2. \"Do Things That Don't Scale\" - In this essay, Graham argues that entrepreneurs should focus on doing things that don't scale, such as building personal relationships with customers, rather than trying to scale their businesses too quickly.\n",
+ "3. \"The Age of the Essay\" - This essay explores the role of essays in the tech industry and argues that they are an important way for entrepreneurs to share their ideas and perspectives.\n",
+ "4. \"The Hundred-Year Language\" - In this essay, Graham discusses the importance of long-term thinking in the tech industry and argues that many of the most successful companies have a long-term perspective.\n",
+ "5. \"The Two Cultures of Startups\" - This essay explores the differences between the \"maker\" and \"manager\" cultures in startups and argues that successful startups need to balance both cultures.\n",
+ "\n",
+ "Graham is also known for his strong opinions and his willingness to speak his mind on a wide range of topics. He has been a vocal critic of some of the tech industry's most prominent companies and has been involved in several high-profile controversies over the years. Despite this, he remains one of the most respected and influential figures in the tech industry.\n"
+ ]
+ }
+ ],
+ "source": [
+ "from octoai.chat import TextModel\n",
+ "\n",
+ "# To customize your API token, do this\n",
+ "# otherwise it will lookup OCTOAI_TOKEN from your env variable\n",
+ "# octoai = OctoAI(model=TextModel.LLAMA_2_13B_CHAT_FP16, token=\"<token>\")\n",
+ "\n",
+ "octoai = OctoAI(model=TextModel.LLAMA_2_13B_CHAT_FP16)\n",
+ "\n",
+ "# ... or provide the string of the model name instead of using the TextModel enum\n",
+ "# octoai = OctoAI(model=\"codellama-13b-instruct-fp16\")\n",
+ "\n",
+ "response = octoai.complete(\"Paul Graham is \")\n",
+ "print(response)"
+ ]
+ }
+ ],
+ "metadata": {
+ "kernelspec": {
+ "display_name": "Python 3 (ipykernel)",
+ "language": "python",
+ "name": "python3"
+ },
+ "language_info": {
+ "codemirror_mode": {
+ "name": "ipython",
+ "version": 3
+ },
+ "file_extension": ".py",
+ "mimetype": "text/x-python",
+ "name": "python",
+ "nbconvert_exporter": "python",
+ "pygments_lexer": "ipython3"
+ },
+ "vscode": {
+ "interpreter": {
+ "hash": "b0fa6594d8f4cbf19f97940f81e996739fb7646882a419484c72d19e05852a7e"
+ }
+ }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}
diff --git a/llama-index-core/pyproject.toml b/llama-index-core/pyproject.toml
index 782a2937c9bcc4ccbc4bae17595f2f7079cd93cc..78da78d128728c93bacf2352e3fcd62205da3953 100644
--- a/llama-index-core/pyproject.toml
+++ b/llama-index-core/pyproject.toml
@@ -141,7 +141,7 @@ types-Deprecated = ">=0.1.0"
types-PyYAML = "^6.0.12.12"
types-protobuf = "^4.24.0.4"
types-redis = "4.5.5.0"
-types-requests = "2.28.11.8" # TODO: unpin when mypy>0.991
+types-requests = ">=2.28.11.8" # TODO: unpin when mypy>0.991
types-setuptools = "67.1.0.0"
vellum-ai = "^0.0.42"
diff --git a/llama-index-integrations/llms/llama-index-llms-octoai/.gitignore b/llama-index-integrations/llms/llama-index-llms-octoai/.gitignore
new file mode 100644
index 0000000000000000000000000000000000000000..990c18de229088f55c6c514fd0f2d49981d1b0e7
--- /dev/null
+++ b/llama-index-integrations/llms/llama-index-llms-octoai/.gitignore
@@ -0,0 +1,153 @@
+llama_index/_static
+.DS_Store
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+bin/
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+etc/
+include/
+lib/
+lib64/
+parts/
+sdist/
+share/
+var/
+wheels/
+pip-wheel-metadata/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+# Usually these files are written by a python script from a template
+# before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+.ruff_cache
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+notebooks/
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+.python-version
+
+# pipenv
+# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+# However, in case of collaboration, if having platform-specific dependencies or dependencies
+# having no cross-platform support, pipenv may install dependencies that don't work, or not
+# install all needed dependencies.
+#Pipfile.lock
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+pyvenv.cfg
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+# Jetbrains
+.idea
+modules/
+*.swp
+
+# VsCode
+.vscode
+
+# pipenv
+Pipfile
+Pipfile.lock
+
+# pyright
+pyrightconfig.json
diff --git a/llama-index-integrations/llms/llama-index-llms-octoai/BUILD b/llama-index-integrations/llms/llama-index-llms-octoai/BUILD
new file mode 100644
index 0000000000000000000000000000000000000000..201d386de5e631b649cf292b96508a46643d03e5
--- /dev/null
+++ b/llama-index-integrations/llms/llama-index-llms-octoai/BUILD
@@ -0,0 +1,4 @@
+poetry_requirements(
+ name="poetry",
+ module_mapping={"octoai-sdk": ["octoai"]}
+)
diff --git a/llama-index-integrations/llms/llama-index-llms-octoai/Makefile b/llama-index-integrations/llms/llama-index-llms-octoai/Makefile
new file mode 100644
index 0000000000000000000000000000000000000000..b9eab05aa370629a4a3de75df3ff64cd53887b68
--- /dev/null
+++ b/llama-index-integrations/llms/llama-index-llms-octoai/Makefile
@@ -0,0 +1,17 @@
+GIT_ROOT ?= $(shell git rev-parse --show-toplevel)
+
+help: ## Show all Makefile targets.
+ @grep -E '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[33m%-30s\033[0m %s\n", $$1, $$2}'
+
+format: ## Run code autoformatters (black).
+ pre-commit install
+ git ls-files | xargs pre-commit run black --files
+
+lint: ## Run linters: pre-commit (black, ruff, codespell) and mypy
+ pre-commit install && git ls-files | xargs pre-commit run --show-diff-on-failure --files
+
+test: ## Run tests via pytest.
+ pytest tests
+
+watch-docs: ## Build and watch documentation.
+ sphinx-autobuild docs/ docs/_build/html --open-browser --watch $(GIT_ROOT)/llama_index/
diff --git a/llama-index-integrations/llms/llama-index-llms-octoai/README.md b/llama-index-integrations/llms/llama-index-llms-octoai/README.md
new file mode 100644
index 0000000000000000000000000000000000000000..d69014436c4d70f5232c06e34530a2f64c8e6d86
--- /dev/null
+++ b/llama-index-integrations/llms/llama-index-llms-octoai/README.md
@@ -0,0 +1 @@
+# LlamaIndex Llms Integration: Octoai
diff --git a/llama-index-integrations/llms/llama-index-llms-octoai/llama_index/llms/octoai/BUILD b/llama-index-integrations/llms/llama-index-llms-octoai/llama_index/llms/octoai/BUILD
new file mode 100644
index 0000000000000000000000000000000000000000..db46e8d6c978c67e301dd6c47bee08c1b3fd141c
--- /dev/null
+++ b/llama-index-integrations/llms/llama-index-llms-octoai/llama_index/llms/octoai/BUILD
@@ -0,0 +1 @@
+python_sources()
diff --git a/llama-index-integrations/llms/llama-index-llms-octoai/llama_index/llms/octoai/__init__.py b/llama-index-integrations/llms/llama-index-llms-octoai/llama_index/llms/octoai/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..ec168be3907783b19949c4541cc60173fbb30e98
--- /dev/null
+++ b/llama-index-integrations/llms/llama-index-llms-octoai/llama_index/llms/octoai/__init__.py
@@ -0,0 +1,4 @@
+from llama_index.llms.octoai.base import OctoAI
+
+
+__all__ = ["OctoAI"]
diff --git a/llama-index-integrations/llms/llama-index-llms-octoai/llama_index/llms/octoai/base.py b/llama-index-integrations/llms/llama-index-llms-octoai/llama_index/llms/octoai/base.py
new file mode 100644
index 0000000000000000000000000000000000000000..3c2bd0a32fa88fa1252b2303de774805ac71e8d5
--- /dev/null
+++ b/llama-index-integrations/llms/llama-index-llms-octoai/llama_index/llms/octoai/base.py
@@ -0,0 +1,227 @@
+from typing import (
+ Any,
+ Callable,
+ Dict,
+ Optional,
+ Sequence,
+)
+
+from llama_index.core.base.llms.generic_utils import (
+ get_from_param_or_env,
+ chat_to_completion_decorator,
+ stream_chat_to_completion_decorator,
+)
+from llama_index.core.base.llms.types import (
+ ChatMessage,
+ ChatResponse,
+ ChatResponseAsyncGen,
+ ChatResponseGen,
+ CompletionResponse,
+ CompletionResponseAsyncGen,
+ CompletionResponseGen,
+ LLMMetadata,
+ MessageRole,
+)
+from llama_index.core.bridge.pydantic import Field, PrivateAttr
+from llama_index.core.callbacks import CallbackManager
+from llama_index.core.constants import DEFAULT_TEMPERATURE
+from llama_index.core.llms.llm import LLM
+from llama_index.core.llms.callbacks import (
+ llm_chat_callback,
+ llm_completion_callback,
+)
+from llama_index.core.types import BaseOutputParser, PydanticProgramMode
+
+from llama_index.llms.octoai.utils import (
+ octoai_modelname_to_contextsize,
+ to_octoai_messages,
+)
+
+from octoai.chat import TextModel
+from octoai.client import Client
+
+DEFAULT_OCTOAI_MODEL = TextModel.MIXTRAL_8X7B_INSTRUCT
+
+
+class OctoAI(LLM):
+ model: str = Field(
+ default=DEFAULT_OCTOAI_MODEL, description="The model to use with OctoAI"
+ )
+ temperature: float = Field(
+ default=DEFAULT_TEMPERATURE,
+ description="The temperature to use during generation.",
+ gte=0.0,
+ lte=1.0,
+ )
+ max_tokens: Optional[int] = Field(
+ description="The maximum number of tokens to generate.",
+ gt=0,
+ )
+ timeout: float = Field(
+ default=120, description="The timeout to use in seconds.", gte=0
+ )
+ additional_kwargs: Dict[str, Any] = Field(
+ default_factory=dict, description="Additional kwargs for the OctoAI SDK."
+ )
+ _client: Optional[Client] = PrivateAttr()
+
+ def __init__(
+ self,
+ model: str = DEFAULT_OCTOAI_MODEL,
+ temperature: float = DEFAULT_TEMPERATURE,
+ max_tokens: Optional[int] = None,
+ timeout: int = 120,
+ token: Optional[str] = None,
+ additional_kwargs: Optional[Dict[str, Any]] = None,
+ callback_manager: Optional[CallbackManager] = None,
+ # base class
+ system_prompt: Optional[str] = None,
+ messages_to_prompt: Optional[Callable[[Sequence[ChatMessage]], str]] = None,
+ completion_to_prompt: Optional[Callable[[str], str]] = None,
+ pydantic_program_mode: PydanticProgramMode = PydanticProgramMode.DEFAULT,
+ output_parser: Optional[BaseOutputParser] = None,
+ ) -> None:
+ additional_kwargs = additional_kwargs or {}
+ callback_manager = callback_manager or CallbackManager([])
+
+ token = get_from_param_or_env("token", token, "OCTOAI_TOKEN", "")
+
+ if not token:
+ raise ValueError(
+ "You must provide an API token to use OctoAI. "
+ "You can either pass it in as an argument or set it `OCTOAI_TOKEN`."
+ "To generate a token in your OctoAI account settings: https://octoai.cloud/settings`."
+ )
+
+ try:
+ self._client = Client(token=token, timeout=timeout)
+ except ImportError as err:
+ raise ImportError(
+ "Could not import OctoAI python package. "
+ "Please install it with `pip install octoai-sdk`."
+ ) from err
+
+ super().__init__(
+ additional_kwargs=additional_kwargs,
+ max_tokens=max_tokens,
+ model=model,
+ callback_manager=callback_manager,
+ temperature=temperature,
+ timeout=timeout,
+ system_prompt=system_prompt,
+ messages_to_prompt=messages_to_prompt,
+ completion_to_prompt=completion_to_prompt,
+ pydantic_program_mode=pydantic_program_mode,
+ output_parser=output_parser,
+ )
+
+ @property
+ def metadata(self) -> LLMMetadata:
+ """Get LLM metadata."""
+ return LLMMetadata(
+ context_window=octoai_modelname_to_contextsize(self.model),
+ num_output=self.max_tokens or -1,
+ is_chat_model=True,
+ model_name=self.model,
+ )
+
+ @property
+ def _model_kwargs(self) -> Dict[str, Any]:
+ base_kwargs = {
+ "model": self.model,
+ "temperature": self.temperature,
+ "max_tokens": self.max_tokens,
+ }
+ return {
+ **base_kwargs,
+ **self.additional_kwargs,
+ }
+
+ def _get_all_kwargs(self, **kwargs: Any) -> Dict[str, Any]:
+ return {
+ **self._model_kwargs,
+ **kwargs,
+ }
+
+ @llm_chat_callback()
+ def chat(self, messages: Sequence[ChatMessage], **kwargs: Any) -> ChatResponse:
+ response = self._client.chat.completions.create(
+ messages=to_octoai_messages(messages), **self._get_all_kwargs(**kwargs)
+ )
+
+ return ChatResponse(
+ message=ChatMessage(
+ role=MessageRole.ASSISTANT, content=response.choices[0].message.content
+ ),
+ raw=dict(response),
+ )
+
+ @llm_chat_callback()
+ def stream_chat(
+ self, messages: Sequence[ChatMessage], **kwargs: Any
+ ) -> ChatResponseGen:
+ streaming_response = self._client.chat.completions.create(
+ messages=to_octoai_messages(messages),
+ stream=True,
+ **self._get_all_kwargs(**kwargs),
+ )
+
+ def gen() -> ChatResponseGen:
+ content = ""
+ role = MessageRole.ASSISTANT
+ for completion in streaming_response:
+ content_delta = completion.choices[0].delta.content
+ if content_delta is None:
+ continue
+ content += content_delta
+
+ yield ChatResponse(
+ message=ChatMessage(role=role, content=content),
+ delta=content_delta,
+ raw=completion,
+ )
+
+ return gen()
+
+ @llm_completion_callback()
+ def complete(
+ self, prompt: str, formatted: bool = False, **kwargs: Any
+ ) -> CompletionResponse:
+ complete_fn = chat_to_completion_decorator(self.chat)
+ return complete_fn(prompt, **kwargs)
+
+ @llm_completion_callback()
+ def stream_complete(
+ self, prompt: str, formatted: bool = False, **kwargs: Any
+ ) -> CompletionResponseGen:
+ stream_complete_fn = stream_chat_to_completion_decorator(self.stream_chat)
+ return stream_complete_fn(prompt, **kwargs)
+
+ # ===== Async Endpoints =====
+ @llm_chat_callback()
+ async def achat(
+ self,
+ messages: Sequence[ChatMessage],
+ **kwargs: Any,
+ ) -> ChatResponse:
+ raise NotImplementedError
+
+ @llm_chat_callback()
+ async def astream_chat(
+ self,
+ messages: Sequence[ChatMessage],
+ **kwargs: Any,
+ ) -> ChatResponseAsyncGen:
+ raise NotImplementedError
+
+ @llm_completion_callback()
+ async def acomplete(
+ self, prompt: str, formatted: bool = False, **kwargs: Any
+ ) -> CompletionResponse:
+ raise NotImplementedError
+
+ @llm_completion_callback()
+ async def astream_complete(
+ self, prompt: str, formatted: bool = False, **kwargs: Any
+ ) -> CompletionResponseAsyncGen:
+ raise NotImplementedError
diff --git a/llama-index-integrations/llms/llama-index-llms-octoai/llama_index/llms/octoai/utils.py b/llama-index-integrations/llms/llama-index-llms-octoai/llama_index/llms/octoai/utils.py
new file mode 100644
index 0000000000000000000000000000000000000000..dd8c143e605a3df5061559bf82f57e74d9875369
--- /dev/null
+++ b/llama-index-integrations/llms/llama-index-llms-octoai/llama_index/llms/octoai/utils.py
@@ -0,0 +1,56 @@
+import logging
+from typing import Dict, Sequence
+from llama_index.core.base.llms.types import ChatMessage
+from octoai.chat import TextModel
+
+TEXT_MODELS: Dict[str, int] = {
+ TextModel.CODELLAMA_13B_INSTRUCT: 16384,
+ TextModel.CODELLAMA_34B_INSTRUCT: 16384,
+ TextModel.CODELLAMA_70B_INSTRUCT: 16384,
+ TextModel.CODELLAMA_7B_INSTRUCT: 4096,
+ TextModel.LLAMA_2_13B_CHAT: 4096,
+ TextModel.LLAMA_2_70B_CHAT: 4096,
+ TextModel.MISTRAL_7B_INSTRUCT: 32768,
+ TextModel.MIXTRAL_8X7B_INSTRUCT: 32768,
+}
+
+ALL_AVAILABLE_MODELS = {**TEXT_MODELS}
+
+MISSING_TOKEN_ERROR_MESSAGE = """No token found for OctoAI.
+Please set the OCTOAI_TOKEN environment \
+variable prior to initialization.
+API keys can be found or created at \
+https://octoai.cloud/settings
+"""
+
+logger = logging.getLogger(__name__)
+
+
+def octoai_modelname_to_contextsize(modelname: str) -> int:
+ """Calculate the maximum number of tokens possible to generate for a model.
+
+ Args:
+ modelname: The modelname we want to know the context size for.
+
+ Returns:
+ The maximum context size
+
+ Examples:
+ .. code-block:: python
+
+ max_tokens = octoai.modelname_to_contextsize(TextModel.CODELLAMA_13B_INSTRUCT)
+ max_tokens = octoai.modelname_to_contextsize("llama-2-13b-chat")
+ """
+ if modelname not in ALL_AVAILABLE_MODELS:
+ raise ValueError(
+ f"Unknown model {modelname!r}. Please provide a supported model name as \
+ a string or using the TextModel enum from the OctoAI SDK:"
+ f" {', '.join(ALL_AVAILABLE_MODELS.keys())}"
+ )
+ return ALL_AVAILABLE_MODELS[modelname]
+
+
+def to_octoai_messages(messages: Sequence[ChatMessage]) -> Sequence[Dict]:
+ return [
+ {"role": message.role.value, "content": message.content} for message in messages
+ ]
diff --git a/llama-index-integrations/llms/llama-index-llms-octoai/pyproject.toml b/llama-index-integrations/llms/llama-index-llms-octoai/pyproject.toml
new file mode 100644
index 0000000000000000000000000000000000000000..c7cbd02921ee62abdf7e7304babbc4660c9feb69
--- /dev/null
+++ b/llama-index-integrations/llms/llama-index-llms-octoai/pyproject.toml
@@ -0,0 +1,55 @@
+[build-system]
+build-backend = "poetry.core.masonry.api"
+requires = ["poetry-core"]
+
+[tool.codespell]
+check-filenames = true
+check-hidden = true
+skip = "*.csv,*.html,*.json,*.jsonl,*.pdf,*.txt,*.ipynb"
+
+[tool.llamahub]
+contains_example = false
+import_path = "llama_index.llms.octoai"
+
+[tool.llamahub.class_authors]
+OctoAI = "hanscl"
+
+[tool.mypy]
+disallow_untyped_defs = true
+# Remove venv skip when integrated with pre-commit
+exclude = ["_static", "build", "examples", "notebooks", "venv"]
+ignore_missing_imports = true
+python_version = "3.8"
+
+[tool.poetry]
+authors = ["hanscl"]
+description = "llama-index llms octoai integration"
+license = "MIT"
+name = "llama-index-llms-octoai"
+packages = [{include = "llama_index/"}]
+readme = "README.md"
+version = "0.1.0"
+
+[tool.poetry.dependencies]
+python = ">=3.8.1,<4.0"
+llama-index-core = "^0.10.0"
+octoai-sdk = ">=0.9.1"
+
+[tool.poetry.group.dev.dependencies]
+black = {extras = ["jupyter"], version = "<=23.9.1,>=23.7.0"}
+codespell = {extras = ["toml"], version = ">=v2.2.6"}
+ipython = "8.10.0"
+jupyter = "^1.0.0"
+mypy = "0.991"
+pre-commit = "3.2.0"
+pylint = "2.15.10"
+pytest = "7.2.1"
+pytest-mock = "3.11.1"
+ruff = "0.0.292"
+tree-sitter-languages = "^1.8.0"
+types-Deprecated = ">=0.1.0"
+types-PyYAML = "^6.0.12.12"
+types-protobuf = "^4.24.0.4"
+types-redis = "4.5.5.0"
+types-requests = ">=2.28.11.8"
+types-setuptools = "67.1.0.0"
diff --git a/llama-index-integrations/llms/llama-index-llms-octoai/tests/BUILD b/llama-index-integrations/llms/llama-index-llms-octoai/tests/BUILD
new file mode 100644
index 0000000000000000000000000000000000000000..dabf212d7e7162849c24a733909ac4f645d75a31
--- /dev/null
+++ b/llama-index-integrations/llms/llama-index-llms-octoai/tests/BUILD
@@ -0,0 +1 @@
+python_tests()
diff --git a/llama-index-integrations/llms/llama-index-llms-octoai/tests/__init__.py b/llama-index-integrations/llms/llama-index-llms-octoai/tests/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
diff --git a/llama-index-integrations/llms/llama-index-llms-octoai/tests/test_llms_octoai.py b/llama-index-integrations/llms/llama-index-llms-octoai/tests/test_llms_octoai.py
new file mode 100644
index 0000000000000000000000000000000000000000..17ffb08add78d2d6a8f3167ba575970e0bd93e64
--- /dev/null
+++ b/llama-index-integrations/llms/llama-index-llms-octoai/tests/test_llms_octoai.py
@@ -0,0 +1,7 @@
+from llama_index.core.base.llms.base import BaseLLM
+from llama_index.llms.octoai import OctoAI
+
+
+def test_embedding_class():
+ names_of_base_classes = [b.__name__ for b in OctoAI.__mro__]
+ assert BaseLLM.__name__ in names_of_base_classes