Add Notebooks to Docs/Sphinx (#1039)

.github/workflows/dev_docs.yml

@@ -10,9 +10,11 @@ jobs:
     steps:
       - uses: actions/checkout@v2
       - uses: cpina/github-action-push-to-another-repository@main
+      - uses: awalsh128/cache-apt-pkgs-action@latest
         env:
           API_TOKEN_GITHUB: ${{ secrets.PAT}}
         with:
+          packages: pandoc
           source-directory: './docs'
           destination-github-username: 'avb-is-me'
           destination-repository-name: 'gpt_index'

.gitignore

@@ -82,7 +82,6 @@ target/
 
 # Jupyter Notebook
 .ipynb_checkpoints
-notebooks/
 
 # IPython
 profile_default/
@@ -138,4 +137,4 @@ dmypy.json
 
 # Jetbrains
 .idea
-modules/
+modules/
\ No newline at end of file

docs/Makefile

@@ -17,4 +17,5 @@ help:
 # Catch-all target: route all unknown targets to Sphinx using the new
 # "make mode" option.  $(O) is meant as a shortcut for $(SPHINXOPTS).
 %: Makefile
+	python ./build_notebooks.py
 	@$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)

docs/build_notebooks.py

+import json
+import os
+
+source_dir = "../examples/"
+dest_dir = "./guides/notebooks/"
+relative_path = "../../../../examples"
+
+
+for example_dir in os.listdir(source_dir):
+    example_dir_path = os.path.join(source_dir, example_dir)
+
+    for nb_name in os.listdir(example_dir_path):
+        if not nb_name.endswith(".ipynb"):
+            continue
+
+        # make dest folder in docs
+        os.makedirs(os.path.join(dest_dir, example_dir), exist_ok=True)
+
+        # build link text
+        relative_nb_path = os.path.join(relative_path, example_dir, nb_name)
+        nb_link_text = json.dumps({"path": relative_nb_path})
+
+        # write nbsphinx-link document
+        nbsphinx_name = nb_name.replace(".ipynb", ".nblink")
+        nbsphinx_path = os.path.join(dest_dir, example_dir, nbsphinx_name)
+        with open(nbsphinx_path, "w") as f:
+            f.write(nb_link_text)

docs/conf.py

@@ -37,6 +37,8 @@ extensions = [
     "sphinx_rtd_theme",
     "sphinx.ext.mathjax",
     "myst_parser",
+    "nbsphinx",
+    "nbsphinx_link",
 ]
 
 myst_heading_anchors = 4
@@ -45,7 +47,7 @@ myst_heading_anchors = 4
 suppress_warnings = ["myst.header"]
 
 templates_path = ["_templates"]
-exclude_patterns = ["_build", "Thumbs.db", ".DS_Store"]
+exclude_patterns = ["_build", "Thumbs.db", ".DS_Store", "**.ipynb_checkpoints"]
 
 
 # -- Options for HTML output -------------------------------------------------
@@ -53,3 +55,7 @@ exclude_patterns = ["_build", "Thumbs.db", ".DS_Store"]
 
 html_theme = "sphinx_rtd_theme"
 html_static_path = ["_static"]
+
+# nbsphinx options
+nbsphinx_execute = "never"
+nbsphinx_allow_errors = True

docs/guides/notebooks.rst

@@ -3,4 +3,173 @@ Notebooks
 
 We offer a wide variety of example notebooks. They are referenced throughout the documentation.
 
-Example notebooks are found `here <https://github.com/jerryjliu/gpt_index/tree/main/examples>`_.
+All examples can be accessed using the menu links or can be found `directly in the repository`_.
\ No newline at end of file
+
+.. _directly in the repository: https://github.com/jerryjliu/llama_index/tree/main/examples
+
+.. toctree::
+  :glob:
+  :maxdepth: 1
+  :caption: Async
+
+  ./notebooks/async/*
+
+
+.. toctree::
+  :glob:
+  :maxdepth: 1
+  :caption: Azure
+
+  notebooks/azure_demo/*
+
+
+.. toctree::
+  :glob:
+  :maxdepth: 1
+  :caption: Chatbot
+
+  notebooks/chatbot/*
+
+
+.. toctree::
+  :glob:
+  :maxdepth: 1
+  :caption: ChatGPT Plugin
+
+  notebooks/chatgpt_plugin/*
+
+
+.. toctree::
+  :glob:
+  :maxdepth: 1
+  :caption: Composable Indices
+
+  notebooks/composable_indices/*
+
+
+.. toctree::
+  :glob:
+  :maxdepth: 1
+  :caption: Cost Analysis
+
+  notebooks/cost_analysis/*
+
+
+.. toctree::
+  :glob:
+  :maxdepth: 1
+  :caption: Data Connectors
+
+  notebooks/data_connectors/*
+
+
+.. toctree::
+  :glob:
+  :maxdepth: 1
+  :caption: Docstore
+
+  notebooks/docstore/*
+
+
+.. toctree::
+  :glob:
+  :maxdepth: 1
+  :caption: Evaluation
+
+  notebooks/evaluation/*
+
+
+.. toctree::
+  :glob:
+  :maxdepth: 1
+  :caption: Gastby
+
+  notebooks/gatsby/*
+
+
+.. toctree::
+  :glob:
+  :maxdepth: 1
+  :caption: Knowledge Graph
+
+  notebooks/knowledge_graph/*
+
+
+.. toctree::
+  :glob:
+  :maxdepth: 1
+  :caption: Langchain Integration
+
+  notebooks/langchain_demo/*
+
+
+.. toctree::
+  :glob:
+  :maxdepth: 1
+  :caption: Multimodal
+
+  notebooks/multimodal/*
+
+
+.. toctree::
+  :glob:
+  :maxdepth: 1
+  :caption: Node Post-Processor
+
+  notebooks/node_postprocessor/*
+
+
+.. toctree::
+  :glob:
+  :maxdepth: 1
+  :caption: Optimizer
+
+  notebooks/optimizer/*
+
+
+.. toctree::
+  :glob:
+  :maxdepth: 1
+  :caption: Paul Graham Essay
+
+  notebooks/paul_graham_essay/*
+
+
+.. toctree::
+  :glob:
+  :maxdepth: 1
+  :caption: Playground
+
+  notebooks/playground/*
+
+
+.. toctree::
+  :glob:
+  :maxdepth: 1
+  :caption: Query Transformations
+
+  notebooks/query_transformations/*
+
+
+.. toctree::
+  :glob:
+  :maxdepth: 1
+  :caption: Structured Indices
+
+  notebooks/struct_indices/*
+
+
+.. toctree::
+  :glob:
+  :maxdepth: 1
+  :caption: Wikipedia
+
+  notebooks/test_wiki/*
+
+
+.. toctree::
+  :glob:
+  :maxdepth: 1
+  :caption: Vector Indices
+
+  notebooks/vector_indices/*

docs/guides/notebooks/async/AsyncComposableIndicesSEC.nblink

+{"path": "../../../../examples/async/AsyncComposableIndicesSEC.ipynb"}
\ No newline at end of file

docs/guides/notebooks/async/AsyncGPTTreeIndexDemo.nblink

+{"path": "../../../../examples/async/AsyncGPTTreeIndexDemo.ipynb"}
\ No newline at end of file

docs/guides/notebooks/async/AsyncLLMPredictorDemo.nblink

+{"path": "../../../../examples/async/AsyncLLMPredictorDemo.ipynb"}
\ No newline at end of file

docs/guides/notebooks/async/AsyncQueryDemo.nblink

+{"path": "../../../../examples/async/AsyncQueryDemo.ipynb"}
\ No newline at end of file

docs/guides/notebooks/azure_demo/AzureOpenAI.nblink

+{"path": "../../../../examples/azure_demo/AzureOpenAI.ipynb"}
\ No newline at end of file

docs/guides/notebooks/chatbot/Chatbot_SEC.nblink

+{"path": "../../../../examples/chatbot/Chatbot_SEC.ipynb"}
\ No newline at end of file

docs/guides/notebooks/chatgpt_plugin/ChatGPTRetrievalPluginIndexDemo.nblink

+{"path": "../../../../examples/chatgpt_plugin/ChatGPTRetrievalPluginIndexDemo.ipynb"}
\ No newline at end of file

docs/guides/notebooks/chatgpt_plugin/ChatGPTRetrievalPluginReaderDemo.nblink

+{"path": "../../../../examples/chatgpt_plugin/ChatGPTRetrievalPluginReaderDemo.ipynb"}
\ No newline at end of file

docs/guides/notebooks/chatgpt_plugin/ChatGPT_Retrieval_Plugin_Upload.nblink

+{"path": "../../../../examples/chatgpt_plugin/ChatGPT_Retrieval_Plugin_Upload.ipynb"}
\ No newline at end of file

docs/guides/notebooks/composable_indices/ComposableIndices-Prior.nblink

+{"path": "../../../../examples/composable_indices/ComposableIndices-Prior.ipynb"}
\ No newline at end of file

docs/guides/notebooks/composable_indices/ComposableIndices-Weaviate.nblink

+{"path": "../../../../examples/composable_indices/ComposableIndices-Weaviate.ipynb"}
\ No newline at end of file

docs/guides/notebooks/composable_indices/ComposableIndices.nblink

+{"path": "../../../../examples/composable_indices/ComposableIndices.ipynb"}
\ No newline at end of file

docs/guides/notebooks/composable_indices/QASummaryGraph.nblink

+{"path": "../../../../examples/composable_indices/QASummaryGraph.ipynb"}
\ No newline at end of file

docs/guides/notebooks/cost_analysis/TokenPredictor.nblink

+{"path": "../../../../examples/cost_analysis/TokenPredictor.ipynb"}
\ No newline at end of file