chore: update docs generation

.github/workflows/docs.yml

@@ -3,6 +3,7 @@ name: Release Docs
 on:
   release:
     types: [ released ]
+  workflow_dispatch:
 
 jobs:
   build-docs:
@@ -37,21 +38,73 @@ jobs:
     - name: Install dependencies
       run: |
         poetry install --all-extras
+
     - name: Build docs
       run: |
-        poetry run sphinx-build -M html docs/source docs/build
-    - name: Authenticate to Google Cloud
-      id: auth
-      uses: google-github-actions/auth@v2
+        mkdir -p docs/build
+        poetry run pydoc-markdown .pydoc-markdown.yml
+    
+    - name: Checkout docs repo
+      uses: actions/checkout@v4
       with:
-        credentials_json: ${{ secrets.GOOGLE_CREDENTIALS }}
-    - name: Upload Docs
+        repository: aurelio-labs/docs
+        path: temp-docs
+        token: ${{ secrets.DOCS_PAT }}
+    
+    - name: Copy and rename docs
+      run: |
+        ROOT_DIR=$(pwd)
+        rm -rf temp-docs/semantic-router/client-reference
+        mkdir -p temp-docs/semantic-router/client-reference/
+        
+        # Find and copy all md files
+        cd docs/build/semantic_router
+        find . -type f -name "*.md" | while read file; do
+          dir=$(dirname "$file")
+          echo "Processing file: $file in directory: $dir"
+          mkdir -p "$ROOT_DIR/temp-docs/semantic-router/client-reference/$dir"
+          cp "$file" "$ROOT_DIR/temp-docs/semantic-router/client-reference/${file%.md}.mdx"
+        done
+      
+    - name: Update frontmatter keys
+      run: |
+        find temp-docs/semantic-router/client-reference -type f -name "*.mdx" -exec sed -i 's/sidebar_label:/sidebarTitle:/g' {} +
+
+    - name: Install jq
+      run: sudo apt-get install -y jq
+
+    - name: Update mintlify docs.json navigation
+      run: |
+        # Get files and create grouped structure using jq
+        files=$(cd temp-docs/semantic-router/client-reference && find . -type f -name "*.mdx" | sed 's|^./||' | sed 's|.mdx$||' | sort | jq -R -s '
+          split("\n")[:-1] 
+          | map("semantic-router/client-reference/" + .) 
+          | reduce .[] as $path (
+              {"root": [], "groups": {}};
+              if ($path | split("/") | length) == 4 then
+                .groups[$path | split("/")[2]] += [$path]
+              else
+                .root += [$path]
+              end
+            )
+          | [.root[], (.groups | to_entries | map({
+              "group": .key,
+              "pages": .value
+            }))[]
+          ]
+        ')
+        
+        # Update the docs.json file
+        jq --arg pages "$files" '
+          (.navigation.tabs[] | select(.tab == "Semantic Router") | 
+           .groups[] | select(.group == "Client Reference") | 
+           .pages) |= ($pages | fromjson)
+        ' temp-docs/docs.json > temp-docs/docs.json.tmp && mv temp-docs/docs.json.tmp temp-docs/docs.json
+
+    - name: Commit and push changes
+      working-directory: temp-docs
       run: |
-        gcloud storage rsync docs/build/html gs://docs-bucket-production/semantic-router --recursive --delete-unmatched-destination-objects
-    # - name: Upload Docs
-    #   id: upload-directory
-    #   uses: google-github-actions/upload-cloud-storage@v2
-    #   with:
+        git config --local user.email "github-actions[bot]@users.noreply.github.com"
+        git config --local user.name "github-actions[bot]"
+        git add .
+        git diff --quiet && git diff --staged --quiet || (git commit -m "Update Semantic Router documentation" && git push)
\ No newline at end of file
-    #     path: docs/build/html
-    #     destination: docs-bucket-production/semantic-router
-    #     parent: false

.pydoc-markdown.yml

+loaders:
+  - type: python
+    packages: 
+      - semantic_router
+    search_path: ["."]
+
+processors:
+  - type: filter
+    skip_empty_modules: true
+    exclude_private: true
+    exclude_special: true
+    documented_only: true
+    do_not_filter_modules: true
+  - type: smart
+  - type: crossref
+
+renderer:
+  type: docusaurus
+  docs_base_path: docs/build
+  relative_output_path: .
\ No newline at end of file

pyproject.toml

@@ -39,6 +39,7 @@ psycopg2-binary = {version = "^2.9.9", optional = true}
 sphinx = {version = "^7.0.0", optional = true}
 sphinxawesome-theme = {version = "^5.2.0", optional = true}
 tornado = {version = "^6.4.2", optional = true}
+urllib3 = ">=1.25.4,<1.27"
 
 [tool.poetry.extras]
 hybrid = ["pinecone-text"]
@@ -66,6 +67,7 @@ mypy = "^1.7.1"
 types-pyyaml = "^6.0.12.12"
 types-requests = "^2.31.0"
 black = {extras = ["jupyter"], version = ">=23.12.1,<24.5.0"}
+pydoc-markdown="^4.8.2"
 
 [build-system]
 requires = ["poetry-core"]