Merge pull request #503 from silentoplayz/main

feat: Enhance scan-missing-env-vars.py to detect defaults, types, and categorize output

docs/getting-started/env-configuration.md

@@ -42,8 +42,6 @@ environment variables, see our [logging documentation](https://docs.openwebui.co
 
 ### General
 
-### Open WebUI
-
 #### `WEBUI_URL`
 
 - Type: `str`
@@ -928,7 +926,7 @@ modeling files for reranking.
 
 - Type: `str`
 - Options:
-- `chroma`, `elasticsearch`, `milvus`, `qdrant`, `opensearch`, `pgvector`
+- `chroma`, `elasticsearch`, `milvus`, `opensearch`, `pgvector`, `qdrant`
 - Default: `chroma`
 - Description: Specifies which vector database system to use. This setting determines which vector storage system will be used for managing embeddings.
 
@@ -1093,7 +1091,7 @@ modeling files for reranking.
 #### `PGVECTOR_INITIALIZE_MAX_VECTOR_LENGTH`
 
 - Type: `str`
-- Default: '1536'
+- Default: `1536`
 - Description: Specifies the maximum vector length for PGVector initialization.
 
 ### Qdrant
@@ -1698,7 +1696,7 @@ the search query. Example: `http://searxng.local/search?q=<query>`
 #### `TAVILY_EXTRACT_DEPTH`
 
 - Type: `str`
-- Default: 'basic'
+- Default: `basic`
 - Description: Specifies the extract depth for Tavily search results.
 - Persistence: This environment variable is a `PersistentConfig` variable.
 
@@ -1743,7 +1741,7 @@ Using a remote Playwright browser via `PLAYWRIGHT_WS_URL` can be beneficial for:
 #### `FIRECRAWL_API_BASE_URL`
 
 - Type: `str`
-- Default: 'https://api.firecrawl.dev'
+- Default: `https://api.firecrawl.dev`
 - Description: Sets the base URL for Firecrawl API.
 - Persistence: This environment variable is a `PersistentConfig` variable.
 
@@ -2588,7 +2586,7 @@ See https://docs.github.com/en/apps/oauth-apps/building-oauth-apps/authorizing-o
 #### `USER_PERMISSIONS_CHAT_CONTROLS`
 
 - Type: `str`
-- Default: 'True'
+- Default: `True`
 - Description: Enables or disables user permission to access chat controls.
 - Persistence: This environment variable is a `PersistentConfig` variable.
 
@@ -2616,28 +2614,28 @@ See https://docs.github.com/en/apps/oauth-apps/building-oauth-apps/authorizing-o
 #### `USER_PERMISSIONS_CHAT_STT`
 
 - Type: `str`
-- Default: 'True'
+- Default: `True`
 - Description: Enables or disables user permission to use Speech-to-Text in chats.
 - Persistence: This environment variable is a `PersistentConfig` variable.
 
 #### `USER_PERMISSIONS_CHAT_TTS`
 
 - Type: `str`
-- Default: 'True'
+- Default: `True`
 - Description: Enables or disables user permission to use Text-to-Speech in chats.
 - Persistence: This environment variable is a `PersistentConfig` variable.
 
 #### `USER_PERMISSIONS_CHAT_CALL`
 
 - Type: `str`
-- Default: 'True'
+- Default: `True`
 - Description: Enables or disables user permission to make calls in chats.
 - Persistence: This environment variable is a `PersistentConfig` variable.
 
 #### `USER_PERMISSIONS_CHAT_MULTIPLE_MODELS`
 
 - Type: `str`
-- Default: 'True'
+- Default: `True`
 - Description: Enables or disables user permission to use multiple models in chats.
 - Persistence: This environment variable is a `PersistentConfig` variable.
 
@@ -2651,7 +2649,7 @@ See https://docs.github.com/en/apps/oauth-apps/building-oauth-apps/authorizing-o
 #### `USER_PERMISSIONS_CHAT_TEMPORARY_ENFORCED`
 
 - Type: `str`
-- Default: 'False'
+- Default: `False`
 - Description: Enables or disables enforced temporary chats for users.
 - Persistence: This environment variable is a `PersistentConfig` variable.
 
@@ -2660,28 +2658,28 @@ See https://docs.github.com/en/apps/oauth-apps/building-oauth-apps/authorizing-o
 #### `USER_PERMISSIONS_FEATURES_DIRECT_TOOL_SERVERS`
 
 - Type: `str`
-- Default: 'False'
+- Default: `False`
 - Description: Enables or disables user permission to access direct tool servers.
 - Persistence: This environment variable is a `PersistentConfig` variable.
 
 #### `USER_PERMISSIONS_FEATURES_WEB_SEARCH`
 
 - Type: `str`
-- Default: 'True'
+- Default: `True`
 - Description: Enables or disables user permission to use the web search feature.
 - Persistence: This environment variable is a `PersistentConfig` variable.
 
 #### `USER_PERMISSIONS_FEATURES_IMAGE_GENERATION`
 
 - Type: `str`
-- Default: 'True'
+- Default: `True`
 - Description: Enables or disables user permission to use the image generation feature.
 - Persistence: This environment variable is a `PersistentConfig` variable.
 
 #### `USER_PERMISSIONS_FEATURES_CODE_INTERPRETER`
 
 - Type: `str`
-- Default: 'True'
+- Default: `True`
 - Description: Enables or disables user permission to use code interpreter feature.
 - Persistence: This environment variable is a `PersistentConfig` variable.
 
@@ -2718,28 +2716,28 @@ See https://docs.github.com/en/apps/oauth-apps/building-oauth-apps/authorizing-o
 #### `USER_PERMISSIONS_WORKSPACE_MODELS_ALLOW_PUBLIC_SHARING`
 
 - Type: `str`
-- Default: 'False'
+- Default: `False`
 - Description: Enables or disables public sharing of workspace models.
 - Persistence: This environment variable is a `PersistentConfig` variable.
 
 #### `USER_PERMISSIONS_WORKSPACE_KNOWLEDGE_ALLOW_PUBLIC_SHARING`
 
 - Type: `str`
-- Default: 'False'
+- Default: `False`
 - Description: Enables or disables public sharing of workspace knowledge.
 - Persistence: This environment variable is a `PersistentConfig` variable.
 
 #### `USER_PERMISSIONS_WORKSPACE_PROMPTS_ALLOW_PUBLIC_SHARING`
 
 - Type: `str`
-- Default: 'False'
+- Default: `False`
 - Description: Enables or disables public sharing of workspace prompts.
 - Persistence: This environment variable is a `PersistentConfig` variable.
 
 #### `USER_PERMISSIONS_WORKSPACE_TOOLS_ALLOW_PUBLIC_SHARING`
 
 - Type: `str`
-- Default: 'False'
+- Default: `False`
 - Description: Enables or disables public sharing of workspace tools.
 - Persistence: This environment variable is a `PersistentConfig` variable.
 
@@ -3002,7 +3000,7 @@ Open WebUI uses the following environment variables:
 separated by commas. For example, setting no_proxy to '.mit.edu' ensures that the proxy is
 bypassed when accessing documents from MIT.
 
-### Install required packages
+### Install Required Python Packages
 
 Open WebUI provides environment variables to customize the pip installation process. Below are the environment variables used by Open WebUI for adjusting package installation behavior:
 

scripts/scan-missing-env-vars.py

@@ -6,12 +6,13 @@ import urllib.request
 import os
 
 
-def find_env_vars(code):
+def find_env_vars(code, filename):
     tree = ast.parse(code)
+    env_vars_found = {}  # Dictionary to store env vars, filenames, defaults, and types
 
     class EnvVarVisitor(ast.NodeVisitor):
         def __init__(self):
-            self.env_vars = set()
+            self.current_env_vars = {}  # Store env vars with potential defaults and types
 
         def visit_Subscript(self, node):
             if isinstance(node.value, ast.Attribute):
@@ -21,32 +22,158 @@ def find_env_vars(code):
                     and node.value.attr == "environ"
                 ):
                     if isinstance(node.slice, ast.Constant):
-                        self.env_vars.add(node.slice.value)
+                        env_var_name = node.slice.value
+                        if env_var_name not in self.current_env_vars:
+                            self.current_env_vars[env_var_name] = {"default": None, "type": "str"} # Default type str for os.environ
                     elif isinstance(node.slice, ast.BinOp):
                         # Handle dynamically constructed env var names like os.environ["VAR_" + "NAME"]
-                        self.env_vars.add(ast.unparse(node.slice))
+                        env_var_name = ast.unparse(node.slice)
+                        if env_var_name not in self.current_env_vars:
+                            self.current_env_vars[env_var_name] = {"default": None, "type": "str"} # Default type str for os.environ
             self.generic_visit(node)
 
         def visit_Call(self, node):
             if isinstance(node.func, ast.Attribute):
+                # Check for os.getenv("VAR_NAME", "default_value")
                 if (
                     isinstance(node.func.value, ast.Name)
                     and node.func.value.id == "os"
-                    and node.func.attr in ("getenv", "get")
-                ) or (
+                    and node.func.attr == "getenv"
+                ):
+                    if node.args and isinstance(node.args[0], ast.Constant):
+                        env_var_name = node.args[0].value
+                        default_value = None
+                        var_type = "str" # Default type str for os.getenv
+                        if len(node.args) > 1:
+                            default_node = node.args[1]
+                            if isinstance(default_node, ast.Constant):
+                                default_value = default_node.value
+                                var_type = "str" # Still str if default is constant string
+                            elif isinstance(default_node, ast.Name) and default_node.id == 'None': # Check for None literal
+                                default_value = None
+                                var_type = "str" # Still str even if default is None in getenv
+                            else:  # Capture other default expressions as unparsed code
+                                default_value = ast.unparse(default_node)
+                                var_type = "str" # Assume str if complex default in getenv
+                        if env_var_name not in self.current_env_vars:
+                            self.current_env_vars[env_var_name] = {"default": default_value, "type": var_type}
+
+                # Check for os.environ.get("VAR_NAME", "default_value")
+                elif (
                     isinstance(node.func.value, ast.Attribute)
                     and isinstance(node.func.value.value, ast.Name)
                     and node.func.value.value.id == "os"
                     and node.func.value.attr == "environ"
                     and node.func.attr == "get"
                 ):
-                    if isinstance(node.args[0], ast.Constant):
-                        self.env_vars.add(node.args[0].value)
+                    if node.args and isinstance(node.args[0], ast.Constant):
+                        env_var_name = node.args[0].value
+                        default_value = None
+                        var_type = "str" # Default type str for os.environ.get
+                        if len(node.args) > 1:
+                            default_node = node.args[1]
+                            if isinstance(default_node, ast.Constant):
+                                default_value = default_node.value
+                                var_type = "str" # Still str if default is constant string
+                            elif isinstance(default_node, ast.Name) and default_node.id == 'None': # Check for None literal
+                                default_value = None
+                                var_type = "str" # Still str even if default is None in get
+                            else:  # Capture other default expressions as unparsed code
+                                default_value = ast.unparse(default_node)
+                                var_type = "str" # Assume str if complex default in get
+                        if env_var_name not in self.current_env_vars:
+                            self.current_env_vars[env_var_name] = {"default": default_value, "type": var_type}
+
+            elif isinstance(node.func, ast.Name) and node.func.id == "PersistentConfig":
+                if node.args and isinstance(node.args[0], ast.Constant):
+                    env_var_name = node.args[0].value
+                    default_value = None
+                    var_type = "str" # Assume str as base type for PersistentConfig, will refine
+                    if len(node.args) > 2:  # Default value is the third argument
+                        default_node = node.args[2]
+                        if isinstance(default_node, ast.Constant):
+                            default_value = default_node.value
+                            if isinstance(default_value, bool):
+                                var_type = "bool"
+                            elif isinstance(default_value, int):
+                                var_type = "int"
+                            elif isinstance(default_value, float):
+                                var_type = "float"
+                            else:
+                                var_type = "str" # String constant
+                        elif isinstance(default_node, ast.List):
+                            default_value = ast.unparse(default_node)
+                            var_type = "list[dict]" # Assuming list of dicts for DEFAULT_PROMPT_SUGGESTIONS case, refine if needed
+                        elif isinstance(default_node, ast.Dict):
+                            default_value = ast.unparse(default_node)
+                            var_type = "dict"
+                        elif isinstance(default_node, ast.Tuple):
+                            default_value = ast.unparse(default_node)
+                            var_type = "tuple"
+                        elif isinstance(default_node, ast.Set):
+                            default_value = ast.unparse(default_node)
+                            var_type = "set"
+                        elif isinstance(default_node, ast.Name):  # Capture variable name as default
+                            default_value = default_node.id
+                            var_type = "str" # Assume str if variable default
+                        elif isinstance(default_node, ast.Call):  # Check if default_node is a Call (function call)
+                            if isinstance(default_node.func, ast.Name) and default_node.func.id == "int":
+                                var_type = "int"
+                            elif isinstance(default_node.func, ast.Name) and default_node.func.id == "float":
+                                var_type = "float"
+                            elif isinstance(default_node.func, ast.Name) and default_node.func.id == "bool":
+                                var_type = "bool"
+                            elif isinstance(default_node.func, ast.Name) and default_node.func.id == "str":
+                                var_type = "str"
+                            elif isinstance(default_node.func, ast.Attribute) and default_node.func.attr == 'getenv' and isinstance(default_node.func.value, ast.Name) and default_node.func.value.id == 'os':
+                                if len(default_node.args) > 1 and isinstance(default_node.args[1], ast.Constant):
+                                    default_value = default_node.args[1].value  # Extract default from os.getenv within PersistentConfig
+                                    var_type = "str" # Still string from getenv
+                                elif len(default_node.args) == 1:
+                                    default_value = None  # No default in os.getenv
+                                    var_type = "str" # Still string from getenv
+                            elif isinstance(default_node.func, ast.Attribute) and default_node.func.attr == 'get' and isinstance(default_node.func.value, ast.Attribute) and default_node.func.value.attr == 'environ' and isinstance(default_node.func.value.value, ast.Name) and default_node.func.value.value.id == 'os':
+                                if len(default_node.args) > 1 and isinstance(default_node.args[1], ast.Constant):
+                                    default_value = default_node.args[1].value  # Extract default from os.environ.get within PersistentConfig
+                                    var_type = "str" # Still string from getenv
+                                elif len(default_node.args) == 1:
+                                    default_value = None  # No default in os.environ.get
+                                    var_type = "str" # Still string from getenv
+                            else:  # Capture other function calls as unparsed code
+                                default_value = ast.unparse(default_node)
+                                var_type = "str" # Assume str for complex call
+                        elif isinstance(default_node, ast.Compare):  # Handle boolean expressions like 'os.getenv(...) == "true"'
+                            default_value = ast.unparse(default_node)  # Capture the whole boolean expression as unparsed code
+                            var_type = "bool" # Likely boolean from comparison
+
+                        elif isinstance(default_node, ast.Name) and default_node.id == 'None':  # Check for None literal in PersistentConfig
+                            default_value = None
+                            var_type = "str" # Could be anything, but let's say str as base
+
+                        elif default_node:  # Capture any other default expressions as unparsed code
+                            default_value = ast.unparse(default_node)
+                            var_type = "str" # Assume str for other expressions
+
+                    if env_var_name not in self.current_env_vars:
+                        self.current_env_vars[env_var_name] = {"default": default_value, "type": var_type}
+
             self.generic_visit(node)
 
+        def finalize_env_vars(self, filename, env_vars_found):
+            for env_var, context in self.current_env_vars.items(): # context is now a dict with default and type
+                if env_var not in env_vars_found:
+                    env_vars_found[env_var] = {"files": set(), "default": None, "type": "str"} # Initialize type as str if not found before
+                env_vars_found[env_var]["files"].add(filename)
+                if env_vars_found[env_var]["default"] is None:  # Only set default if not already set
+                    env_vars_found[env_var]["default"] = context["default"]
+                if env_vars_found[env_var]["type"] == "str": # Only set type if still default str, otherwise keep more specific type
+                     env_vars_found[env_var]["type"] = context["type"]
+
+
     visitor = EnvVarVisitor()
     visitor.visit(tree)
-    return visitor.env_vars
+    visitor.finalize_env_vars(filename, env_vars_found)  # Pass filename to finalize
+    return env_vars_found
 
 
 def main():
@@ -65,15 +192,24 @@ def main():
     ]
     filenames = ["config.py", "env.py", "migrations/env.py"]
 
-    all_env_vars = set()
+    all_env_vars_with_context = {}  # Changed to dictionary to store context
 
     try:
         for url, filename in zip(urls, filenames):
             with urllib.request.urlopen(url) as response:
                 contents = response.read().decode("utf-8")
 
-            for env_var in find_env_vars(contents):
-                all_env_vars.add(env_var)
+            file_env_vars = find_env_vars(contents, filename)  # Pass filename here
+            for env_var, context in file_env_vars.items():  # context is now a dict
+                if env_var not in all_env_vars_with_context:
+                    all_env_vars_with_context[env_var] = {"files": set(), "default": None, "type": "str"} # Initialize type as str
+                all_env_vars_with_context[env_var]["files"].update(context["files"])  # Merge file sets
+                if all_env_vars_with_context[env_var]["default"] is None:  # Only set default if not already set
+                    all_env_vars_with_context[env_var]["default"] = context["default"]
+                if all_env_vars_with_context[env_var]["type"] == "str": # Only update type if still default str, keep more specific type
+                    all_env_vars_with_context[env_var]["type"] = context["type"]
+
+
     except urllib.error.URLError as e:
         print(f"Failed to open URL: {e}")
         sys.exit(1)
@@ -91,7 +227,7 @@ def main():
     documented_env_vars = set()
     script_dir = os.path.dirname(os.path.abspath(__file__))
     docs_file = os.path.join(
-        script_dir, *[part for part in ["..", "docs", "getting-started", "advanced-topics", "env-configuration.md"]]
+        script_dir, *[part for part in ["..", "docs", "getting-started", "env-configuration.md"]]
     )
 
     try:
@@ -105,14 +241,48 @@ def main():
         sys.exit(1)
 
     print("\nEnvironment variables accessed but not documented:")
-    not_documented_env_vars = all_env_vars - documented_env_vars - ignored_env_vars
-    for env_var in sorted(not_documented_env_vars):
-        print(env_var)
-    if not not_documented_env_vars:
-        print("None")
+    not_documented_env_vars_with_context = {
+        env_var: context
+        for env_var, context in all_env_vars_with_context.items()
+        if env_var not in documented_env_vars and env_var not in ignored_env_vars
+    }
+
+    persistent_config_vars = {}
+    other_undocumented_vars = {}
+
+    for env_var, context in not_documented_env_vars_with_context.items():
+        if "config.py" in context["files"]:  # Check if 'config.py' is in the set of files
+            persistent_config_vars[env_var] = context
+        else:
+            other_undocumented_vars[env_var] = context
+
+    def format_default_output(default, var_type):
+        if default is None:
+            return "(default: None)"
+        elif var_type == "list[dict]" or var_type == "dict" or var_type == "tuple" or var_type == "set":
+            return f"(default: {default}, type: {var_type})" # Show full default for complex types
+        else:
+            return f"(default: '{default}', type: {var_type})" # Quote string defaults
+
+    if persistent_config_vars:
+        print("\n  PersistentConfig environment variables (accessed in config.py):")
+        for env_var in sorted(persistent_config_vars.keys()):
+            default_str = format_default_output(persistent_config_vars[env_var]['default'], persistent_config_vars[env_var]['type'])
+            print(f"    - {env_var} {default_str}")
+
+    if other_undocumented_vars:
+        print("\n  Other undocumented environment variables:")
+        for env_var in sorted(other_undocumented_vars.keys()):
+            default_str = format_default_output(other_undocumented_vars[env_var]['default'], other_undocumented_vars[env_var]['type'])
+            print(
+                f"    - {env_var} {default_str} (in files: {', '.join(sorted(other_undocumented_vars[env_var]['files']))})"
+            )  # Show files and defaults and types
+
+    if not persistent_config_vars and not other_undocumented_vars:
+        print("  None")
 
     print("\nEnvironment variables documented but not accessed:")
-    diff = documented_env_vars - all_env_vars - ignored_env_vars
+    diff = documented_env_vars - set(all_env_vars_with_context.keys()) - ignored_env_vars  # Use keys of the dict
     for env_var in sorted(diff):
         print(env_var)
     if not diff:
@@ -120,4 +290,4 @@ def main():
 
 
 if __name__ == "__main__":
-    main()
\ No newline at end of file
+    main()