jl-db-comp 0.1.0__py3-none-any.whl → 0.1.1__py3-none-any.whl

jl_db_comp/_version.py

@@ -1,4 +1,4 @@
 # This file is auto-generated by Hatchling. As such, do not:
 #   - modify
 #   - track in version control e.g. be sure to add to .gitignore
-__version__ = VERSION = '0.1.0'
+__version__ = VERSION = '0.1.1'

jl_db_comp/connections.py

@@ -0,0 +1,173 @@
+"""
+Connection management module for jupysql connections.ini integration.
+
+This module handles reading database connection configurations from jupysql's
+connections.ini file format and building connection URLs from them.
+"""
+
+import configparser
+import os
+from pathlib import Path
+from typing import Optional
+from urllib.parse import quote_plus
+
+
+# Default locations to search for connections.ini
+DEFAULT_CONNECTIONS_PATHS = [
+    Path.home() / '.jupysql' / 'connections.ini',  # jupysql default
+    Path.cwd() / 'connections.ini',  # Current working directory
+]
+
+
+def find_connections_file(custom_path: Optional[str] = None) -> Optional[Path]:
+    """Find the connections.ini file.
+
+    Args:
+        custom_path: Optional custom path to connections.ini file
+
+    Returns:
+        Path to the connections.ini file, or None if not found
+    """
+    # Check custom path first
+    if custom_path:
+        custom = Path(custom_path)
+        if custom.exists():
+            return custom
+
+    # Check default locations
+    for path in DEFAULT_CONNECTIONS_PATHS:
+        if path.exists():
+            return path
+
+    return None
+
+
+def parse_connections_file(file_path: Path) -> dict[str, dict[str, str]]:
+    """Parse connections.ini file into a dictionary.
+
+    Args:
+        file_path: Path to the connections.ini file
+
+    Returns:
+        Dictionary mapping connection names to their configuration
+    """
+    config = configparser.ConfigParser()
+    config.read(file_path)
+
+    connections = {}
+    for section in config.sections():
+        connections[section] = dict(config[section])
+
+    return connections
+
+
+def build_connection_url(connection_config: dict[str, str]) -> str:
+    """Build a database URL from connection configuration.
+
+    Args:
+        connection_config: Dictionary with connection parameters:
+            - drivername: database backend (e.g., 'postgresql')
+            - username: database user
+            - password: database password
+            - host: database host
+            - port: database port
+            - database: database name
+            - query: optional query parameters (as string or dict)
+
+    Returns:
+        Connection URL string
+    """
+    drivername = connection_config.get('drivername', 'postgresql')
+    username = connection_config.get('username', '')
+    password = connection_config.get('password', '')
+    host = connection_config.get('host', 'localhost')
+    port = connection_config.get('port', '5432')
+    database = connection_config.get('database', '')
+
+    # URL encode username and password for special characters
+    if username:
+        username = quote_plus(username)
+    if password:
+        password = quote_plus(password)
+
+    # Build the URL
+    if username and password:
+        auth = f'{username}:{password}@'
+    elif username:
+        auth = f'{username}@'
+    else:
+        auth = ''
+
+    url = f'{drivername}://{auth}{host}:{port}/{database}'
+
+    # Handle query parameters if present
+    query = connection_config.get('query', '')
+    if query:
+        # Query could be a string representation of a dict or a plain string
+        if isinstance(query, str) and query.startswith('{'):
+            # Try to parse as dict-like string
+            try:
+                import ast
+                query_dict = ast.literal_eval(query)
+                if isinstance(query_dict, dict):
+                    query_str = '&'.join(
+                        f'{k}={v}' for k, v in query_dict.items()
+                    )
+                    url = f'{url}?{query_str}'
+            except (ValueError, SyntaxError):
+                pass
+        elif isinstance(query, str) and query:
+            url = f'{url}?{query}'
+
+    return url
+
+
+def get_connection_url(
+    connection_name: str,
+    connections_file_path: Optional[str] = None
+) -> Optional[str]:
+    """Get a connection URL by name from connections.ini.
+
+    Args:
+        connection_name: Name of the connection (section name in ini file)
+        connections_file_path: Optional custom path to connections.ini
+
+    Returns:
+        Connection URL string, or None if connection not found
+    """
+    file_path = find_connections_file(connections_file_path)
+    if not file_path:
+        return None
+
+    connections = parse_connections_file(file_path)
+    if connection_name not in connections:
+        return None
+
+    return build_connection_url(connections[connection_name])
+
+
+def list_connections(
+    connections_file_path: Optional[str] = None
+) -> dict[str, dict]:
+    """List all available connections from connections.ini.
+
+    Args:
+        connections_file_path: Optional custom path to connections.ini
+
+    Returns:
+        Dictionary mapping connection names to their config (without password)
+    """
+    file_path = find_connections_file(connections_file_path)
+    if not file_path:
+        return {}
+
+    connections = parse_connections_file(file_path)
+
+    # Return connections without exposing passwords
+    safe_connections = {}
+    for name, config in connections.items():
+        safe_config = {k: v for k, v in config.items() if k != 'password'}
+        safe_config['has_password'] = 'password' in config
+        safe_connections[name] = safe_config
+
+    return safe_connections

jl_db_comp/routes.py

@@ -1,11 +1,16 @@
 import json
-import os
 from urllib.parse import unquote
 
 from jupyter_server.base.handlers import APIHandler
 from jupyter_server.utils import url_path_join
 import tornado
 
+from .connections import (
+    find_connections_file,
+    get_connection_url,
+    list_connections,
+)
+
 try:
     import psycopg2
     PSYCOPG2_AVAILABLE = True
@@ -21,7 +26,8 @@ class PostgresCompletionsHandler(APIHandler):
         """Fetch completions from PostgreSQL database.
 
         Query parameters:
-        - db_url: URL-encoded PostgreSQL connection string
+        - connection: Connection name from connections.ini (preferred)
+        - db_url: URL-encoded PostgreSQL connection string (fallback)
         - prefix: Optional prefix to filter results
         - schema: Database schema (default: 'public')
         - table: Optional table name to filter columns (only returns columns from this table)
@@ -36,6 +42,8 @@ class PostgresCompletionsHandler(APIHandler):
             return
 
         try:
+            connection_name = self.get_argument('connection', None)
+            connections_file = self.get_argument('connections_file', None)
             db_url = self.get_argument('db_url', None)
             prefix = self.get_argument('prefix', '').lower()
             schema = self.get_argument('schema', 'public')
@@ -44,9 +52,20 @@ class PostgresCompletionsHandler(APIHandler):
             jsonb_column = self.get_argument('jsonb_column', None)
             jsonb_path_str = self.get_argument('jsonb_path', None)
 
-            if not db_url:
-                db_url = os.environ.get('POSTGRES_URL')
-            else:
+            # Priority: connection name -> db_url parameter
+            if connection_name:
+                db_url = get_connection_url(connection_name, connections_file)
+                if not db_url:
+                    file_info = f" (searched in: {connections_file})" if connections_file else ""
+                    self.finish(json.dumps({
+                        "status": "error",
+                        "tables": [],
+                        "columns": [],
+                        "jsonbKeys": [],
+                        "message": f"Connection '{connection_name}' not found in connections.ini{file_info}"
+                    }))
+                    return
+            elif db_url:
                 db_url = unquote(db_url)
 
             if not db_url:
@@ -55,7 +74,7 @@ class PostgresCompletionsHandler(APIHandler):
                     "tables": [],
                     "columns": [],
                     "jsonbKeys": [],
-                    "message": "No database URL provided"
+                    "message": "No connection specified. Configure a connection in connections.ini or provide connection name."
                 }))
                 return
 
@@ -276,9 +295,18 @@ class PostgresCompletionsHandler(APIHandler):
 
                 result = cursor.fetchone()
                 if not result:
+                    self.log.warning(
+                        f"JSONB completion: No JSONB column '{jsonb_column}' found "
+                        f"in schema '{schema}'. Verify the column exists and has "
+                        f"data_type='jsonb'."
+                    )
                     return []
 
                 table_name = result[0]
+                self.log.info(
+                    f"JSONB completion: Found column '{jsonb_column}' in "
+                    f"table '{schema}.{table_name}'"
+                )
 
             # Build the JSONB path expression
             if jsonb_path and len(jsonb_path) > 0:
@@ -290,6 +318,44 @@ class PostgresCompletionsHandler(APIHandler):
                 # For top-level keys: just the column
                 path_expr = jsonb_column
 
+            # First, check the data distribution at this path for diagnostics
+            diag_query = f"""
+                SELECT
+                    COUNT(*) as total_rows,
+                    COUNT({path_expr}) as non_null_count,
+                    COUNT(CASE WHEN jsonb_typeof({path_expr}) = 'object' THEN 1 END) as object_count,
+                    COUNT(CASE WHEN jsonb_typeof({path_expr}) = 'array' THEN 1 END) as array_count,
+                    COUNT(CASE WHEN jsonb_typeof({path_expr}) IN ('string', 'number', 'boolean') THEN 1 END) as scalar_count
+                FROM {schema}.{table_name}
+                LIMIT 1000
+            """
+            cursor.execute(diag_query)
+            diag = cursor.fetchone()
+
+            total_rows, non_null, obj_count, arr_count, scalar_count = diag
+
+            if non_null == 0:
+                self.log.warning(
+                    f"JSONB completion: Column '{jsonb_column}' in "
+                    f"'{schema}.{table_name}' has no non-NULL values at "
+                    f"path '{path_expr}'. Keys cannot be extracted from NULL data."
+                )
+                return []
+
+            if obj_count == 0:
+                type_info = []
+                if arr_count > 0:
+                    type_info.append(f"{arr_count} arrays")
+                if scalar_count > 0:
+                    type_info.append(f"{scalar_count} scalars")
+                self.log.warning(
+                    f"JSONB completion: Path '{path_expr}' in "
+                    f"'{schema}.{table_name}' contains no JSON objects "
+                    f"(found: {', '.join(type_info) if type_info else 'only NULL'}). "
+                    f"Keys can only be extracted from object types."
+                )
+                return []
+
             # Query to extract unique keys
             # LIMIT to 1000 rows for performance (sample the table)
             query = f"""
@@ -303,6 +369,14 @@ class PostgresCompletionsHandler(APIHandler):
             cursor.execute(query)
             keys = cursor.fetchall()
 
+            if len(keys) == 0:
+                self.log.warning(
+                    f"JSONB completion: No keys found at path '{path_expr}' in "
+                    f"'{schema}.{table_name}' despite {obj_count} objects. "
+                    f"Objects may be empty {{}}."
+                )
+                return []
+
             # Filter by prefix and format results
             result = []
             for row in keys:
@@ -314,6 +388,11 @@ class PostgresCompletionsHandler(APIHandler):
                         "keyPath": (jsonb_path or []) + [key]
                     })
 
+            self.log.info(
+                f"JSONB completion: Found {len(keys)} unique keys at '{path_expr}' "
+                f"in '{schema}.{table_name}' (sampled {obj_count} objects)"
+            )
+
             return result
 
         except psycopg2.Error as e:
@@ -321,12 +400,279 @@ class PostgresCompletionsHandler(APIHandler):
             return []
 
 
+class JsonbDiagnosticsHandler(APIHandler):
+    """Handler for diagnosing JSONB column issues."""
+
+    @tornado.web.authenticated
+    def get(self):
+        """Get diagnostic information about JSONB columns.
+
+        Query parameters:
+        - connection: Connection name from connections.ini (preferred)
+        - db_url: URL-encoded PostgreSQL connection string (fallback)
+        - schema: Database schema (default: 'public')
+        - table: Optional table name to check
+        - column: Optional JSONB column name to check
+        - jsonb_path: Optional JSON-encoded path array for nested diagnostics
+        """
+        if not PSYCOPG2_AVAILABLE:
+            self.set_status(500)
+            self.finish(json.dumps({
+                "status": "error",
+                "message": "psycopg2 is not installed"
+            }))
+            return
+
+        try:
+            connection_name = self.get_argument('connection', None)
+            db_url = self.get_argument('db_url', None)
+            schema = self.get_argument('schema', 'public')
+            table = self.get_argument('table', None)
+            column = self.get_argument('column', None)
+            jsonb_path_str = self.get_argument('jsonb_path', None)
+
+            # Priority: connection name -> db_url parameter
+            if connection_name:
+                db_url = get_connection_url(connection_name)
+                if not db_url:
+                    self.finish(json.dumps({
+                        "status": "error",
+                        "message": f"Connection '{connection_name}' not found in connections.ini"
+                    }))
+                    return
+            elif db_url:
+                db_url = unquote(db_url)
+
+            if not db_url:
+                self.finish(json.dumps({
+                    "status": "error",
+                    "message": "No connection specified. Configure a connection in connections.ini."
+                }))
+                return
+
+            jsonb_path = None
+            if jsonb_path_str:
+                try:
+                    jsonb_path = json.loads(jsonb_path_str)
+                except json.JSONDecodeError:
+                    jsonb_path = []
+
+            diagnostics = self._get_diagnostics(
+                db_url, schema, table, column, jsonb_path
+            )
+            self.finish(json.dumps(diagnostics))
+
+        except psycopg2.Error as e:
+            error_msg = str(e).split('\n')[0]
+            self.log.error(f"JSONB diagnostics error: {error_msg}")
+            self.set_status(500)
+            self.finish(json.dumps({
+                "status": "error",
+                "message": f"Database error: {error_msg}"
+            }))
+        except Exception as e:
+            error_msg = str(e)
+            self.log.error(f"JSONB diagnostics error: {error_msg}")
+            self.set_status(500)
+            self.finish(json.dumps({
+                "status": "error",
+                "message": f"Server error: {error_msg}"
+            }))
+
+    def _get_diagnostics(
+        self,
+        db_url: str,
+        schema: str,
+        table: str = None,
+        column: str = None,
+        jsonb_path: list = None
+    ) -> dict:
+        """Get diagnostic information about JSONB columns."""
+        conn = None
+        try:
+            conn = psycopg2.connect(db_url)
+            cursor = conn.cursor()
+
+            result = {
+                "status": "success",
+                "schema": schema,
+                "jsonbColumns": [],
+                "columnDiagnostics": None
+            }
+
+            # Find all JSONB columns in the schema
+            query_params = [schema]
+            query = """
+                SELECT table_name, column_name
+                FROM information_schema.columns
+                WHERE table_schema = %s
+                  AND data_type = 'jsonb'
+            """
+            if table:
+                query += " AND LOWER(table_name) = %s"
+                query_params.append(table.lower())
+            if column:
+                query += " AND LOWER(column_name) = %s"
+                query_params.append(column.lower())
+
+            query += " ORDER BY table_name, column_name"
+
+            cursor.execute(query, query_params)
+            jsonb_columns = cursor.fetchall()
+
+            result["jsonbColumns"] = [
+                {"table": row[0], "column": row[1]}
+                for row in jsonb_columns
+            ]
+
+            # If specific table and column provided, get detailed diagnostics
+            if table and column and len(jsonb_columns) > 0:
+                actual_table = jsonb_columns[0][0]
+                actual_column = jsonb_columns[0][1]
+
+                # Build path expression
+                if jsonb_path and len(jsonb_path) > 0:
+                    path_expr = actual_column
+                    for key in jsonb_path:
+                        path_expr = f"{path_expr}->'{key}'"
+                else:
+                    path_expr = actual_column
+
+                # Get type distribution
+                diag_query = f"""
+                    SELECT
+                        COUNT(*) as total_rows,
+                        COUNT({path_expr}) as non_null_count,
+                        COUNT(CASE WHEN jsonb_typeof({path_expr}) = 'object' THEN 1 END) as object_count,
+                        COUNT(CASE WHEN jsonb_typeof({path_expr}) = 'array' THEN 1 END) as array_count,
+                        COUNT(CASE WHEN jsonb_typeof({path_expr}) = 'string' THEN 1 END) as string_count,
+                        COUNT(CASE WHEN jsonb_typeof({path_expr}) = 'number' THEN 1 END) as number_count,
+                        COUNT(CASE WHEN jsonb_typeof({path_expr}) = 'boolean' THEN 1 END) as boolean_count,
+                        COUNT(CASE WHEN jsonb_typeof({path_expr}) = 'null' THEN 1 END) as json_null_count
+                    FROM {schema}.{actual_table}
+                """
+                cursor.execute(diag_query)
+                diag = cursor.fetchone()
+
+                result["columnDiagnostics"] = {
+                    "table": actual_table,
+                    "column": actual_column,
+                    "pathExpression": path_expr,
+                    "totalRows": diag[0],
+                    "nonNullCount": diag[1],
+                    "typeDistribution": {
+                        "object": diag[2],
+                        "array": diag[3],
+                        "string": diag[4],
+                        "number": diag[5],
+                        "boolean": diag[6],
+                        "null": diag[7]
+                    },
+                    "canExtractKeys": diag[2] > 0,
+                    "recommendation": self._get_recommendation(diag)
+                }
+
+                # If there are objects, get sample keys
+                if diag[2] > 0:
+                    try:
+                        key_query = f"""
+                            SELECT DISTINCT jsonb_object_keys({path_expr})
+                            FROM {schema}.{actual_table}
+                            WHERE {path_expr} IS NOT NULL
+                              AND jsonb_typeof({path_expr}) = 'object'
+                            LIMIT 20
+                        """
+                        cursor.execute(key_query)
+                        keys = [row[0] for row in cursor.fetchall()]
+                        result["columnDiagnostics"]["sampleKeys"] = keys
+                    except psycopg2.Error:
+                        result["columnDiagnostics"]["sampleKeys"] = []
+
+            cursor.close()
+            return result
+
+        finally:
+            if conn:
+                conn.close()
+
+    def _get_recommendation(self, diag) -> str:
+        """Generate a recommendation based on diagnostic data."""
+        total, non_null, obj, arr, string, number, boolean, json_null = diag
+
+        if non_null == 0:
+            return (
+                "All values are NULL. JSONB autocompletion requires non-NULL data. "
+                "Check that the column contains actual JSON data."
+            )
+
+        if obj == 0:
+            types_found = []
+            if arr > 0:
+                types_found.append(f"{arr} arrays")
+            if string > 0:
+                types_found.append(f"{string} strings")
+            if number > 0:
+                types_found.append(f"{number} numbers")
+            if boolean > 0:
+                types_found.append(f"{boolean} booleans")
+            if json_null > 0:
+                types_found.append(f"{json_null} JSON nulls")
+
+            return (
+                f"No JSON objects found. Found: {', '.join(types_found)}. "
+                f"JSONB key extraction only works with object types ({{}}). "
+                f"If your data contains arrays, you may need to navigate into "
+                f"array elements first."
+            )
+
+        return f"JSONB autocompletion should work. Found {obj} objects with extractable keys."
+
+
+class ConnectionsHandler(APIHandler):
+    """Handler for listing available database connections."""
+
+    @tornado.web.authenticated
+    def get(self):
+        """List available connections from connections.ini.
+
+        Returns:
+            JSON response with:
+            - connections: Dictionary of available connections (without passwords)
+            - file_path: Path to the connections.ini file found
+        """
+        try:
+            connections = list_connections()
+            file_path = find_connections_file()
+
+            self.finish(json.dumps({
+                "status": "success",
+                "connections": connections,
+                "filePath": str(file_path) if file_path else None
+            }))
+
+        except Exception as e:
+            self.log.error(f"Error listing connections: {e}")
+            self.set_status(500)
+            self.finish(json.dumps({
+                "status": "error",
+                "message": f"Error reading connections: {str(e)}",
+                "connections": {}
+            }))
+
+
 def setup_route_handlers(web_app):
     """Register route handlers with the Jupyter server."""
     host_pattern = ".*$"
     base_url = web_app.settings["base_url"]
 
     completions_route = url_path_join(base_url, "jl-db-comp", "completions")
-    handlers = [(completions_route, PostgresCompletionsHandler)]
+    diagnostics_route = url_path_join(base_url, "jl-db-comp", "jsonb-diagnostics")
+    connections_route = url_path_join(base_url, "jl-db-comp", "connections")
+
+    handlers = [
+        (completions_route, PostgresCompletionsHandler),
+        (diagnostics_route, JsonbDiagnosticsHandler),
+        (connections_route, ConnectionsHandler),
+    ]
 
     web_app.add_handlers(host_pattern, handlers)

jl_db_comp/tests/test_routes.py

@@ -15,7 +15,7 @@ async def test_completions_no_db_url(jp_fetch):
     assert payload["status"] == "success"
     assert payload["tables"] == []
     assert payload["columns"] == []
-    assert "No database URL provided" in payload.get("message", "")
+    assert "No connection specified" in payload.get("message", "")
 
 
 async def test_completions_with_invalid_db_url(jp_fetch):

{jl_db_comp/labextension → jl_db_comp-0.1.1.data/data/share/jupyter/labextensions/jl_db_comp}/package.json

@@ -1,6 +1,6 @@
 {
   "name": "jl_db_comp",
-  "version": "0.1.0",
+  "version": "0.1.1",
   "description": "A JupyterLab extension to complete db queries in jupyterlab notebooks",
   "keywords": [
     "jupyter",
@@ -60,6 +60,7 @@
     "@jupyterlab/application": "^4.0.0",
     "@jupyterlab/completer": "^4.0.0",
     "@jupyterlab/coreutils": "^6.0.0",
+    "@jupyterlab/notebook": "^4.0.0",
     "@jupyterlab/services": "^7.0.0",
     "@jupyterlab/settingregistry": "^4.0.0"
   },
@@ -117,7 +118,7 @@
     "outputDir": "jl_db_comp/labextension",
     "schemaDir": "schema",
     "_build": {
-      "load": "static/remoteEntry.5763ae02737e035e938c.js",
+      "load": "static/remoteEntry.48c365d3de0d860be537.js",
       "extension": "./extension",
       "style": "./style"
     }

{jl_db_comp-0.1.0.data → jl_db_comp-0.1.1.data}/data/share/jupyter/labextensions/jl_db_comp/schemas/jl_db_comp/package.json.orig

@@ -1,6 +1,6 @@
 {
     "name": "jl_db_comp",
-    "version": "0.1.0",
+    "version": "0.1.1",
     "description": "A JupyterLab extension to complete db queries in jupyterlab notebooks",
     "keywords": [
         "jupyter",
@@ -60,6 +60,7 @@
         "@jupyterlab/application": "^4.0.0",
         "@jupyterlab/completer": "^4.0.0",
         "@jupyterlab/coreutils": "^6.0.0",
+        "@jupyterlab/notebook": "^4.0.0",
         "@jupyterlab/services": "^7.0.0",
         "@jupyterlab/settingregistry": "^4.0.0"
     },