det-core 0.2.2__tar.gz → 0.2.3__tar.gz

{det_core-0.2.2 → det_core-0.2.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: det-core
-Version: 0.2.2
+Version: 0.2.3
 Summary: Fast, local-first SQL ETL workspace and pipeline orchestration engine powered by Apache DataFusion
 Project-URL: Homepage, https://github.com/det-Labs/det
 Project-URL: Repository, https://github.com/det-Labs/det
@@ -25,6 +25,8 @@ Classifier: Topic :: Scientific/Engineering :: Information Analysis
 Classifier: Topic :: Software Development :: Build Tools
 Classifier: Topic :: Software Development :: Libraries :: Python Modules
 Requires-Python: >=3.14
+Requires-Dist: adbc-driver-manager>=1.0.0
+Requires-Dist: adbc-driver-postgresql>=1.0.0
 Requires-Dist: datafusion<56,>=55.0.0
 Requires-Dist: deltalake>=1.0.0
 Requires-Dist: fastexcel>=0.20.0

{det_core-0.2.2 → det_core-0.2.3}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "det-core"
-version = "0.2.2"
+version = "0.2.3"
 description = "Fast, local-first SQL ETL workspace and pipeline orchestration engine powered by Apache DataFusion"
 readme = "README.md"
 license = "MIT"
@@ -52,6 +52,8 @@ dependencies = [
     "fastexcel>=0.20.0",
     "openpyxl>=3.1.0",
     "deltalake>=1.0.0",
+    "adbc-driver-postgresql>=1.0.0",
+    "adbc-driver-manager>=1.0.0",
 ]
 
 [project.optional-dependencies]
@@ -74,3 +76,6 @@ Issues = "https://github.com/det-Labs/det/issues"
 
 [tool.hatch.build.targets.wheel]
 packages = ["src/det"]
+
+[tool.pytest.ini_options]
+pythonpath = ["."]

{det_core-0.2.2 → det_core-0.2.3}/src/det/__init__.py

@@ -1,4 +1,4 @@
 from det.workspace import Workspace
 
-__version__ = "0.2.2"
+__version__ = "0.2.3"
 __all__ = ["Workspace", "__version__"]

{det_core-0.2.2 → det_core-0.2.3}/src/det/api.py

@@ -3,9 +3,10 @@
 from dataclasses import asdict, dataclass
 from pathlib import Path
 from typing import Any
+from uuid import uuid4
 
 from det import __version__
-from det.catalog import CatalogStore
+from det.catalog import CatalogStore, resolve_connection_options
 from det.connectors import connector_metadata, scan_builtin
 from det.engine import create_session_context, run_sql
 from det.monitoring import MonitoringStore
@@ -90,11 +91,13 @@ def infer_connection_schema(connection_id: str) -> dict[str, Any]:
     if not connection["is_source"]:
         raise ValueError("Schema inference is available only for source connections.")
     workspace = get_ui_state()["workspace"]
+    catalog_doc = store.read()["catalog"]
+    options = resolve_connection_options(connection, catalog_doc)
     context = create_session_context()
     dataframe = scan_builtin(
         context,
         connection["connector"],
-        connection["options"],
+        options,
         workspace,
     )
     inferred = [
@@ -116,11 +119,13 @@ def preview_connection(connection_id: str) -> dict[str, Any]:
     if not connection["is_source"]:
         raise ValueError("Data preview is available only for source connections.")
     workspace = get_ui_state()["workspace"]
+    catalog_doc = store.read()["catalog"]
+    options = resolve_connection_options(connection, catalog_doc)
     context = create_session_context()
     dataframe = scan_builtin(
         context,
         connection["connector"],
-        connection["options"],
+        options,
         workspace,
         saved_columns=connection["columns"] or None,
     )
@@ -146,6 +151,187 @@ def preview_connection(connection_id: str) -> dict[str, Any]:
     }
 
 
+def preview_adhoc_connection(
+    connector: str,
+    options: dict[str, Any],
+    columns: list[dict[str, Any]] | None = None,
+    limit: int = 25,
+) -> dict[str, Any]:
+    """Scan and preview up to limit rows directly from connection parameters without registration."""
+    import time
+
+    start_time = time.perf_counter()
+    workspace = get_ui_state()["workspace"]
+    context = create_session_context()
+    try:
+        from det.connectors import connector_definitions
+        for c_def in connector_definitions():
+            if connector in (c_def.key, c_def.guid):
+                connector = c_def.guid
+                break
+
+        dataframe = scan_builtin(
+            context,
+            connector,
+            options,
+            workspace,
+            saved_columns=columns or None,
+        )
+        view_name = f"det_adhoc_{uuid4().hex[:8]}"
+        context.register_view(view_name, dataframe)
+        preview = run_sql(context, f"SELECT * FROM {view_name} LIMIT {limit}")
+        batches = preview.collect()
+        import pyarrow as pa
+
+        rows = pa.Table.from_batches(batches, schema=preview.schema()).to_pylist() if batches else []
+        elapsed_ms = round((time.perf_counter() - start_time) * 1000, 1)
+        inferred_columns = [
+            {
+                "name": field.name,
+                "data_type": str(field.type),
+                "nullable": field.nullable,
+                "description": "",
+            }
+            for field in preview.schema()
+        ]
+        return {
+            "success": True,
+            "columns": columns or inferred_columns,
+            "inferred_columns": inferred_columns,
+            "rows": rows,
+            "row_count": len(rows),
+            "row_limit": limit,
+            "elapsed_ms": elapsed_ms,
+        }
+    except Exception as exc:
+        return {
+            "success": False,
+            "error": str(exc),
+            "elapsed_ms": round((time.perf_counter() - start_time) * 1000, 1),
+        }
+
+
+def test_database_connection(
+    connector: str,
+    options: dict[str, Any],
+    profile_id: str | None = None,
+) -> dict[str, Any]:
+    """Test connectivity to a database without fetching tables."""
+    import time
+
+    start_time = time.perf_counter()
+    workspace = get_ui_state()["workspace"]
+    try:
+        from det.connectors import build_postgres_uri, connector_definitions, resolve_env_vars
+
+        if profile_id:
+            store = get_catalog_store()
+            profile = store.get_profile(profile_id)
+            connector = profile["connector"]
+            options = {**profile["options"], **options}
+
+        connector_key = connector
+        for c_def in connector_definitions():
+            if connector in (c_def.key, c_def.guid):
+                connector_key = c_def.key
+                break
+
+        resolved_options = resolve_env_vars(options, workspace)
+
+        if connector_key == "postgresql":
+            import adbc_driver_postgresql.dbapi as pg_dbapi
+
+            uri = build_postgres_uri(resolved_options)
+            with pg_dbapi.connect(uri) as conn, conn.cursor() as cursor:
+                cursor.execute("SELECT 1;")
+                cursor.fetchone()
+                elapsed_ms = round((time.perf_counter() - start_time) * 1000, 1)
+                return {
+                    "success": True,
+                    "message": "Connected successfully!",
+                    "elapsed_ms": elapsed_ms,
+                }
+
+        return {
+            "success": False,
+            "error": f"Connector '{connector_key}' does not support connection testing.",
+        }
+    except Exception as exc:
+        elapsed_ms = round((time.perf_counter() - start_time) * 1000, 1)
+        return {
+            "success": False,
+            "error": str(exc),
+            "elapsed_ms": elapsed_ms,
+        }
+
+
+def discover_database_tables(
+    connector: str,
+    options: dict[str, Any],
+    profile_id: str | None = None,
+) -> dict[str, Any]:
+    """Discover available schemas and tables from a database connection."""
+    import time
+
+    start_time = time.perf_counter()
+    workspace = get_ui_state()["workspace"]
+    try:
+        from det.connectors import build_postgres_uri, connector_definitions, resolve_env_vars
+
+        if profile_id:
+            store = get_catalog_store()
+            profile = store.get_profile(profile_id)
+            connector = profile["connector"]
+            options = {**profile["options"], **options}
+
+        connector_key = connector
+        for c_def in connector_definitions():
+            if connector in (c_def.key, c_def.guid):
+                connector_key = c_def.key
+                break
+
+        resolved_options = resolve_env_vars(options, workspace)
+
+        if connector_key == "postgresql":
+            import adbc_driver_postgresql.dbapi as pg_dbapi
+
+            uri = build_postgres_uri(resolved_options)
+            with pg_dbapi.connect(uri) as conn, conn.cursor() as cursor:
+                cursor.execute(
+                    "SELECT table_schema, table_name "
+                    "FROM information_schema.tables "
+                    "WHERE table_schema NOT IN ('information_schema', 'pg_catalog') "
+                    "ORDER BY table_schema, table_name;"
+                )
+                table = cursor.fetch_arrow_table()
+                rows = table.to_pylist() if table else []
+                tables = [
+                    {"schema": r["table_schema"], "name": r["table_name"]}
+                    for r in rows
+                ]
+                elapsed_ms = round((time.perf_counter() - start_time) * 1000, 1)
+                return {
+                    "success": True,
+                    "tables": tables,
+                    "count": len(tables),
+                    "elapsed_ms": elapsed_ms,
+                }
+
+        return {
+            "success": False,
+            "error": f"Connector '{connector_key}' does not support table discovery.",
+            "tables": [],
+        }
+    except Exception as exc:
+        elapsed_ms = round((time.perf_counter() - start_time) * 1000, 1)
+        return {
+            "success": False,
+            "error": str(exc),
+            "tables": [],
+            "elapsed_ms": elapsed_ms,
+        }
+
+
 def get_pipeline_store() -> PipelineStore:
     """Return the workspace pipeline store configured for this process."""
     if _pipeline_store is None:

{det_core-0.2.2 → det_core-0.2.3}/src/det/catalog.py

@@ -21,12 +21,46 @@ def _canonical_guid(value: Any) -> str:
         return str(uuid.uuid5(uuid.NAMESPACE_URL, f"det:legacy-id:{value}"))
 
 
+class CatalogProfile(BaseModel):
+    model_config = ConfigDict(extra="forbid")
+
+    id: str = Field(default_factory=lambda: str(uuid.uuid4()))
+    name: str = Field(min_length=1, max_length=100)
+    connector: str
+    description: str = Field(default="", max_length=2000)
+    options: dict[str, Any] = Field(default_factory=dict)
+
+    @field_validator("id", mode="before")
+    @classmethod
+    def canonicalize_id(cls, value: Any) -> str:
+        return _canonical_guid(value) if value is not None else str(uuid.uuid4())
+
+    @field_validator("connector")
+    @classmethod
+    def validate_connector_guid(cls, value: str) -> str:
+        try:
+            return str(uuid.UUID(value))
+        except ValueError as error:
+            raise ValueError("Connector type must be identified by a GUID.") from error
+
+    @field_validator("name")
+    @classmethod
+    def validate_profile_name(cls, value: str) -> str:
+        if not re.fullmatch(r"[A-Za-z][A-Za-z0-9_]{0,99}", value):
+            raise ValueError(
+                "Profile names must start with a letter, be at most 100 characters, "
+                "and contain only letters, numbers, and underscores."
+            )
+        return value
+
+
 class CatalogConnection(BaseModel):
     model_config = ConfigDict(extra="forbid")
 
     id: str = Field(default_factory=lambda: str(uuid.uuid4()))
-    name: str = Field(min_length=1, max_length=30)
+    name: str = Field(min_length=1, max_length=100)
     connector: str
+    profile: str | None = None
     is_source: bool = True
     is_destination: bool = False
     description: str = Field(default="", max_length=2000)
@@ -49,9 +83,9 @@ class CatalogConnection(BaseModel):
     @field_validator("name")
     @classmethod
     def validate_sql_identifier(cls, value: str) -> str:
-        if not re.fullmatch(r"[A-Za-z][A-Za-z0-9_]{0,29}", value):
+        if not re.fullmatch(r"[A-Za-z][A-Za-z0-9_]{0,99}", value):
             raise ValueError(
-                "Connection names must start with a letter, be at most 30 characters, "
+                "Connection names must start with a letter, be at most 100 characters, "
                 "and contain only letters, numbers, and underscores."
             )
         return value
@@ -100,6 +134,7 @@ class CatalogDocument(BaseModel):
     id: str = Field(default_factory=lambda: str(uuid.uuid4()))
     version: int = 1
     name: str = Field(min_length=1, max_length=120)
+    profiles: list[CatalogProfile] = Field(default_factory=list)
     folders: list[CatalogFolder] = Field(default_factory=list)
 
     @field_validator("id", mode="before")
@@ -186,16 +221,16 @@ class CatalogStore:
             self._active_filename = filename
             return self.read()
 
-    def read(self) -> dict[str, Any]:
+    def read(self, filename: str | None = None) -> dict[str, Any]:
         with self._lock:
-            filename = self._validate_filename(self._active_filename)
-            path = self.directory / filename
+            target_filename = self._validate_filename(filename or self._active_filename)
+            path = self.directory / target_filename
             try:
                 raw = yaml.safe_load(path.read_text(encoding="utf-8"))
                 if raw is None:
                     raw = {}
                 if not isinstance(raw, dict):
-                    raise ValueError(f"Catalog root must be a YAML mapping: {filename}")
+                    raise ValueError(f"Catalog root must be a YAML mapping: {target_filename}")
                 needs_migration = False
                 if "folders" not in raw or not raw["folders"]:
                     raw["folders"] = [{
@@ -210,6 +245,14 @@ class CatalogStore:
                 if "id" not in raw:
                     raw["id"] = str(uuid.uuid4())
                     needs_migration = True
+                if "profiles" not in raw or raw["profiles"] is None:
+                    raw["profiles"] = []
+                elif isinstance(raw["profiles"], dict):
+                    raw["profiles"] = [
+                        {"name": k, **v} if isinstance(v, dict) else v
+                        for k, v in raw["profiles"].items()
+                    ]
+                    needs_migration = True
                 for folder in raw["folders"]:
                     for connection in folder.get("connections", []):
                         if "direction" in connection:
@@ -219,13 +262,13 @@ class CatalogStore:
                             needs_migration = True
                 catalog = CatalogDocument.model_validate(raw)
             except (OSError, ValueError, yaml.YAMLError, ValidationError) as error:
-                raise ValueError(f"Could not load catalog {filename}: {error}") from error
+                raise ValueError(f"Could not load catalog {target_filename}: {error}") from error
             self._validate_ids(catalog)
             if needs_migration:
-                self._write_file(filename, catalog)
-                path = self.directory / filename
+                self._write_file(target_filename, catalog)
+                path = self.directory / target_filename
             return {
-                "filename": filename,
+                "filename": target_filename,
                 "catalog": catalog.model_dump(mode="json"),
                 "yaml": path.read_text(encoding="utf-8"),
             }
@@ -363,6 +406,78 @@ class CatalogStore:
                     }
         raise FileNotFoundError(f"Connection does not exist: {connection_id}")
 
+    def add_profile(self, values: dict[str, Any]) -> dict[str, Any]:
+        with self._lock:
+            catalog = self._read_document()
+            profile = CatalogProfile.model_validate(values)
+            if any(p.name.casefold() == profile.name.casefold() for p in catalog.profiles):
+                raise ValueError(f"Profile with name '{profile.name}' already exists.")
+            catalog.profiles.append(profile)
+            return self._save_document(catalog)
+
+    def update_profile(self, profile_id: str, values: dict[str, Any]) -> dict[str, Any]:
+        with self._lock:
+            catalog = self._read_document()
+            existing = None
+            for idx, p in enumerate(catalog.profiles):
+                if p.id == profile_id:
+                    existing = p
+                    updated = CatalogProfile.model_validate({
+                        **p.model_dump(),
+                        **values,
+                        "id": profile_id,
+                    })
+                    catalog.profiles[idx] = updated
+                    break
+            if not existing:
+                raise FileNotFoundError(f"Profile does not exist: {profile_id}")
+            return self._save_document(catalog)
+
+    def delete_profile(self, profile_id: str) -> dict[str, Any]:
+        with self._lock:
+            catalog = self._read_document()
+            profile = None
+            for p in catalog.profiles:
+                if p.id == profile_id:
+                    profile = p
+                    break
+            if not profile:
+                raise FileNotFoundError(f"Profile does not exist: {profile_id}")
+
+            in_use = [
+                conn.name
+                for folder in catalog.folders
+                for conn in folder.connections
+                if conn.profile in (profile.id, profile.name)
+            ]
+            if in_use:
+                raise ValueError(
+                    f"Cannot delete profile '{profile.name}' because it is in use by {len(in_use)} connection(s): {', '.join(in_use[:3])}"
+                )
+
+            catalog.profiles.remove(profile)
+            return self._save_document(catalog)
+
+    def get_profile(self, profile_id: str) -> dict[str, Any]:
+        catalog = self._read_document()
+        for p in catalog.profiles:
+            if p.id == profile_id or p.name == profile_id:
+                return p.model_dump(mode="json")
+        raise FileNotFoundError(f"Profile does not exist: {profile_id}")
+
+    def bulk_add_connections(
+        self, folder_id: str, connections_data: list[dict[str, Any]]
+    ) -> dict[str, Any]:
+        with self._lock:
+            catalog = self._read_document()
+            folder = self._folder(catalog, folder_id)
+            for values in connections_data:
+                connection_values = dict(values)
+                connection_values.pop("folder_id", None)
+                conn = CatalogConnection.model_validate(connection_values)
+                folder.connections.append(conn)
+            return self._save_document(catalog)
+
     def _read_document(self) -> CatalogDocument:
         return CatalogDocument.model_validate(self.read()["catalog"])
 
@@ -425,6 +540,8 @@ class CatalogStore:
     @staticmethod
     def _validate_ids(catalog: CatalogDocument) -> None:
         item_ids = [catalog.id]
+        if hasattr(catalog, "profiles"):
+            item_ids.extend(profile.id for profile in catalog.profiles)
         item_ids.extend(folder.id for folder in catalog.folders)
         item_ids.extend(
             item_id
@@ -444,3 +561,41 @@ class CatalogStore:
         if not normalized:
             raise ValueError(f"{item_type} name cannot be empty.")
         return normalized
+
+
+def resolve_connection_options(
+    connection: dict[str, Any] | CatalogConnection,
+    catalog_doc: dict[str, Any] | CatalogDocument | None = None,
+) -> dict[str, Any]:
+    """Merge options from referenced profile (if any) with connection options."""
+    if isinstance(connection, CatalogConnection):
+        conn_dict = connection.model_dump(mode="json")
+    else:
+        conn_dict = dict(connection)
+
+    options = dict(conn_dict.get("options") or {})
+    profile_ref = conn_dict.get("profile")
+    if not profile_ref or not catalog_doc:
+        return options
+
+    if isinstance(catalog_doc, CatalogDocument):
+        doc_dict = catalog_doc.model_dump(mode="json")
+    else:
+        doc_dict = dict(catalog_doc)
+
+    profiles = doc_dict.get("profiles") or []
+    profile = None
+    if isinstance(profiles, list):
+        for p in profiles:
+            if p.get("id") == profile_ref or p.get("name") == profile_ref:
+                profile = p
+                break
+    elif isinstance(profiles, dict):
+        profile = profiles.get(profile_ref)
+
+    if profile:
+        profile_opts = profile.get("options") or {}
+        # Profile options are base, connection options override
+        return {**profile_opts, **options}
+
+    return options