det-core 0.1.0__py3-none-any.whl

det/__init__.py

@@ -0,0 +1,3 @@
+"""det: Data Engineering Toolkit by det Labs."""
+
+__version__ = "0.1.0"

det/__main__.py

@@ -0,0 +1,7 @@
+"""Entrypoint for python -m det."""
+
+from det.cli import main
+
+if __name__ == "__main__":
+    main()
+

det/catalog/__init__.py

@@ -0,0 +1,17 @@
+"""det.catalog - YAML metadata catalog module."""
+
+from det.catalog.config import get_default_catalog_path, set_default_catalog_path
+from det.catalog.models import CatalogFile, ColumnMetadata, DatasetMetadata, ScanResult
+from det.catalog.scanner import scan_location
+from det.catalog.service import CatalogService
+
+__all__ = [
+    "CatalogFile",
+    "CatalogService",
+    "ColumnMetadata",
+    "DatasetMetadata",
+    "ScanResult",
+    "get_default_catalog_path",
+    "scan_location",
+    "set_default_catalog_path",
+]

det/catalog/config.py

@@ -0,0 +1,170 @@
+"""det.catalog.config - Manages catalog preferences and default YAML paths."""
+
+from __future__ import annotations
+
+import json
+from pathlib import Path
+from typing import Any
+
+# Local configuration directory for project-level settings
+CONFIG_DIR = Path(".det")
+CONFIG_FILE = CONFIG_DIR / "config.json"
+DEFAULT_CATALOG_FILENAME = "catalog.yaml"
+
+
+def get_default_catalog_path() -> Path:
+    """Retrieve the configured default catalog path from local configuration.
+
+    If no configuration file exists or reading fails, safely defaults to
+    './catalog.yaml' in the current working directory.
+
+    Returns:
+        Path: Absolute resolved path to the default catalog YAML file.
+    """
+    if CONFIG_FILE.is_file():
+        try:
+            with open(CONFIG_FILE, "r", encoding="utf-8") as f:
+                data = json.load(f)
+                if isinstance(data, dict) and data.get("default_catalog_path"):
+                    return Path(data["default_catalog_path"]).resolve()
+        except (json.JSONDecodeError, OSError):
+            return Path(DEFAULT_CATALOG_FILENAME).resolve()
+
+    return Path(DEFAULT_CATALOG_FILENAME).resolve()
+
+
+def set_default_catalog_path(path: str | Path) -> Path:
+    """Persist the default catalog path into the local configuration file.
+
+    Args:
+        path: Path string or Path object pointing to the catalog YAML file.
+
+    Returns:
+        Path: Absolute resolved path that was persisted.
+
+    Raises:
+        OSError: If creating directories or writing the configuration fails.
+    """
+    resolved_path = Path(path).resolve()
+    CONFIG_DIR.mkdir(parents=True, exist_ok=True)
+
+    data: dict[str, Any] = {}
+    if CONFIG_FILE.is_file():
+        try:
+            with open(CONFIG_FILE, "r", encoding="utf-8") as f:
+                loaded = json.load(f)
+                if isinstance(loaded, dict):
+                    data = loaded
+        except (json.JSONDecodeError, OSError):
+            data = {}
+
+    data["default_catalog_path"] = str(resolved_path)
+
+    # Ensure default is also in known catalogs
+    known = data.get("known_catalogs", [])
+    if not isinstance(known, list):
+        known = []
+    path_str = str(resolved_path)
+    if path_str not in known:
+        known.append(path_str)
+    data["known_catalogs"] = known
+
+    # Write configuration safely
+    with open(CONFIG_FILE, "w", encoding="utf-8") as f:
+        json.dump(data, f, indent=2)
+
+    return resolved_path
+
+
+def get_known_catalogs() -> list[Path]:
+    """Retrieve all known catalog YAML paths.
+
+    Includes the default catalog, any registered catalogs from local config,
+    and any existing 'catalog.yaml' in the project root.
+
+    Returns:
+        list[Path]: List of resolved Path objects for all known catalogs.
+    """
+    catalogs: list[Path] = []
+    default_cat = get_default_catalog_path()
+    if default_cat.is_file():
+        catalogs.append(default_cat)
+
+    if CONFIG_FILE.is_file():
+        try:
+            with open(CONFIG_FILE, "r", encoding="utf-8") as f:
+                data = json.load(f)
+                if isinstance(data, dict):
+                    raw_list = data.get("known_catalogs", [])
+                    if isinstance(raw_list, list):
+                        for p in raw_list:
+                            resolved = Path(p).resolve()
+                            if resolved.is_file() and resolved not in catalogs:
+                                catalogs.append(resolved)
+        except (json.JSONDecodeError, OSError):
+            pass
+
+    # If no existing file found yet, ensure default_cat is listed
+    if not catalogs:
+        catalogs.append(default_cat)
+
+    # Deduplicate while preserving order
+    result: list[Path] = []
+    for c in catalogs:
+        if c not in result:
+            result.append(c)
+    return result
+
+
+def add_known_catalog(path: str | Path) -> Path:
+    """Register a catalog path in the known catalogs configuration list."""
+    resolved_path = Path(path).resolve()
+    CONFIG_DIR.mkdir(parents=True, exist_ok=True)
+
+    data: dict[str, Any] = {}
+    if CONFIG_FILE.is_file():
+        try:
+            with open(CONFIG_FILE, "r", encoding="utf-8") as f:
+                loaded = json.load(f)
+                if isinstance(loaded, dict):
+                    data = loaded
+        except (json.JSONDecodeError, OSError):
+            data = {}
+
+    known = data.get("known_catalogs", [])
+    if not isinstance(known, list):
+        known = []
+
+    path_str = str(resolved_path)
+    if path_str not in known:
+        known.append(path_str)
+
+    data["known_catalogs"] = known
+    with open(CONFIG_FILE, "w", encoding="utf-8") as f:
+        json.dump(data, f, indent=2)
+
+    return resolved_path
+
+
+def remove_known_catalog(path: str | Path) -> bool:
+    """Remove a catalog from the known catalogs configuration list."""
+    resolved_path = Path(path).resolve()
+    if not CONFIG_FILE.is_file():
+        return False
+
+    try:
+        with open(CONFIG_FILE, "r", encoding="utf-8") as f:
+            data = json.load(f)
+        if not isinstance(data, dict):
+            return False
+
+        known = data.get("known_catalogs", [])
+        if isinstance(known, list) and str(resolved_path) in known:
+            known.remove(str(resolved_path))
+            data["known_catalogs"] = known
+            with open(CONFIG_FILE, "w", encoding="utf-8") as f:
+                json.dump(data, f, indent=2)
+            return True
+    except (json.JSONDecodeError, OSError):
+        pass
+    return False

det/catalog/models.py

@@ -0,0 +1,50 @@
+"""det.catalog.models - Pydantic data models for Catalog items, schemas, and files."""
+
+from __future__ import annotations
+
+from typing import Any
+
+from pydantic import BaseModel, Field
+
+
+class ColumnMetadata(BaseModel):
+    """Metadata for a single dataset column."""
+
+    name: str = Field(..., description="Column identifier / name")
+    type: str = Field(..., description="Logical or native data type (e.g. string, int64, float64)")
+    nullable: bool = Field(True, description="Whether the column accepts null values")
+    description: str = Field("", description="Human-readable column description")
+
+
+class DatasetMetadata(BaseModel):
+    """Metadata describing a table or file registered in the catalog."""
+
+    id: str = Field(..., description="Unique dataset identifier (e.g. namespace.table_name)")
+    name: str = Field(..., description="Dataset or table name")
+    namespace: str = Field("default", description="Namespace / schema grouping")
+    format: str = Field("parquet", description="Data format (parquet, csv, json, excel, directory)")
+    location: str = Field(..., description="Absolute or relative file / folder path")
+    description: str = Field("", description="Detailed description of the dataset")
+    row_count: int | None = Field(None, description="Exact or estimated total row count")
+    columns: list[ColumnMetadata] = Field(default_factory=list, description="Dataset column definitions")
+    tags: list[str] = Field(default_factory=list, description="Categorization tags")
+    last_scanned: str | None = Field(None, description="ISO timestamp of when schema was last scanned")
+
+
+class CatalogFile(BaseModel):
+    """The root representation serialized to catalog.yaml."""
+
+    catalog_version: int = Field(1, description="Catalog specification version")
+    catalog_name: str = Field("det_catalog", description="Logical name of this catalog")
+    namespaces: list[str] = Field(default_factory=lambda: ["default"], description="Configured namespaces / categories")
+    datasets: list[DatasetMetadata] = Field(default_factory=list, description="List of registered datasets")
+
+
+class ScanResult(BaseModel):
+    """Result of auto-scanning a file or folder location."""
+
+    location: str
+    format: str
+    columns: list[ColumnMetadata]
+    row_count: int | None = None
+    sample_preview: list[dict[str, Any]] | None = None

det/catalog/scanner.py

@@ -0,0 +1,274 @@
+"""det.catalog.scanner - Schema auto-scanner for files and directory datasets."""
+
+from __future__ import annotations
+
+from pathlib import Path
+
+# Python 3.15 lazy imports: modules are deferred until attribute access
+lazy import polars as pl
+lazy import pyarrow.dataset as ds
+lazy import pyarrow.parquet as pq
+
+from det.catalog.models import ColumnMetadata, ScanResult
+
+
+def _normalize_polars_type(pl_type: pl.DataType) -> str:
+    """Normalize a Polars DataType to a standard logical type string.
+
+    Args:
+        pl_type: The raw Polars datatype.
+
+    Returns:
+        str: Clean logical type string (e.g. 'int64', 'string', 'datetime').
+    """
+    name = str(pl_type).lower()
+    if "string" in name or "utf8" in name:
+        return "string"
+    if "int64" in name:
+        return "int64"
+    if "int32" in name:
+        return "int32"
+    if "int16" in name:
+        return "int16"
+    if "int8" in name:
+        return "int8"
+    if "uint" in name:
+        return "uint64"
+    if "float64" in name:
+        return "float64"
+    if "float32" in name:
+        return "float32"
+    if "bool" in name:
+        return "boolean"
+    if "datetime" in name:
+        return "datetime"
+    if "date" in name:
+        return "date"
+    if "decimal" in name:
+        return "decimal"
+    if "list" in name:
+        return "list"
+    if "struct" in name:
+        return "struct"
+    return str(pl_type)
+
+
+def scan_location(location: str | Path, format_hint: str | None = None) -> ScanResult:
+    """Scan a file or directory path and infer schema, format, and row count.
+
+    Args:
+        location: Path to a file or directory.
+        format_hint: Optional manual format indicator (e.g. 'parquet', 'csv').
+
+    Returns:
+        ScanResult: Object containing inferred columns, row count, and sample records.
+
+    Raises:
+        FileNotFoundError: If the specified location does not exist.
+        ValueError: If the location is not a file/dir or file format is unsupported.
+    """
+    path = Path(location).resolve()
+    if not path.exists():
+        raise FileNotFoundError(f"Location does not exist: {location}")
+
+    if path.is_file():
+        return _scan_file(path, format_hint)
+    elif path.is_dir():
+        return _scan_directory(path, format_hint)
+    else:
+        raise ValueError(f"Path is neither a regular file nor a directory: {location}")
+
+
+def _scan_file(path: Path, format_hint: str | None = None) -> ScanResult:
+    """Inspect and extract schema from a single file."""
+    ext = path.suffix.lower().lstrip(".")
+    detected_format = (format_hint or ext).lower()
+
+    if detected_format in ("parquet", "pq"):
+        try:
+            pq_file = pq.ParquetFile(str(path))
+            schema = pq_file.schema_arrow
+            row_count = pq_file.metadata.num_rows
+
+            columns = [
+                ColumnMetadata(
+                    name=field.name,
+                    type=str(field.type),
+                    nullable=field.nullable,
+                )
+                for field in schema
+            ]
+
+            # Sample preview of up to 5 rows
+            sample_df = pl.read_parquet(str(path), n_rows=5)
+            sample_data = sample_df.to_dicts()
+
+            return ScanResult(
+                location=str(path),
+                format="parquet",
+                columns=columns,
+                row_count=row_count,
+                sample_preview=sample_data,
+            )
+        except (OSError, RuntimeError):
+            # Fallback to Polars lazy scan
+            df = pl.scan_parquet(str(path)).collect().head(5)
+            columns = [
+                ColumnMetadata(name=col, type=_normalize_polars_type(df.schema[col]))
+                for col in df.columns
+            ]
+            return ScanResult(
+                location=str(path),
+                format="parquet",
+                columns=columns,
+                sample_preview=df.to_dicts(),
+            )
+
+    elif detected_format in ("csv", "tsv", "txt"):
+        separator = "\t" if detected_format == "tsv" or ext == "tsv" else ","
+        df = pl.read_csv(str(path), separator=separator, n_rows=100, ignore_errors=True)
+        columns = [
+            ColumnMetadata(name=col, type=_normalize_polars_type(df.schema[col]))
+            for col in df.columns
+        ]
+
+        # Calculate exact row count safely
+        try:
+            total_rows = (
+                pl.scan_csv(str(path), separator=separator)
+                .select(pl.len())
+                .collect()
+                .item()
+            )
+        except (pl.exceptions.PolarsError, OSError):
+            total_rows = len(df)
+
+        return ScanResult(
+            location=str(path),
+            format="csv" if separator == "," else "tsv",
+            columns=columns,
+            row_count=total_rows,
+            sample_preview=df.head(5).to_dicts(),
+        )
+
+    elif detected_format in ("json", "ndjson", "jsonl"):
+        try:
+            df = pl.read_ndjson(str(path), n_rows=100)
+            columns = [
+                ColumnMetadata(name=col, type=_normalize_polars_type(df.schema[col]))
+                for col in df.columns
+            ]
+            return ScanResult(
+                location=str(path),
+                format="ndjson",
+                columns=columns,
+                row_count=len(df),
+                sample_preview=df.head(5).to_dicts(),
+            )
+        except (pl.exceptions.PolarsError, ValueError):
+            # Fallback to standard JSON
+            df = pl.read_json(str(path))
+            columns = [
+                ColumnMetadata(name=col, type=_normalize_polars_type(df.schema[col]))
+                for col in df.columns
+            ]
+            return ScanResult(
+                location=str(path),
+                format="json",
+                columns=columns,
+                row_count=len(df),
+                sample_preview=df.head(5).to_dicts(),
+            )
+
+    elif detected_format in ("xlsx", "xls"):
+        df = pl.read_excel(str(path), read_options={"n_rows": 100})
+        columns = [
+            ColumnMetadata(name=col, type=_normalize_polars_type(df.schema[col]))
+            for col in df.columns
+        ]
+        return ScanResult(
+            location=str(path),
+            format="excel",
+            columns=columns,
+            row_count=len(df),
+            sample_preview=df.head(5).to_dicts(),
+        )
+
+    else:
+        raise ValueError(f"Unsupported file format: .{ext} for file '{path.name}'")
+
+
+def _scan_directory(dir_path: Path, format_hint: str | None = None) -> ScanResult:
+    """Inspect and extract schema from a directory dataset."""
+    parquet_files = list(dir_path.rglob("*.parquet")) + list(dir_path.rglob("*.pq"))
+    if parquet_files or format_hint == "parquet":
+        source = [str(f) for f in parquet_files] if parquet_files else str(dir_path)
+        dataset = ds.dataset(source, format="parquet")
+        schema = dataset.schema
+        columns = [
+            ColumnMetadata(
+                name=field.name,
+                type=str(field.type),
+                nullable=field.nullable,
+            )
+            for field in schema
+        ]
+        sample_data: list[dict] = []
+        try:
+            parquet_glob = f"{dir_path.as_posix()}/**/*.parquet"
+            sample_df = pl.scan_parquet(parquet_glob).collect().head(5)
+            sample_data = sample_df.to_dicts()
+        except (pl.exceptions.PolarsError, OSError, RuntimeError, ValueError):
+            if parquet_files:
+                sample_data = pl.read_parquet(str(parquet_files[0]), n_rows=5).to_dicts()
+
+        return ScanResult(
+            location=str(dir_path),
+            format="parquet (directory)",
+            columns=columns,
+            sample_preview=sample_data,
+        )
+
+    csv_files = list(dir_path.rglob("*.csv"))
+    if csv_files or format_hint == "csv":
+        sample_file = csv_files[0] if csv_files else None
+        if sample_file:
+            df = pl.read_csv(str(sample_file), n_rows=100)
+            columns = [
+                ColumnMetadata(name=col, type=_normalize_polars_type(df.schema[col]))
+                for col in df.columns
+            ]
+            return ScanResult(
+                location=str(dir_path),
+                format="csv (directory)",
+                columns=columns,
+                sample_preview=df.head(5).to_dicts(),
+            )
+
+    json_files = (
+        list(dir_path.rglob("*.ndjson"))
+        + list(dir_path.rglob("*.jsonl"))
+        + list(dir_path.rglob("*.json"))
+    )
+    if json_files or format_hint in ("json", "ndjson", "jsonl"):
+        sample_file = json_files[0] if json_files else None
+        if sample_file:
+            try:
+                df = pl.read_ndjson(str(sample_file), n_rows=100)
+                fmt = "ndjson (directory)"
+            except (pl.exceptions.PolarsError, OSError, ValueError):
+                df = pl.read_json(str(sample_file))
+                fmt = "json (directory)"
+
+            columns = [
+                ColumnMetadata(name=col, type=_normalize_polars_type(df.schema[col]))
+                for col in df.columns
+            ]
+            return ScanResult(
+                location=str(dir_path),
+                format=fmt,
+                columns=columns,
+                sample_preview=df.head(5).to_dicts(),
+            )
+
+    raise ValueError(f"Could not automatically detect dataset files inside directory: '{dir_path}'")