vibedata-dlt-core 0.1.0__tar.gz

vibedata_dlt_core-0.1.0/.gitignore

@@ -0,0 +1,8 @@
+__pycache__/
+*.py[cod]
+*.egg-info/
+dist/
+build/
+.venv/
+.env
+.pytest_cache/

vibedata_dlt_core-0.1.0/PKG-INFO

@@ -0,0 +1,25 @@
+Metadata-Version: 2.4
+Name: vibedata-dlt-core
+Version: 0.1.0
+Summary: VibeData dlt runtime — cross-platform shared core (audit + pipeline context). Depended on by every platform's runtime; never installed to run a pipeline on its own.
+Author-email: Vibedata <eng@acceleratedata.ai>
+License: MIT
+Requires-Python: >=3.11
+Requires-Dist: dlt>=1.0.0
+Provides-Extra: dev
+Requires-Dist: pytest-mock>=3.12; extra == 'dev'
+Requires-Dist: pytest>=8.0; extra == 'dev'
+Description-Content-Type: text/markdown
+
+# vibedata-dlt-core
+
+Cross-platform shared core for the VibeData dlt runtime. Provides the
+platform-agnostic building blocks every execution unit depends on:
+
+- **audit** — extract dlt trace metrics into the `audit_runs` / `audit_table_loads` delta tables.
+- **pipeline_context** — resolve the run's pipeline/connection context from the project's dlt config.
+
+Import namespace: `vibedata.dlt.core`. This is a dependency package — it is not
+installed to run a pipeline on its own. Platform cores
+(`vibedata-dlt-fabric-core`, `vibedata-dlt-duckdb-core`) and the execution-unit
+distributions build on it.

vibedata_dlt_core-0.1.0/README.md

@@ -0,0 +1,12 @@
+# vibedata-dlt-core
+
+Cross-platform shared core for the VibeData dlt runtime. Provides the
+platform-agnostic building blocks every execution unit depends on:
+
+- **audit** — extract dlt trace metrics into the `audit_runs` / `audit_table_loads` delta tables.
+- **pipeline_context** — resolve the run's pipeline/connection context from the project's dlt config.
+
+Import namespace: `vibedata.dlt.core`. This is a dependency package — it is not
+installed to run a pipeline on its own. Platform cores
+(`vibedata-dlt-fabric-core`, `vibedata-dlt-duckdb-core`) and the execution-unit
+distributions build on it.

vibedata_dlt_core-0.1.0/pyproject.toml

@@ -0,0 +1,29 @@
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[project]
+name = "vibedata-dlt-core"
+version = "0.1.0"
+description = "VibeData dlt runtime — cross-platform shared core (audit + pipeline context). Depended on by every platform's runtime; never installed to run a pipeline on its own."
+readme = "README.md"
+requires-python = ">=3.11"
+license = { text = "MIT" }
+authors = [{ name = "Vibedata", email = "eng@acceleratedata.ai" }]
+
+dependencies = [
+    "dlt>=1.0.0",
+]
+
+[project.optional-dependencies]
+dev = ["pytest>=8.0", "pytest-mock>=3.12"]
+
+# PEP 420 namespace package: `vibedata` and `vibedata.dlt` carry no __init__.py
+# so other distributions (vibedata-dlt-fabric-core, vibedata-dlt-duckdb-core, …)
+# can contribute their own subpackages to the same namespace.
+[tool.hatch.build.targets.wheel]
+packages = ["src/vibedata"]
+
+[tool.pytest.ini_options]
+testpaths = ["tests"]
+pythonpath = ["src"]

vibedata_dlt_core-0.1.0/src/vibedata/dlt/core/__init__.py

@@ -0,0 +1,25 @@
+"""VibeData dlt runtime — cross-platform shared core.
+
+Platform-agnostic building blocks shared by every execution unit:
+
+- ``audit``: extract dlt trace metrics into the ``audit_runs`` /
+  ``audit_table_loads`` delta tables.
+- ``pipeline_context``: resolve the run's pipeline/connection context from
+  the project's dlt config.
+
+Platform packages (``vibedata.dlt.fabric_core``, ``vibedata.dlt.duckdb_core``)
+and the execution-unit facades (``vibedata.dlt.fabric``, ``vibedata.dlt.duckdb``)
+build on these. This package is a dependency, not a runnable distribution.
+"""
+
+from .audit import build_audit_data, persist_audit
+from .pipeline_context import PipelineContext, load_pipeline_context
+from .tuning import register_pipeline_tuning
+
+__all__ = [
+    "build_audit_data",
+    "persist_audit",
+    "PipelineContext",
+    "load_pipeline_context",
+    "register_pipeline_tuning",
+]

vibedata_dlt_core-0.1.0/src/vibedata/dlt/core/audit.py

@@ -0,0 +1,231 @@
+"""
+Audit — Pipeline run tracking via delta tables.
+
+Extracts runtime metrics from dlt's in-memory trace object and persists
+them into two append-only delta tables:
+
+- audit_runs: one row per pipeline run (timing, status, row counts)
+- audit_table_loads: one row per table per run (per-table metrics)
+
+Both tables share `load_id` as a foreign key.
+"""
+
+from __future__ import annotations
+
+import logging
+import uuid
+from typing import Any, Optional
+
+logger = logging.getLogger(__name__)
+
+
+def build_audit_data(
+    trace: Any,
+    error_message: Optional[str] = None,
+) -> tuple[dict[str, Any], list[dict[str, Any]]]:
+    """Extract audit_runs row and audit_table_loads rows from a pipeline trace.
+
+    Handles partial traces gracefully — when pipeline.run() throws during
+    extract/normalize/load, the trace step's step_info is the Exception
+    object (no .metrics attribute). Those steps are skipped safely.
+
+    Args:
+        trace: The captured pipeline trace (pipeline.last_trace).
+        error_message: Optional error message from a caught exception.
+
+    Returns:
+        (audit_run dict, list of audit_table_loads dicts)
+    """
+    extract_metrics: dict[str, dict] = {}
+    normalize_metrics: dict[str, dict] = {}
+    load_job_metrics: dict[str, dict] = {}
+    step_timing: dict[str, dict] = {}
+    load_id = None
+    pipeline_name = None
+    destination_name = None
+    dataset_name = None
+    first_run = None
+
+    for step in trace.steps:
+        si = step.step_info
+        step_name = step.step
+
+        if step_name == "run":
+            continue
+
+        step_timing[step_name] = {
+            "started_at": step.started_at.isoformat() if step.started_at else None,
+            "finished_at": step.finished_at.isoformat() if step.finished_at else None,
+        }
+
+        if not hasattr(si, "metrics"):
+            continue
+
+        for lid, metric_list in si.metrics.items():
+            if load_id is None:
+                load_id = lid
+
+            for m in metric_list:
+                if step_name == "extract" and "table_metrics" in m:
+                    for tname, tm in m["table_metrics"].items():
+                        if tname.startswith("_dlt_"):
+                            continue
+                        extract_metrics[tname] = {
+                            "items_count": tm.items_count,
+                            "file_size": tm.file_size,
+                        }
+
+                elif step_name == "normalize" and "table_metrics" in m:
+                    for tname, tm in m["table_metrics"].items():
+                        if tname.startswith("_dlt_"):
+                            continue
+                        normalize_metrics[tname] = {
+                            "items_count": tm.items_count,
+                            "file_size": tm.file_size,
+                        }
+
+                elif step_name == "load" and "job_metrics" in m:
+                    for jid, jm in m["job_metrics"].items():
+                        tname = jm.table_name
+                        if tname.startswith("_dlt_"):
+                            continue
+                        if tname not in load_job_metrics:
+                            load_job_metrics[tname] = {
+                                "state": jm.state,
+                                "elapsed": 0.0,
+                                "failed_message": None,
+                            }
+                        load_job_metrics[tname]["elapsed"] += (
+                            (jm.finished_at - jm.started_at).total_seconds()
+                            if jm.started_at and jm.finished_at
+                            else 0.0
+                        )
+                        if jm.state != "completed":
+                            load_job_metrics[tname]["state"] = jm.state
+                        if hasattr(jm, "failed_message") and jm.failed_message:
+                            load_job_metrics[tname]["failed_message"] = str(
+                                jm.failed_message
+                            )
+
+        if step_name == "extract" and pipeline_name is None:
+            try:
+                pipeline_name = si.pipeline.pipeline_name
+                first_run = si.first_run
+            except AttributeError:
+                pass
+        if step_name == "load":
+            try:
+                destination_name = si.destination_name
+                dataset_name = si.dataset_name
+            except AttributeError:
+                pass
+
+    if load_id is None:
+        load_id = f"failed_{uuid.uuid4().hex[:16]}"
+
+    failed_tables = [
+        t for t, m in load_job_metrics.items() if m["state"] != "completed"
+    ]
+    if error_message:
+        status = "failed"
+    else:
+        status = "failed" if failed_tables else "completed"
+
+    table_error_messages = [
+        load_job_metrics[t]["failed_message"]
+        for t in failed_tables
+        if load_job_metrics[t]["failed_message"]
+    ]
+    all_error_parts = []
+    if error_message:
+        all_error_parts.append(error_message)
+    all_error_parts.extend(table_error_messages)
+
+    total_rows = sum(m["items_count"] for m in extract_metrics.values())
+
+    audit_run = {
+        "load_id": load_id,
+        "pipeline_name": pipeline_name,
+        "destination_name": destination_name,
+        "dataset_name": dataset_name,
+        "first_run": first_run,
+        "status": status,
+        "started_at": trace.started_at.isoformat() if trace.started_at else None,
+        "finished_at": trace.finished_at.isoformat() if trace.finished_at else None,
+        "extract_started_at": step_timing.get("extract", {}).get("started_at"),
+        "extract_finished_at": step_timing.get("extract", {}).get("finished_at"),
+        "normalize_started_at": step_timing.get("normalize", {}).get("started_at"),
+        "normalize_finished_at": step_timing.get("normalize", {}).get("finished_at"),
+        "load_started_at": step_timing.get("load", {}).get("started_at"),
+        "load_finished_at": step_timing.get("load", {}).get("finished_at"),
+        "total_rows_extracted": total_rows,
+        "error_message": "; ".join(all_error_parts) if all_error_parts else None,
+    }
+
+    all_tables = set(extract_metrics) | set(normalize_metrics) | set(load_job_metrics)
+
+    audit_table_loads = []
+    for tname in sorted(all_tables):
+        row = {
+            "load_id": load_id,
+            "table_name": tname,
+            "rows_extracted": extract_metrics.get(tname, {}).get("items_count"),
+            "rows_normalized": normalize_metrics.get(tname, {}).get("items_count"),
+            "file_size": normalize_metrics.get(tname, {}).get("file_size"),
+            "load_state": load_job_metrics.get(tname, {}).get("state"),
+            "load_elapsed": load_job_metrics.get(tname, {}).get("elapsed"),
+            "failed_message": load_job_metrics.get(tname, {}).get("failed_message"),
+        }
+        audit_table_loads.append(row)
+
+    return audit_run, audit_table_loads
+
+
+def persist_audit(
+    pipeline: Any,
+    trace: Any,
+    table_format: Optional[str] = "delta",
+    error_message: Optional[str] = None,
+) -> dict[str, Any]:
+    """Build audit data from trace and persist to audit tables.
+
+    Args:
+        pipeline: The dlt pipeline instance.
+        trace: The captured pipeline trace (pipeline.last_trace).
+        table_format: Table format (default "delta").
+        error_message: Optional error message from a caught exception.
+
+    Returns:
+        Dict with audit summary (load_id, status, tables_tracked).
+    """
+    audit_run, audit_table_loads = build_audit_data(trace, error_message=error_message)
+
+    run_kwargs: dict[str, Any] = {
+        "table_name": "audit_runs",
+        "write_disposition": "append",
+    }
+    if table_format:
+        run_kwargs["table_format"] = table_format
+    pipeline.run([audit_run], **run_kwargs)
+
+    if audit_table_loads:
+        load_kwargs: dict[str, Any] = {
+            "table_name": "audit_table_loads",
+            "write_disposition": "append",
+        }
+        if table_format:
+            load_kwargs["table_format"] = table_format
+        pipeline.run(audit_table_loads, **load_kwargs)
+
+    logger.info(
+        "Audit persisted: load_id=%s status=%s tables=%d",
+        audit_run["load_id"],
+        audit_run["status"],
+        len(audit_table_loads),
+    )
+
+    return {
+        "load_id": audit_run["load_id"],
+        "status": audit_run["status"],
+        "tables_tracked": len(audit_table_loads),
+    }

vibedata_dlt_core-0.1.0/src/vibedata/dlt/core/pipeline_context.py

@@ -0,0 +1,67 @@
+"""
+Pipeline-folder context — the run contract for the two-registry layout:
+
+    ingestion/
+    ├── connections/<connection_name>/.dlt/config.toml   # dlt project dir at run time
+    └── pipelines/<pipeline_name>/
+        ├── pipeline.py          # pure logic; runs with cwd = this folder
+        └── config.toml          # [pipeline].connection + optional dlt tuning tables
+
+`load_pipeline_context()` reads ./config.toml, resolves the referenced
+connection folder, and returns the tuning tables (everything except the
+SDK-owned [pipeline] table, which must never resolve as dlt config).
+Returns None when there is no pipeline config — callers fall back to the
+legacy single-project behavior.
+"""
+
+from __future__ import annotations
+
+import os
+import tomllib
+from dataclasses import dataclass
+from typing import Any
+
+PIPELINE_CONFIG_FILE = 'config.toml'
+
+
+@dataclass
+class PipelineContext:
+    connection_name: str
+    """Absolute path to ingestion/connections/<name>/ — the DLT_PROJECT_DIR."""
+    connection_dir: str
+    """dlt tuning tables from the pipeline config ([pipeline] stripped)."""
+    tuning: dict[str, Any]
+
+
+def load_pipeline_context(cwd: str = '.') -> PipelineContext | None:
+    config_path = os.path.join(cwd, PIPELINE_CONFIG_FILE)
+    try:
+        with open(config_path, 'rb') as f:
+            cfg = tomllib.load(f)
+    except FileNotFoundError:
+        return None
+
+    pipeline_table = cfg.get('pipeline')
+    if not isinstance(pipeline_table, dict):
+        return None
+    connection_name = pipeline_table.get('connection')
+    if not isinstance(connection_name, str) or not connection_name.strip():
+        return None
+    connection_name = connection_name.strip()
+
+    # Fail fast on a dangling reference — referential integrity invariant.
+    connection_dir = os.path.abspath(
+        os.path.join(cwd, '..', '..', 'connections', connection_name)
+    )
+    if not os.path.isdir(connection_dir):
+        raise FileNotFoundError(
+            f"Pipeline references connection '{connection_name}' but "
+            f'{connection_dir} does not exist.'
+        )
+
+    tuning = {key: value for key, value in cfg.items() if key != 'pipeline'}
+    return PipelineContext(
+        connection_name=connection_name,
+        connection_dir=connection_dir,
+        tuning=tuning,
+    )

vibedata_dlt_core-0.1.0/src/vibedata/dlt/core/tuning.py

@@ -0,0 +1,24 @@
+"""Cross-platform pipeline-tuning provider.
+
+The pipeline's ``config.toml`` tuning tables (everything except the SDK-owned
+``[pipeline]`` table) register as the LOWEST-precedence dlt config provider, so
+``env > connection toml > pipeline toml``. Identical for every platform, so it
+lives in core rather than each platform's setup flow.
+"""
+
+from __future__ import annotations
+
+import logging
+from typing import Any
+
+logger = logging.getLogger(__name__)
+
+
+def register_pipeline_tuning(tuning: dict[str, Any]) -> None:
+    """Register the pipeline config.toml tuning tables as a dlt config provider."""
+    import dlt
+    from dlt.common.configuration.providers import CustomLoaderDocProvider
+
+    provider = CustomLoaderDocProvider("pipeline_config", lambda: tuning, supports_secrets=False)
+    dlt.config.register_provider(provider)
+    logger.info("Registered pipeline_config provider (%d tables)", len(tuning))

vibedata_dlt_core-0.1.0/tests/test_audit.py

@@ -0,0 +1,97 @@
+"""Tests for audit module."""
+
+from unittest.mock import MagicMock
+from datetime import datetime, timezone
+
+from vibedata.dlt.core.audit import build_audit_data
+
+
+def _make_mock_trace(*, has_error=False):
+    """Create a minimal mock dlt trace for testing."""
+    trace = MagicMock()
+    trace.started_at = datetime(2024, 1, 1, 12, 0, 0, tzinfo=timezone.utc)
+    trace.finished_at = datetime(2024, 1, 1, 12, 5, 0, tzinfo=timezone.utc)
+
+    # Extract step
+    extract_step = MagicMock()
+    extract_step.step = "extract"
+    extract_step.started_at = trace.started_at
+    extract_step.finished_at = datetime(2024, 1, 1, 12, 1, 0, tzinfo=timezone.utc)
+
+    extract_info = MagicMock()
+    extract_info.pipeline.pipeline_name = "test_pipeline"
+    extract_info.first_run = True
+
+    table_metric = MagicMock()
+    table_metric.items_count = 100
+    table_metric.file_size = 5000
+
+    extract_info.metrics = {
+        "load_1": [{"table_metrics": {"contacts": table_metric}}]
+    }
+    extract_step.step_info = extract_info
+
+    # Load step
+    load_step = MagicMock()
+    load_step.step = "load"
+    load_step.started_at = datetime(2024, 1, 1, 12, 3, 0, tzinfo=timezone.utc)
+    load_step.finished_at = datetime(2024, 1, 1, 12, 5, 0, tzinfo=timezone.utc)
+
+    load_info = MagicMock()
+    load_info.destination_name = "filesystem"
+    load_info.dataset_name = "salesforce"
+
+    job_metric = MagicMock()
+    job_metric.table_name = "contacts"
+    job_metric.state = "failed" if has_error else "completed"
+    job_metric.started_at = load_step.started_at
+    job_metric.finished_at = load_step.finished_at
+    job_metric.failed_message = "Load error" if has_error else None
+
+    load_info.metrics = {
+        "load_1": [{"job_metrics": {"job_1": job_metric}}]
+    }
+    load_step.step_info = load_info
+
+    trace.steps = [extract_step, load_step]
+    return trace
+
+
+class TestBuildAuditData:
+    def test_successful_run(self):
+        trace = _make_mock_trace()
+        audit_run, table_loads = build_audit_data(trace)
+
+        assert audit_run["status"] == "completed"
+        assert audit_run["pipeline_name"] == "test_pipeline"
+        assert audit_run["total_rows_extracted"] == 100
+        assert audit_run["error_message"] is None
+        assert len(table_loads) == 1
+        assert table_loads[0]["table_name"] == "contacts"
+        assert table_loads[0]["rows_extracted"] == 100
+
+    def test_failed_run(self):
+        trace = _make_mock_trace(has_error=True)
+        audit_run, table_loads = build_audit_data(trace)
+
+        assert audit_run["status"] == "failed"
+        assert "Load error" in audit_run["error_message"]
+
+    def test_forced_error_message(self):
+        trace = _make_mock_trace()
+        audit_run, _ = build_audit_data(trace, error_message="Pipeline crashed")
+
+        assert audit_run["status"] == "failed"
+        assert "Pipeline crashed" in audit_run["error_message"]
+
+    def test_empty_trace(self):
+        trace = MagicMock()
+        trace.started_at = None
+        trace.finished_at = None
+        trace.steps = []
+
+        audit_run, table_loads = build_audit_data(trace)
+
+        assert audit_run["load_id"].startswith("failed_")
+        assert audit_run["status"] == "completed"
+        assert table_loads == []