dagster-drt 0.1.0__tar.gz

dagster_drt-0.1.0/.gitignore

@@ -0,0 +1,44 @@
+# Python
+__pycache__/
+*.py[cod]
+*.pyo
+*.pyd
+*.so
+*.egg
+*.egg-info/
+dist/
+build/
+.eggs/
+.venv/
+venv/
+env/
+
+# uv
+.python-version
+uv.lock
+
+# drt runtime
+.drt/
+state.json
+
+# Testing
+.pytest_cache/
+.coverage
+htmlcov/
+.mypy_cache/
+.ruff_cache/
+
+# IDE
+.vscode/
+.idea/
+*.swp
+*.swo
+
+# OS
+.DS_Store
+Thumbs.db
+
+# Secrets
+*.env
+.env*
+!.env.example

dagster_drt-0.1.0/PKG-INFO

@@ -0,0 +1,139 @@
+Metadata-Version: 2.4
+Name: dagster-drt
+Version: 0.1.0
+Summary: Community-maintained Dagster integration for drt (data reverse tool)
+License: Apache-2.0
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Developers
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Requires-Python: >=3.10
+Requires-Dist: dagster>=1.6
+Requires-Dist: drt-core>=0.4.1
+Description-Content-Type: text/markdown
+
+# dagster-drt
+
+[![PyPI](https://img.shields.io/pypi/v/dagster-drt)](https://pypi.org/project/dagster-drt/)
+
+Community-maintained [Dagster](https://dagster.io/) integration for [drt](https://github.com/drt-hub/drt) (data reverse tool).
+
+Expose drt syncs as Dagster assets with full observability — metrics, dependencies, and dry-run support.
+
+## Installation
+
+```bash
+pip install dagster-drt
+```
+
+## Quick Start
+
+```python
+from dagster import Definitions
+from dagster_drt import drt_assets
+
+defs = Definitions(assets=drt_assets(project_dir="path/to/drt-project"))
+```
+
+## Features
+
+### DagsterDrtTranslator
+
+Customise how drt syncs map to Dagster assets. Subclass and override methods:
+
+```python
+from dagster import AssetKey
+from dagster_drt import DagsterDrtTranslator, drt_assets
+
+class MyTranslator(DagsterDrtTranslator):
+    def get_group_name(self, sync_config):
+        return "reverse_etl"
+
+    def get_deps(self, sync_config):
+        deps_map = {
+            "trigger_bq_meeting_gha": [AssetKey("bq_meeting_cloudsql")],
+        }
+        return deps_map.get(sync_config.name, [])
+
+assets = drt_assets(
+    project_dir="pipeline/data-reverse",
+    dagster_drt_translator=MyTranslator(),
+)
+```
+
+**Available methods:**
+
+| Method | Default | Purpose |
+|--------|---------|---------|
+| `get_asset_key(sync_config)` | `AssetKey(f"drt_{name}")` | Asset key |
+| `get_group_name(sync_config)` | `None` | Group name |
+| `get_description(sync_config)` | `sync_config.description` | Asset description |
+| `get_deps(sync_config)` | `[]` | Upstream dependencies |
+| `get_metadata(sync_config)` | `{}` | Static metadata |
+
+### Dry-Run Support (DrtConfig)
+
+Control dry-run mode per-run from the Dagster UI, or set a build-time default:
+
+```python
+# Build-time default: all syncs in dry-run mode
+assets = drt_assets(project_dir="...", dry_run=True)
+
+# Override per-run via Dagster UI → Run Config:
+# ops:
+#   drt_my_sync:
+#     config:
+#       dry_run: false
+```
+
+### MaterializeResult
+
+Assets return `MaterializeResult` with structured metadata visible in the Dagster UI:
+
+- `sync_name` — sync identifier
+- `rows_synced` — successful row count
+- `rows_failed` — failed row count
+- `rows_skipped` — skipped row count
+- `dry_run` — whether dry-run was active
+- `row_errors_count` — number of row-level errors (details in logs)
+
+### Filtering Syncs
+
+```python
+# Only expose specific syncs as assets
+assets = drt_assets(
+    project_dir="...",
+    sync_names=["sync_a", "sync_b"],
+)
+```
+
+## Usage with dagster-dbt
+
+```python
+from dagster import Definitions
+from dagster_dbt import dbt_assets
+from dagster_drt import drt_assets, DagsterDrtTranslator
+
+defs = Definitions(
+    assets=[*my_dbt_assets, *drt_assets("path/to/drt-project")],
+)
+```
+
+## API Reference
+
+### `drt_assets()`
+
+| Parameter | Type | Default | Description |
+|-----------|------|---------|-------------|
+| `project_dir` | `str \| Path` | required | Path to drt project root |
+| `sync_names` | `list[str] \| None` | `None` | Filter to specific syncs |
+| `dagster_drt_translator` | `DagsterDrtTranslator \| None` | `None` | Custom translator |
+| `dry_run` | `bool` | `False` | Default dry-run mode |
+
+Returns: `list[AssetsDefinition]`
+
+## License
+
+Apache-2.0

dagster_drt-0.1.0/README.md

@@ -0,0 +1,123 @@
+# dagster-drt
+
+[![PyPI](https://img.shields.io/pypi/v/dagster-drt)](https://pypi.org/project/dagster-drt/)
+
+Community-maintained [Dagster](https://dagster.io/) integration for [drt](https://github.com/drt-hub/drt) (data reverse tool).
+
+Expose drt syncs as Dagster assets with full observability — metrics, dependencies, and dry-run support.
+
+## Installation
+
+```bash
+pip install dagster-drt
+```
+
+## Quick Start
+
+```python
+from dagster import Definitions
+from dagster_drt import drt_assets
+
+defs = Definitions(assets=drt_assets(project_dir="path/to/drt-project"))
+```
+
+## Features
+
+### DagsterDrtTranslator
+
+Customise how drt syncs map to Dagster assets. Subclass and override methods:
+
+```python
+from dagster import AssetKey
+from dagster_drt import DagsterDrtTranslator, drt_assets
+
+class MyTranslator(DagsterDrtTranslator):
+    def get_group_name(self, sync_config):
+        return "reverse_etl"
+
+    def get_deps(self, sync_config):
+        deps_map = {
+            "trigger_bq_meeting_gha": [AssetKey("bq_meeting_cloudsql")],
+        }
+        return deps_map.get(sync_config.name, [])
+
+assets = drt_assets(
+    project_dir="pipeline/data-reverse",
+    dagster_drt_translator=MyTranslator(),
+)
+```
+
+**Available methods:**
+
+| Method | Default | Purpose |
+|--------|---------|---------|
+| `get_asset_key(sync_config)` | `AssetKey(f"drt_{name}")` | Asset key |
+| `get_group_name(sync_config)` | `None` | Group name |
+| `get_description(sync_config)` | `sync_config.description` | Asset description |
+| `get_deps(sync_config)` | `[]` | Upstream dependencies |
+| `get_metadata(sync_config)` | `{}` | Static metadata |
+
+### Dry-Run Support (DrtConfig)
+
+Control dry-run mode per-run from the Dagster UI, or set a build-time default:
+
+```python
+# Build-time default: all syncs in dry-run mode
+assets = drt_assets(project_dir="...", dry_run=True)
+
+# Override per-run via Dagster UI → Run Config:
+# ops:
+#   drt_my_sync:
+#     config:
+#       dry_run: false
+```
+
+### MaterializeResult
+
+Assets return `MaterializeResult` with structured metadata visible in the Dagster UI:
+
+- `sync_name` — sync identifier
+- `rows_synced` — successful row count
+- `rows_failed` — failed row count
+- `rows_skipped` — skipped row count
+- `dry_run` — whether dry-run was active
+- `row_errors_count` — number of row-level errors (details in logs)
+
+### Filtering Syncs
+
+```python
+# Only expose specific syncs as assets
+assets = drt_assets(
+    project_dir="...",
+    sync_names=["sync_a", "sync_b"],
+)
+```
+
+## Usage with dagster-dbt
+
+```python
+from dagster import Definitions
+from dagster_dbt import dbt_assets
+from dagster_drt import drt_assets, DagsterDrtTranslator
+
+defs = Definitions(
+    assets=[*my_dbt_assets, *drt_assets("path/to/drt-project")],
+)
+```
+
+## API Reference
+
+### `drt_assets()`
+
+| Parameter | Type | Default | Description |
+|-----------|------|---------|-------------|
+| `project_dir` | `str \| Path` | required | Path to drt project root |
+| `sync_names` | `list[str] \| None` | `None` | Filter to specific syncs |
+| `dagster_drt_translator` | `DagsterDrtTranslator \| None` | `None` | Custom translator |
+| `dry_run` | `bool` | `False` | Default dry-run mode |
+
+Returns: `list[AssetsDefinition]`
+
+## License
+
+Apache-2.0

dagster_drt-0.1.0/dagster_drt/__init__.py

@@ -0,0 +1,4 @@
+from dagster_drt.assets import DrtConfig, drt_assets
+from dagster_drt.translator import DagsterDrtTranslator
+
+__all__ = ["DagsterDrtTranslator", "DrtConfig", "drt_assets"]

dagster_drt-0.1.0/dagster_drt/assets.py

@@ -0,0 +1,134 @@
+"""Expose drt syncs as Dagster assets."""
+
+from pathlib import Path
+from typing import Any, Union
+
+from dagster import (
+    AssetExecutionContext,
+    AssetsDefinition,
+    Config,
+    MaterializeResult,
+    MetadataValue,
+    asset,
+)
+
+from dagster_drt.translator import DagsterDrtTranslator
+
+
+class DrtConfig(Config):
+    """Run configuration for drt assets — configurable from Dagster UI.
+
+    Follows the same pattern as dagster-dbt's ``--full-refresh`` flag.
+    """
+
+    dry_run: bool = False
+
+
+def drt_assets(
+    project_dir: Union[str, Path],
+    sync_names: Union[list[str], None] = None,
+    dagster_drt_translator: DagsterDrtTranslator | None = None,
+    dry_run: bool = False,
+) -> list[AssetsDefinition]:
+    """Create Dagster assets from drt sync definitions.
+
+    Args:
+        project_dir: Path to drt project root.
+        sync_names: Optional filter. If None, discovers all syncs.
+        dagster_drt_translator: Translator for customising asset keys, groups,
+            deps, and metadata. If None, uses the default translator.
+        dry_run: Default dry-run mode. Can be overridden per-run via
+            ``DrtConfig`` in the Dagster UI.
+
+    Returns:
+        List of Dagster asset definitions.
+    """
+    from drt.config.parser import load_syncs
+
+    translator = dagster_drt_translator or DagsterDrtTranslator()
+    project_path = Path(project_dir)
+    syncs = load_syncs(project_path)
+    if sync_names:
+        syncs = [s for s in syncs if s.name in sync_names]
+
+    assets_list: list[AssetsDefinition] = []
+    for sync_config in syncs:
+        asset_key = translator.get_asset_key(sync_config)
+        group_name = translator.get_group_name(sync_config)
+        description = translator.get_description(sync_config)
+        deps = translator.get_deps(sync_config)
+        static_metadata = translator.get_metadata(sync_config)
+
+        asset_kwargs: dict[str, Any] = {
+            "key": asset_key,
+            "description": description,
+        }
+        if group_name is not None:
+            asset_kwargs["group_name"] = group_name
+        if deps:
+            asset_kwargs["deps"] = deps
+        if static_metadata:
+            asset_kwargs["metadata"] = static_metadata
+
+        def _make_asset_fn(
+            _sync_cfg: Any, _default_dry_run: bool
+        ) -> AssetsDefinition:
+            @asset(**asset_kwargs)
+            def _asset_fn(
+                context: AssetExecutionContext,
+                config: DrtConfig,
+            ) -> MaterializeResult:
+                from drt.cli.main import _get_destination, _get_source
+                from drt.config.credentials import load_profile
+                from drt.config.parser import load_project
+                from drt.engine.sync import run_sync
+                from drt.state.manager import StateManager
+
+                effective_dry_run = config.dry_run or _default_dry_run
+
+                project = load_project(project_path)
+                profile = load_profile(project.profile)
+                source = _get_source(profile)
+                destination = _get_destination(_sync_cfg)
+                state_mgr = StateManager(project_path)
+
+                result = run_sync(
+                    _sync_cfg,
+                    source,
+                    destination,
+                    profile,
+                    project_path,
+                    dry_run=effective_dry_run,
+                    state_manager=state_mgr,
+                )
+
+                context.log.info(
+                    f"drt sync '{_sync_cfg.name}': "
+                    f"{result.success} synced, {result.failed} failed, "
+                    f"{result.skipped} skipped (dry_run={effective_dry_run})"
+                )
+
+                # Log row-level error details for debugging
+                for row_error in result.row_errors:
+                    context.log.warning(
+                        f"Row error in '{_sync_cfg.name}': {row_error}"
+                    )
+
+                return MaterializeResult(
+                    metadata={
+                        "sync_name": MetadataValue.text(_sync_cfg.name),
+                        "rows_synced": MetadataValue.int(result.success),
+                        "rows_failed": MetadataValue.int(result.failed),
+                        "rows_skipped": MetadataValue.int(result.skipped),
+                        "dry_run": MetadataValue.bool(effective_dry_run),
+                        "row_errors_count": MetadataValue.int(
+                            len(result.row_errors)
+                        ),
+                    }
+                )
+
+            return _asset_fn
+
+        assets_list.append(_make_asset_fn(sync_config, dry_run))
+
+    return assets_list

dagster_drt-0.1.0/dagster_drt/translator.py

@@ -0,0 +1,47 @@
+"""DagsterDrtTranslator — controls how drt syncs map to Dagster assets.
+
+Follows the same Translator pattern as dagster-dbt and dagster-dlt.
+Subclass and override methods to customise asset keys, groups, deps, etc.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Mapping, Sequence
+from typing import Any
+
+from dagster import AssetKey
+
+from drt.config.models import SyncConfig
+
+
+class DagsterDrtTranslator:
+    """Default translator from drt SyncConfig to Dagster asset properties.
+
+    Override any method in a subclass to customise behavior::
+
+        class MyTranslator(DagsterDrtTranslator):
+            def get_group_name(self, sync_config):
+                return "reverse_etl"
+
+        drt_assets(project_dir="...", dagster_drt_translator=MyTranslator())
+    """
+
+    def get_asset_key(self, sync_config: SyncConfig) -> AssetKey:
+        """Return the Dagster AssetKey for a sync."""
+        return AssetKey(f"drt_{sync_config.name}")
+
+    def get_group_name(self, sync_config: SyncConfig) -> str | None:
+        """Return the group name for a sync asset, or None for default."""
+        return None
+
+    def get_description(self, sync_config: SyncConfig) -> str | None:
+        """Return the asset description."""
+        return sync_config.description
+
+    def get_deps(self, sync_config: SyncConfig) -> Sequence[AssetKey]:
+        """Return upstream asset dependencies for a sync."""
+        return []
+
+    def get_metadata(self, sync_config: SyncConfig) -> Mapping[str, Any]:
+        """Return static metadata to attach to the asset definition."""
+        return {}

dagster_drt-0.1.0/pyproject.toml

@@ -0,0 +1,23 @@
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[project]
+name = "dagster-drt"
+version = "0.1.0"
+description = "Community-maintained Dagster integration for drt (data reverse tool)"
+readme = "README.md"
+license = {text = "Apache-2.0"}
+requires-python = ">=3.10"
+dependencies = [
+    "dagster>=1.6",
+    "drt-core>=0.4.1",
+]
+classifiers = [
+    "Development Status :: 3 - Alpha",
+    "Intended Audience :: Developers",
+    "Programming Language :: Python :: 3",
+    "Programming Language :: Python :: 3.10",
+    "Programming Language :: Python :: 3.11",
+    "Programming Language :: Python :: 3.12",
+]

dagster_drt-0.1.0/tests/test_assets.py

@@ -0,0 +1,160 @@
+from pathlib import Path
+
+from dagster import AssetKey
+
+SYNC_YAML = "name: test_sync\nmodel: ref('users')\ndestination:\n  type: rest_api\n  url: http://example.com\n"
+PROJECT_YAML = "name: test\nversion: '0.1'\nprofile: local\n"
+
+
+def _setup_project(tmp_path: Path, syncs: dict[str, str] | None = None) -> Path:
+    """Create a minimal drt project structure for testing."""
+    (tmp_path / "drt_project.yml").write_text(PROJECT_YAML)
+    syncs_dir = tmp_path / "syncs"
+    syncs_dir.mkdir()
+    if syncs is None:
+        syncs = {"test_sync.yml": SYNC_YAML}
+    for filename, content in syncs.items():
+        (syncs_dir / filename).write_text(content)
+    return tmp_path
+
+
+# --- Basic asset creation ---
+
+
+def test_drt_assets_creates_asset_list(tmp_path: Path) -> None:
+    project = _setup_project(tmp_path)
+    from dagster_drt.assets import drt_assets
+
+    assets = drt_assets(project_dir=project)
+    assert len(assets) == 1
+
+
+def test_drt_assets_filter_by_name(tmp_path: Path) -> None:
+    project = _setup_project(
+        tmp_path,
+        {
+            "a.yml": "name: a\nmodel: ref('t')\ndestination:\n  type: rest_api\n  url: http://x.com\n",
+            "b.yml": "name: b\nmodel: ref('t')\ndestination:\n  type: rest_api\n  url: http://x.com\n",
+        },
+    )
+    from dagster_drt.assets import drt_assets
+
+    assets = drt_assets(project_dir=project, sync_names=["a"])
+    assert len(assets) == 1
+
+
+# --- Translator: default ---
+
+
+def test_default_translator_asset_key(tmp_path: Path) -> None:
+    project = _setup_project(tmp_path)
+    from dagster_drt.assets import drt_assets
+
+    assets = drt_assets(project_dir=project)
+    assert assets[0].key == AssetKey("drt_test_sync")
+
+
+def test_default_translator_no_group(tmp_path: Path) -> None:
+    project = _setup_project(tmp_path)
+    from dagster_drt.assets import drt_assets
+
+    assets = drt_assets(project_dir=project)
+    assert assets[0].group_names_by_key.get(AssetKey("drt_test_sync")) == "default"
+
+
+# --- Translator: custom ---
+
+
+def test_custom_translator_group_name(tmp_path: Path) -> None:
+    project = _setup_project(tmp_path)
+    from dagster_drt.assets import drt_assets
+    from dagster_drt.translator import DagsterDrtTranslator
+
+    class MyTranslator(DagsterDrtTranslator):
+        def get_group_name(self, sync_config):
+            return "reverse_etl"
+
+    assets = drt_assets(project_dir=project, dagster_drt_translator=MyTranslator())
+    assert assets[0].group_names_by_key[AssetKey("drt_test_sync")] == "reverse_etl"
+
+
+def test_custom_translator_asset_key(tmp_path: Path) -> None:
+    project = _setup_project(tmp_path)
+    from dagster_drt.assets import drt_assets
+    from dagster_drt.translator import DagsterDrtTranslator
+
+    class MyTranslator(DagsterDrtTranslator):
+        def get_asset_key(self, sync_config):
+            return AssetKey(["my_prefix", sync_config.name])
+
+    assets = drt_assets(project_dir=project, dagster_drt_translator=MyTranslator())
+    assert assets[0].key == AssetKey(["my_prefix", "test_sync"])
+
+
+def test_custom_translator_deps(tmp_path: Path) -> None:
+    project = _setup_project(tmp_path)
+    from dagster_drt.assets import drt_assets
+    from dagster_drt.translator import DagsterDrtTranslator
+
+    class MyTranslator(DagsterDrtTranslator):
+        def get_deps(self, sync_config):
+            return [AssetKey("upstream_model")]
+
+    assets = drt_assets(project_dir=project, dagster_drt_translator=MyTranslator())
+    dep_keys = {dep.asset_key for dep in assets[0].specs_by_key[AssetKey("drt_test_sync")].deps}
+    assert AssetKey("upstream_model") in dep_keys
+
+
+# --- DrtConfig ---
+
+
+def test_drt_assets_accepts_dry_run_default(tmp_path: Path) -> None:
+    """dry_run parameter should not break asset creation."""
+    project = _setup_project(tmp_path)
+    from dagster_drt.assets import drt_assets
+
+    assets = drt_assets(project_dir=project, dry_run=True)
+    assert len(assets) == 1
+
+
+def test_drt_config_defaults() -> None:
+    from dagster_drt.assets import DrtConfig
+
+    config = DrtConfig()
+    assert config.dry_run is False
+
+
+def test_drt_config_dry_run() -> None:
+    from dagster_drt.assets import DrtConfig
+
+    config = DrtConfig(dry_run=True)
+    assert config.dry_run is True
+
+
+# --- Return type ---
+
+
+def test_asset_returns_materialize_result(tmp_path: Path) -> None:
+    """Verify asset function return annotation is MaterializeResult."""
+    project = _setup_project(tmp_path)
+    from dagster import MaterializeResult
+
+    from dagster_drt.assets import drt_assets
+
+    assets = drt_assets(project_dir=project)
+    # Check the output type from the asset spec
+    spec = assets[0].specs_by_key[AssetKey("drt_test_sync")]
+    # MaterializeResult assets have no output type annotation on the spec
+    # but we can verify the asset was created successfully with the new signature
+    assert spec is not None
+
+
+# --- Exports ---
+
+
+def test_public_exports() -> None:
+    import dagster_drt
+
+    assert hasattr(dagster_drt, "drt_assets")
+    assert hasattr(dagster_drt, "DrtConfig")
+    assert hasattr(dagster_drt, "DagsterDrtTranslator")