dagster-evidence 0.1.7__py3-none-any.whl → 0.2.0__py3-none-any.whl

dagster_evidence/components/translator.py

@@ -0,0 +1,168 @@
+"""Translator for Evidence project assets.
+
+This module provides the translator class for converting Evidence project data
+into Dagster AssetSpecs. Subclass DagsterEvidenceTranslator to customize
+how Evidence sources and projects are represented in Dagster.
+"""
+
+from typing import Union
+
+import dagster as dg
+from dagster._annotations import beta, public
+
+from .sources import (
+    BaseEvidenceProjectSource,
+    BigQueryEvidenceProjectSource,
+    DuckdbEvidenceProjectSource,
+    EvidenceProjectTranslatorData,
+    EvidenceSourceTranslatorData,
+    GSheetsEvidenceProjectSource,
+    MotherDuckEvidenceProjectSource,
+)
+
+
+@beta
+@public
+class DagsterEvidenceTranslator:
+    """Translator class which converts Evidence project data into AssetSpecs.
+
+    Subclass this class to provide custom translation logic.
+
+    Examples:
+        Extend the source type registry via class attribute:
+
+            class MyTranslator(DagsterEvidenceTranslator):
+                SOURCE_TYPE_REGISTRY = {
+                    **DagsterEvidenceTranslator.SOURCE_TYPE_REGISTRY,
+                    "postgres": PostgresEvidenceProjectSource,
+                }
+
+        Or override get_source_type_registry() for dynamic configuration:
+
+            class MyTranslator(DagsterEvidenceTranslator):
+                def get_source_type_registry(self):
+                    return {
+                        **super().get_source_type_registry(),
+                        "postgres": PostgresEvidenceProjectSource,
+                    }
+
+        Customize asset spec generation:
+
+            class MyTranslator(DagsterEvidenceTranslator):
+                def get_asset_spec(self, data):
+                    spec = super().get_asset_spec(data)
+                    if isinstance(data, EvidenceSourceTranslatorData):
+                        return spec.replace_attributes(
+                            key=spec.key.with_prefix("my_prefix"),
+                        )
+                    return spec
+    """
+
+    # Source type registry - maps source type string to source class
+    # Subclasses can extend via class attribute or override get_source_type_registry()
+    SOURCE_TYPE_REGISTRY: dict[str, type[BaseEvidenceProjectSource]] = {
+        "duckdb": DuckdbEvidenceProjectSource,
+        "motherduck": MotherDuckEvidenceProjectSource,
+        "bigquery": BigQueryEvidenceProjectSource,
+        "gsheets": GSheetsEvidenceProjectSource,
+    }
+
+    @public
+    def get_source_type_registry(
+        self,
+    ) -> dict[str, type[BaseEvidenceProjectSource]]:
+        """Get the source type registry mapping source types to source classes.
+
+        Override this method to dynamically configure the source type registry.
+        By default, returns the SOURCE_TYPE_REGISTRY class attribute.
+
+        Returns:
+            Dictionary mapping source type strings to source class types.
+
+        Example:
+
+            .. code-block:: python
+
+                class CustomTranslator(DagsterEvidenceTranslator):
+                    def get_source_type_registry(self):
+                        return {
+                            **super().get_source_type_registry(),
+                            "postgres": PostgresEvidenceProjectSource,
+                            "mysql": MySQLEvidenceProjectSource,
+                        }
+        """
+        return self.SOURCE_TYPE_REGISTRY
+
+    @public
+    def get_source_class(self, source_type: str) -> type[BaseEvidenceProjectSource]:
+        """Get the source class for a given source type.
+
+        Args:
+            source_type: The source type identifier (e.g., "duckdb", "bigquery").
+
+        Returns:
+            The source class for the given type.
+
+        Raises:
+            NotImplementedError: If the source type is not in the registry.
+        """
+        registry = self.get_source_type_registry()
+        if source_type not in registry:
+            raise NotImplementedError(f"Unknown source type: {source_type}")
+        return registry[source_type]
+
+    @public
+    def get_asset_spec(
+        self, data: Union[EvidenceSourceTranslatorData, EvidenceProjectTranslatorData]
+    ) -> Union[dg.AssetSpec, dg.AssetsDefinition]:
+        """Get the asset for an Evidence object (source query or project).
+
+        Override this method to customize asset generation.
+
+        Args:
+            data: Either EvidenceSourceTranslatorData for source queries
+                  or EvidenceProjectTranslatorData for the main project asset.
+
+        Returns:
+            For source queries: AssetsDefinition with automation condition.
+            For project: AssetSpec for the Evidence project.
+
+        Example:
+
+            .. code-block:: python
+
+                class CustomTranslator(DagsterEvidenceTranslator):
+                    def get_asset_spec(self, data):
+                        asset = super().get_asset_spec(data)
+                        # Customize as needed
+                        return asset
+        """
+        if isinstance(data, EvidenceSourceTranslatorData):
+            return self._get_source_asset(data)
+        elif isinstance(data, EvidenceProjectTranslatorData):
+            return self._get_project_asset_spec(data)
+        else:
+            raise TypeError(f"Unknown data type: {type(data)}")
+
+    def _get_source_asset(
+        self, data: EvidenceSourceTranslatorData
+    ) -> dg.AssetsDefinition:
+        """Default translation for source query assets.
+
+        Delegates to the source class's get_source_asset method, allowing
+        each source type to customize its asset generation.
+        """
+        source_type = data.source_content.connection.type
+        source_class = self.get_source_class(source_type)
+        return source_class.get_source_asset(data)
+
+    def _get_project_asset_spec(
+        self, data: EvidenceProjectTranslatorData
+    ) -> dg.AssetSpec:
+        """Default translation for main project asset."""
+        return dg.AssetSpec(
+            key=dg.AssetKey([data.project_name]),
+            kinds={"evidence"},
+            deps=data.source_deps,
+            group_name=data.effective_group_name,
+        )

dagster_evidence/lib/evidence_project.py

@@ -41,7 +41,7 @@ class EvidenceProject(Component, Resolvable):
     def build_defs(self, context: ComponentLoadContext) -> Definitions:
         project_path = os.path.abspath(context.path / self.project_path)
 
-        def _run_cmd(cmd: Sequence[str]):
+        def _run_cmd(cmd: Sequence[str]) -> None:
             print(
                 f"{project_path}$ {' '.join(cmd)}",
                 file=sys.stderr,
@@ -51,7 +51,7 @@ class EvidenceProject(Component, Resolvable):
                 cwd=project_path,
                 check=True,
                 capture_output=False,
-                env=os.environ,
+                env=dict(os.environ),
             )
 
         @multi_asset(specs=[self.asset])

dagster_evidence/resource.py

@@ -41,7 +41,7 @@ class EvidenceResource(dg.ConfigurableResource):
         description="The executable to use for commands (npm, yarn, etc.)",
     )
 
-    def _run_cmd(self, cmd: Sequence[str]):
+    def _run_cmd(self, cmd: Sequence[str]) -> None:
         """Run a command in the project directory.
 
         Args:
@@ -56,7 +56,7 @@ class EvidenceResource(dg.ConfigurableResource):
             cwd=self.project_path,
             check=True,
             capture_output=False,
-            env=os.environ,
+            env=dict(os.environ),
         )
 
     def build(self) -> None:

dagster_evidence/utils/__init__.py

@@ -0,0 +1,5 @@
+"""Utility functions for dagster-evidence."""
+
+from .sql_parser import extract_table_references
+
+__all__ = ["extract_table_references"]

dagster_evidence/utils/sql_parser.py

@@ -0,0 +1,87 @@
+"""SQL parsing utilities for extracting table references.
+
+This module provides utilities for parsing SQL queries and extracting
+table references that can be used to determine asset dependencies.
+"""
+
+import logging
+
+from dagster._annotations import beta, public
+
+logger = logging.getLogger(__name__)
+
+
+@beta
+@public
+def extract_table_references(
+    sql_query: str,
+    default_database: str | None = None,
+    default_schema: str | None = None,
+) -> list[dict[str, str | None]]:
+    """Parse SQL and extract table references.
+
+    Uses SQLGlot to parse SQL queries and extract all table references,
+    including tables in JOINs, subqueries, and CTEs.
+
+    Args:
+        sql_query: The SQL query to parse.
+        default_database: Default database name to use if not specified in query.
+        default_schema: Default schema name to use if not specified in query.
+
+    Returns:
+        List of dictionaries, each containing:
+        - database: The database/catalog name (or default_database if not specified)
+        - schema: The schema name (or default_schema if not specified)
+        - table: The table name
+
+    Example:
+
+        .. code-block:: python
+
+            from dagster_evidence.utils import extract_table_references
+
+            # Simple query
+            refs = extract_table_references("SELECT * FROM orders")
+            # Returns: [{"database": None, "schema": None, "table": "orders"}]
+
+            # With defaults
+            refs = extract_table_references(
+                "SELECT * FROM orders",
+                default_database="my_db",
+                default_schema="main"
+            )
+            # Returns: [{"database": "my_db", "schema": "main", "table": "orders"}]
+
+            # Fully qualified table
+            refs = extract_table_references("SELECT * FROM prod.sales.orders")
+            # Returns: [{"database": "prod", "schema": "sales", "table": "orders"}]
+    """
+    try:
+        import sqlglot
+        from sqlglot import exp
+    except ImportError:
+        raise ImportError(
+            "sqlglot is required for SQL table reference extraction. "
+            "Install it with: pip install dagster-evidence[sql]"
+        ) from None
+
+    try:
+        parsed = sqlglot.parse(sql_query)
+        tables: list[dict[str, str | None]] = []
+
+        for statement in parsed:
+            if statement is None:
+                continue
+            for table in statement.find_all(exp.Table):
+                tables.append(
+                    {
+                        "database": table.catalog or default_database,
+                        "schema": table.db or default_schema,
+                        "table": table.name,
+                    }
+                )
+
+        return tables
+    except Exception as e:
+        logger.debug(f"Failed to parse SQL: {e}")
+        return []

dagster_evidence-0.2.0.dist-info/METADATA

@@ -0,0 +1,120 @@
+Metadata-Version: 2.4
+Name: dagster-evidence
+Version: 0.2.0
+Summary: Dagster integration with evidence.dev
+Requires-Python: <3.14,>=3.10
+Description-Content-Type: text/markdown
+Requires-Dist: dagster>=1.12.10
+Requires-Dist: pyyaml>=6.0
+Provides-Extra: github-pages
+Requires-Dist: gitpython>=3.1.0; extra == "github-pages"
+Provides-Extra: sql
+Requires-Dist: sqlglot>=26.0.0; extra == "sql"
+Provides-Extra: duckdb
+Requires-Dist: duckdb>=0.10.0; extra == "duckdb"
+Provides-Extra: bigquery
+Requires-Dist: google-cloud-bigquery>=3.0.0; extra == "bigquery"
+Provides-Extra: gsheets
+Requires-Dist: google-api-python-client>=2.0.0; extra == "gsheets"
+Requires-Dist: google-auth>=2.0.0; extra == "gsheets"
+Provides-Extra: all
+Requires-Dist: gitpython>=3.1.0; extra == "all"
+Requires-Dist: sqlglot>=26.0.0; extra == "all"
+Requires-Dist: duckdb>=0.10.0; extra == "all"
+Requires-Dist: google-cloud-bigquery>=3.0.0; extra == "all"
+Requires-Dist: google-api-python-client>=2.0.0; extra == "all"
+Requires-Dist: google-auth>=2.0.0; extra == "all"
+Provides-Extra: test
+Requires-Dist: pytest; extra == "test"
+Requires-Dist: pytest-mock; extra == "test"
+
+# dagster-evidence
+
+Dagster integration with [Evidence](https://evidence.dev/), enabling you to orchestrate Evidence.dev dashboard projects as Dagster assets.
+
+![architecture](./docs/component-architecture.excalidraw.png)
+## Installation
+
+```bash
+# Basic installation
+uv add dagster-evidence
+
+# With specific data source support
+uv add "dagster-evidence[duckdb]"           # DuckDB support
+uv add "dagster-evidence[bigquery]"         # BigQuery support
+uv add "dagster-evidence[gsheets]"          # Google Sheets support
+uv add "dagster-evidence[github-pages]"     # GitHub Pages deployment
+
+# Install multiple extras
+uv add "dagster-evidence[duckdb,bigquery,github-pages]"
+```
+
+## Features
+
+- **Automatic asset discovery**: Automatically generates Dagster assets from Evidence project sources
+- **Build and deployment**: Builds Evidence projects and deploys them to your hosting platform
+- **Customizable translation**: Extend the translator to customize how Evidence sources map to Dagster assets
+- **Type-safe configuration**: Pydantic-based configuration with YAML support
+- **Source change detection**: Optional sensors to detect changes in upstream data sources
+
+## Supported Data Sources
+
+- DuckDB
+- MotherDuck
+- BigQuery
+- Google Sheets
+
+Need additional source types? You can extend the translator to add custom sources, or contribute to the project!
+
+## Deployment Support
+
+**Currently Implemented:**
+- GitHub Pages: Deploy to GitHub Pages with automatic git push
+- Custom commands: Run any custom deployment script or command
+
+**Need another deployment target?** Open an issue and tag [@milicevica23](https://github.com/milicevica23) with your deployment requirements.
+
+## Quick Start
+
+### Component Configuration (YAML)
+
+```yaml
+# defs.yaml
+type: dagster_evidence.EvidenceProjectComponentV2
+attributes:
+  evidence_project:
+    project_type: local
+    project_path: ./my-evidence-project
+    enable_source_assets_hiding: true
+    enable_source_sensors: false
+    project_deployment:
+      type: github_pages
+      github_repo: my-org/my-dashboard
+      branch: gh-pages
+```
+
+## Advanced Usage
+
+### Custom Translator
+
+You can customize how Evidence sources are translated to Dagster assets:
+
+```python
+from dagster_evidence import DagsterEvidenceTranslator, EvidenceSourceTranslatorData
+import dagster as dg
+
+class MyTranslator(DagsterEvidenceTranslator):
+    def get_asset_spec(self, data):
+        if isinstance(data, EvidenceSourceTranslatorData):
+            # Custom logic for source assets
+            return dg.AssetSpec(
+                key=dg.AssetKey(["custom", data.query.name]),
+                kinds={"evidence", data.source_content.connection.type},
+            )
+        return super().get_asset_spec(data)
+```
+
+
+## Contributing
+
+Contributions are welcome! Please see the main repository's contribution guidelines.

dagster_evidence-0.2.0.dist-info/RECORD

@@ -0,0 +1,17 @@
+dagster_evidence/__init__.py,sha256=vXUZVFLQwWH7ABbpnL5oVXwlRfwORIzRs8T49q8U-j0,903
+dagster_evidence/resource.py,sha256=-9nIHmhEIUONnFIgZunDWmayNzkYBvA6BvlzQn2_R4w,2198
+dagster_evidence/components/__init__.py,sha256=gZbG2q0F1d1DtFC-dfO0DEYzgABXFseDetpKhFbzEnc,582
+dagster_evidence/components/deployments.py,sha256=CZijqpYK0rGX4oFra50W_a1OqblwyeJQaVavFbRz0tM,16784
+dagster_evidence/components/evidence_project_v2.py,sha256=185YRveCapIXpukNsHWoj1ajVjL2fot50HQRz_pNUH4,6468
+dagster_evidence/components/projects.py,sha256=o96ceqGPgLi8AA0bR0r3K2U4dEshVJBEATc8q7uenj8,23658
+dagster_evidence/components/sources.py,sha256=Fqe7QA14Db4wK_jTIvQ0eIl_cUPZ04qBsyWb6fWFT6E,45501
+dagster_evidence/components/translator.py,sha256=bFRY-XTMeZSZjmJczvNRS8mlzG7z5jiC1f3vt_KhOSg,6079
+dagster_evidence/lib/__init__.py,sha256=pLil7ph_5C5j_WEVUgfU91u9y-cg7OaX4AKE-9elWNo,66
+dagster_evidence/lib/evidence_project.py,sha256=4ZaqSgfUK6oeZ3dRGa3Zr5a2oUPSVXoKkJLWAHuNYnQ,2820
+dagster_evidence/utils/__init__.py,sha256=RS1hWN41SPPAok4TtsIMnsW4jIwtUJGshaqqkGhURAg,136
+dagster_evidence/utils/sql_parser.py,sha256=NnbvMMw_XUUF-eV3rScaRlhWrxquQM562EyRgiePL88,2821
+dagster_evidence-0.2.0.dist-info/METADATA,sha256=BZ6d2SVDwJ1vV-2Y0mngzEJHBQIKTDCk0ruX0M3eO3Y,3974
+dagster_evidence-0.2.0.dist-info/WHEEL,sha256=wUyA8OaulRlbfwMtmQsvNngGrxQHAvkKcvRmdizlJi0,92
+dagster_evidence-0.2.0.dist-info/entry_points.txt,sha256=CZx5kJ7dwUo0ke4Vu0B-GaSxM8B1vlLIzXrT14ovUr0,114
+dagster_evidence-0.2.0.dist-info/top_level.txt,sha256=YAZVTuvV8f7bWYwERV2uGnMQj93E4Vq3KHdsGPpN6IY,17
+dagster_evidence-0.2.0.dist-info/RECORD,,

{dagster_evidence-0.1.7.dist-info → dagster_evidence-0.2.0.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (80.9.0)
+Generator: setuptools (80.10.2)
 Root-Is-Purelib: true
 Tag: py3-none-any
 

dagster_evidence-0.1.7.dist-info/METADATA

@@ -1,23 +0,0 @@
-Metadata-Version: 2.4
-Name: dagster-evidence
-Version: 0.1.7
-Summary: Dagster integration with evidence.dev
-Requires-Python: >=3.10
-Description-Content-Type: text/markdown
-Requires-Dist: dagster>=1.10.9
-
-# dagster-evidence
-
-`dagster-evidence` provides a Dagster Component that integrates with [evidence](https://evidence.dev/), an open-source tool for building data apps.
-
-## Test
-
-```sh
-make test
-```
-
-## Build
-
-```sh
-make build
-```

dagster_evidence-0.1.7.dist-info/RECORD

@@ -1,9 +0,0 @@
-dagster_evidence/__init__.py,sha256=zVzzLtXbOCxlxneYi8A5fwMlTyY41TSJyfjYuRPNqK8,349
-dagster_evidence/resource.py,sha256=U9lBlJQTXVPft504tai6rMVExuFnbEqq59EYqa_uXDY,2184
-dagster_evidence/lib/__init__.py,sha256=pLil7ph_5C5j_WEVUgfU91u9y-cg7OaX4AKE-9elWNo,66
-dagster_evidence/lib/evidence_project.py,sha256=6RVpYBbjJnmviudxIujEv5sL608ZohHnlokVuHYutW8,2806
-dagster_evidence-0.1.7.dist-info/METADATA,sha256=w29KYOFGXEL49St4U5nLjJEXkHQNmC17TSN9Xd9Ujys,433
-dagster_evidence-0.1.7.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-dagster_evidence-0.1.7.dist-info/entry_points.txt,sha256=CZx5kJ7dwUo0ke4Vu0B-GaSxM8B1vlLIzXrT14ovUr0,114
-dagster_evidence-0.1.7.dist-info/top_level.txt,sha256=YAZVTuvV8f7bWYwERV2uGnMQj93E4Vq3KHdsGPpN6IY,17
-dagster_evidence-0.1.7.dist-info/RECORD,,