dagster-dlt 0.25.9__tar.gz

dagster-dlt-0.25.9/LICENSE

@@ -0,0 +1,201 @@
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "{}"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright 2023 Dagster Labs, Inc".
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

dagster-dlt-0.25.9/MANIFEST.in

@@ -0,0 +1,4 @@
+include README.md
+include LICENSE
+include integration.yaml
+include dagster_dlt/py.typed

dagster-dlt-0.25.9/PKG-INFO

@@ -0,0 +1,17 @@
+Metadata-Version: 2.1
+Name: dagster-dlt
+Version: 0.25.9
+Summary: Package for performing ETL/ELT tasks with dlt in Dagster.
+Home-page: https://github.com/dagster-io/dagster/tree/master/python_modules/libraries/dagster-dlt
+Author: Dagster Labs
+Author-email: hello@dagsterlabs.com
+License: Apache-2.0
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: License :: OSI Approved :: Apache Software License
+Classifier: Operating System :: OS Independent
+Requires-Python: >=3.9,<3.13
+Provides-Extra: test
+License-File: LICENSE

dagster-dlt-0.25.9/README.md

@@ -0,0 +1,4 @@
+# dagster-dlt
+
+The docs for `dagster-dlt` can be found
+[here](https://docs.dagster.io/_apidocs/libraries/dagster-dlt).

dagster-dlt-0.25.9/dagster_dlt/__init__.py

@@ -0,0 +1,11 @@
+from dagster._core.libraries import DagsterLibraryRegistry
+
+from dagster_dlt.asset_decorator import build_dlt_asset_specs, dlt_assets
+from dagster_dlt.resource import DagsterDltResource
+from dagster_dlt.translator import DagsterDltTranslator
+from dagster_dlt.version import __version__
+
+__all__ = ["DagsterDltResource", "DagsterDltTranslator", "build_dlt_asset_specs", "dlt_assets"]
+
+
+DagsterLibraryRegistry.register("dagster-dlt", __version__)

dagster-dlt-0.25.9/dagster_dlt/asset_decorator.py

@@ -0,0 +1,151 @@
+from collections.abc import Mapping, Sequence
+from typing import Any, Callable, Optional
+
+from dagster import (
+    AssetsDefinition,
+    AssetSpec,
+    PartitionsDefinition,
+    _check as check,
+    multi_asset,
+)
+from dlt.extract.source import DltSource
+from dlt.pipeline.pipeline import Pipeline
+
+from dagster_dlt.constants import META_KEY_PIPELINE, META_KEY_SOURCE, META_KEY_TRANSLATOR
+from dagster_dlt.translator import DagsterDltTranslator
+
+
+def build_dlt_asset_specs(
+    dlt_source: DltSource,
+    dlt_pipeline: Pipeline,
+    dagster_dlt_translator: Optional[DagsterDltTranslator] = None,
+) -> Sequence[AssetSpec]:
+    """Build a list of asset specs from a dlt source and pipeline.
+
+    Args:
+        dlt_source (DltSource): dlt source object
+        dlt_pipeline (Pipeline): dlt pipeline object
+        dagster_dlt_translator (Optional[DagsterDltTranslator]): Allows customizing how to
+            map dlt project to asset keys and asset metadata.
+
+    Returns:
+        List[AssetSpec] list of asset specs from dlt source and pipeline
+
+    """
+    dagster_dlt_translator = dagster_dlt_translator or DagsterDltTranslator()
+    return [
+        AssetSpec(
+            key=dagster_dlt_translator.get_asset_key(dlt_source_resource),
+            automation_condition=dagster_dlt_translator.get_automation_condition(
+                dlt_source_resource
+            ),
+            deps=dagster_dlt_translator.get_deps_asset_keys(dlt_source_resource),
+            description=dagster_dlt_translator.get_description(dlt_source_resource),
+            group_name=dagster_dlt_translator.get_group_name(dlt_source_resource),
+            metadata={
+                **dagster_dlt_translator.get_metadata(dlt_source_resource),
+                META_KEY_SOURCE: dlt_source,
+                META_KEY_PIPELINE: dlt_pipeline,
+                META_KEY_TRANSLATOR: dagster_dlt_translator,
+            },
+            owners=dagster_dlt_translator.get_owners(dlt_source_resource),
+            tags=dagster_dlt_translator.get_tags(dlt_source_resource),
+            kinds=dagster_dlt_translator.get_kinds(dlt_source_resource, dlt_pipeline.destination),
+        )
+        for dlt_source_resource in dlt_source.selected_resources.values()
+    ]
+
+
+def dlt_assets(
+    *,
+    dlt_source: DltSource,
+    dlt_pipeline: Pipeline,
+    name: Optional[str] = None,
+    group_name: Optional[str] = None,
+    dagster_dlt_translator: Optional[DagsterDltTranslator] = None,
+    partitions_def: Optional[PartitionsDefinition] = None,
+    op_tags: Optional[Mapping[str, Any]] = None,
+) -> Callable[[Callable[..., Any]], AssetsDefinition]:
+    """Asset Factory for using data load tool (dlt).
+
+    Args:
+        dlt_source (DltSource): The DltSource to be ingested.
+        dlt_pipeline (Pipeline): The dlt Pipeline defining the destination parameters.
+        name (Optional[str], optional): The name of the op.
+        group_name (Optional[str], optional): The name of the asset group.
+        dagster_dlt_translator (DagsterDltTranslator, optional): Customization object for defining asset parameters from dlt resources.
+        partitions_def (Optional[PartitionsDefinition]): Optional partitions definition.
+        op_tags (Optional[Mapping[str, Any]]): The tags for the underlying op.
+
+    Examples:
+        Loading Hubspot data to Snowflake with an auto materialize policy using the dlt verified source:
+
+        .. code-block:: python
+
+            from dagster_dlt import DagsterDltResource, DagsterDltTranslator, dlt_assets
+
+
+            class HubspotDagsterDltTranslator(DagsterDltTranslator):
+                @public
+                def get_auto_materialize_policy(self, resource: DltResource) -> Optional[AutoMaterializePolicy]:
+                    return AutoMaterializePolicy.eager().with_rules(
+                        AutoMaterializeRule.materialize_on_cron("0 0 * * *")
+                    )
+
+
+            @dlt_assets(
+                dlt_source=hubspot(include_history=True),
+                dlt_pipeline=pipeline(
+                    pipeline_name="hubspot",
+                    dataset_name="hubspot",
+                    destination="snowflake",
+                    progress="log",
+                ),
+                name="hubspot",
+                group_name="hubspot",
+                dagster_dlt_translator=HubspotDagsterDltTranslator(),
+            )
+            def hubspot_assets(context: AssetExecutionContext, dlt: DagsterDltResource):
+                yield from dlt.run(context=context)
+
+        Loading Github issues to snowflake:
+
+        .. code-block:: python
+
+            from dagster_dlt import DagsterDltResource, dlt_assets
+
+
+            @dlt_assets(
+                dlt_source=github_reactions(
+                    "dagster-io", "dagster", items_per_page=100, max_items=250
+                ),
+                dlt_pipeline=pipeline(
+                    pipeline_name="github_issues",
+                    dataset_name="github",
+                    destination="snowflake",
+                    progress="log",
+                ),
+                name="github",
+                group_name="github",
+            )
+            def github_reactions_dagster_assets(context: AssetExecutionContext, dlt: DagsterDltResource):
+                yield from dlt.run(context=context)
+
+    """
+    dagster_dlt_translator = check.inst_param(
+        dagster_dlt_translator or DagsterDltTranslator(),
+        "dagster_dlt_translator",
+        DagsterDltTranslator,
+    )
+    return multi_asset(
+        name=name,
+        group_name=group_name,
+        can_subset=True,
+        partitions_def=partitions_def,
+        op_tags=op_tags,
+        specs=build_dlt_asset_specs(
+            dlt_source=dlt_source,
+            dlt_pipeline=dlt_pipeline,
+            dagster_dlt_translator=dagster_dlt_translator,
+        ),
+    )

dagster-dlt-0.25.9/dagster_dlt/constants.py

@@ -0,0 +1,3 @@
+META_KEY_SOURCE = "dagster_dlt/source"
+META_KEY_PIPELINE = "dagster_dlt/pipeline"
+META_KEY_TRANSLATOR = "dagster_dlt/translator"

dagster-dlt-0.25.9/dagster_dlt/dlt_event_iterator.py

@@ -0,0 +1,108 @@
+from collections.abc import Iterator
+from typing import Optional, Union
+
+from dagster import AssetMaterialization, MaterializeResult
+from dagster._annotations import experimental, public
+from dagster._core.definitions.metadata.metadata_set import TableMetadataSet
+from dagster._core.execution.context.asset_execution_context import AssetExecutionContext
+from dagster._core.execution.context.op_execution_context import OpExecutionContext
+from dlt import Pipeline
+from typing_extensions import TypeVar
+
+DltEventType = Union[AssetMaterialization, MaterializeResult]
+T = TypeVar("T", bound=DltEventType)
+
+
+def _fetch_row_count(
+    dlt_pipeline: Pipeline,
+    table_name: str,
+) -> Optional[int]:
+    """Exists mostly for ease of testing."""
+    with dlt_pipeline.sql_client() as client:
+        with client.execute_query(
+            f"select count(*) as row_count from {table_name}",
+        ) as cursor:
+            result = cursor.fetchone()
+            if result is not None and isinstance(result[0], int):
+                return result[0]
+            else:
+                return None
+
+
+def fetch_row_count_metadata(
+    materialization: DltEventType,
+    context: Union[OpExecutionContext, AssetExecutionContext],
+    dlt_pipeline: Pipeline,
+) -> TableMetadataSet:
+    if not materialization.metadata:
+        raise Exception("Missing required metadata to retrieve row count.")
+    if context.has_partition_key:
+        rows_loaded = materialization.metadata.get("rows_loaded")
+        return TableMetadataSet(
+            partition_row_count=rows_loaded.value if rows_loaded else 0  # type: ignore
+        )
+
+    jobs_metadata = materialization.metadata.get("jobs")
+    if not jobs_metadata or not isinstance(jobs_metadata, list):
+        raise Exception("Missing jobs metadata to retrieve row count.")
+    table_name = jobs_metadata[0].get("table_name")
+    try:
+        return TableMetadataSet(row_count=_fetch_row_count(dlt_pipeline, table_name))
+    # Filesystem does not have a SQL client and table might not be found
+    except Exception as e:
+        context.log.error(
+            f"An error occurred while fetching row count for {table_name}. Row count metadata"
+            " will not be included in the event.\n\n"
+            f"Exception: {e}"
+        )
+        return TableMetadataSet(row_count=None)
+
+
+class DltEventIterator(Iterator[T]):
+    """A wrapper around an iterator of Dlt events which contains additional methods for
+    post-processing the events, such as fetching column metadata.
+    """
+
+    def __init__(
+        self,
+        events: Iterator[T],
+        context: Union[OpExecutionContext, AssetExecutionContext],
+        dlt_pipeline: Pipeline,
+    ) -> None:
+        self._inner_iterator = events
+        self._context = context
+        self._dlt_pipeline = dlt_pipeline
+
+    def __next__(self) -> T:
+        return next(self._inner_iterator)
+
+    def __iter__(self) -> "DltEventIterator[T]":
+        return self
+
+    @public
+    @experimental
+    def fetch_row_count(self) -> "DltEventIterator":
+        """Fetches row count metadata for each resource loaded by dlt.
+
+        Retrieves the row count for each resource.
+
+        Returns:
+            DltEventIterator: An iterator of Dagster events with row count metadata attached.
+        """
+
+        def _fetch_row_count() -> Iterator[T]:
+            for event in self:
+                row_count_metadata = fetch_row_count_metadata(
+                    event,
+                    context=self._context,
+                    # table_name=str(self._resource.table_name),
+                    dlt_pipeline=self._dlt_pipeline,
+                )
+                if event.metadata:
+                    yield event._replace(metadata={**row_count_metadata, **event.metadata})
+
+        return DltEventIterator[T](
+            _fetch_row_count(),
+            self._context,
+            self._dlt_pipeline,
+        )

dagster-dlt-0.25.9/dagster_dlt/py.typed

@@ -0,0 +1 @@
+partial

dagster-dlt-0.25.9/dagster_dlt/resource.py

@@ -0,0 +1,301 @@
+from collections.abc import Iterator, Mapping
+from datetime import datetime, timezone
+from typing import Any, Optional, Union
+
+from dagster import (
+    AssetExecutionContext,
+    AssetMaterialization,
+    ConfigurableResource,
+    MaterializeResult,
+    MetadataValue,
+    OpExecutionContext,
+    TableColumnConstraints,
+    _check as check,
+)
+from dagster._annotations import experimental, public
+from dagster._core.definitions.metadata.metadata_set import TableMetadataSet
+from dagster._core.definitions.metadata.table import TableColumn, TableSchema
+from dlt.common.pipeline import LoadInfo
+from dlt.common.schema import Schema
+from dlt.extract.resource import DltResource
+from dlt.extract.source import DltSource
+from dlt.pipeline.pipeline import Pipeline
+
+from dagster_dlt.constants import META_KEY_PIPELINE, META_KEY_SOURCE, META_KEY_TRANSLATOR
+from dagster_dlt.dlt_event_iterator import DltEventIterator, DltEventType
+from dagster_dlt.translator import DagsterDltTranslator
+
+
+@experimental
+class DagsterDltResource(ConfigurableResource):
+    @classmethod
+    def _is_dagster_maintained(cls) -> bool:
+        return True
+
+    def _cast_load_info_metadata(self, mapping: Mapping[Any, Any]) -> Mapping[Any, Any]:
+        """Converts datetime and timezone values in a mapping to strings.
+
+        Workaround for dagster._core.errors.DagsterInvalidMetadata: Could not resolve the metadata
+        value for "jobs" to a known type. Value is not JSON serializable.
+
+        Args:
+            mapping (Mapping): Dictionary possibly containing datetime/timezone values
+
+        Returns:
+            Mapping[Any, Any]: Metadata with datetime and timezone values casted to strings
+
+        """
+        try:
+            # zoneinfo is python >= 3.9
+            from zoneinfo import ZoneInfo
+
+            casted_instance_types = (datetime, timezone, ZoneInfo)
+        except:
+            from dateutil.tz import tzfile
+
+            casted_instance_types = (datetime, timezone, tzfile)
+
+        def _recursive_cast(value: Any):
+            if isinstance(value, dict):
+                return {k: _recursive_cast(v) for k, v in value.items()}
+            elif isinstance(value, list):
+                return [_recursive_cast(item) for item in value]
+            elif isinstance(value, casted_instance_types):
+                return str(value)
+            else:
+                return value
+
+        return {k: _recursive_cast(v) for k, v in mapping.items()}
+
+    def _extract_table_schema_metadata(self, table_name: str, schema: Schema) -> TableSchema:
+        # Pyright does not detect the default value from 'pop' and 'get'
+        try:
+            table_schema = TableSchema(
+                columns=[
+                    TableColumn(
+                        name=column.pop("name", ""),  # type: ignore
+                        type=column.pop("data_type", "string"),  # type: ignore
+                        constraints=TableColumnConstraints(
+                            nullable=column.pop("nullable", True),  # type: ignore
+                            unique=column.pop("unique", False),  # type: ignore
+                            other=[*column.keys()],  # e.g. "primary_key" or "foreign_key"
+                        ),
+                    )
+                    for column in schema.get_table_columns(table_name).values()
+                ]
+            )
+        except KeyError:
+            # We want the asset to materialize even if we failed to extract the table schema.
+            table_schema = TableSchema(columns=[])
+        return table_schema
+
+    def extract_resource_metadata(
+        self,
+        context: Union[OpExecutionContext, AssetExecutionContext],
+        resource: DltResource,
+        load_info: LoadInfo,
+        dlt_pipeline: Pipeline,
+    ) -> Mapping[str, Any]:
+        """Helper method to extract dlt resource metadata from load info dict.
+
+        Args:
+            context (Union[OpExecutionContext, AssetExecutionContext]): Asset or op execution context
+            resource (DltResource): The dlt resource being materialized
+            load_info (LoadInfo): Run metadata from dlt `pipeline.run(...)`
+            dlt_pipeline (Pipeline): The dlt pipeline used by `resource`
+
+        Returns:
+            Mapping[str, Any]: Asset-specific metadata dictionary
+
+        """
+        dlt_base_metadata_types = {
+            "first_run",
+            "started_at",
+            "finished_at",
+            "dataset_name",
+            "destination_name",
+            "destination_type",
+        }
+
+        load_info_dict = self._cast_load_info_metadata(load_info.asdict())
+
+        # shared metadata that is displayed for all assets
+        base_metadata = {k: v for k, v in load_info_dict.items() if k in dlt_base_metadata_types}
+        default_schema = dlt_pipeline.default_schema
+        normalized_table_name = default_schema.naming.normalize_table_identifier(
+            str(resource.table_name)
+        )
+        # job metadata for specific target `normalized_table_name`
+        base_metadata["jobs"] = [
+            job
+            for load_package in load_info_dict.get("load_packages", [])
+            for job in load_package.get("jobs", [])
+            if job.get("table_name") == normalized_table_name
+        ]
+        rows_loaded = dlt_pipeline.last_trace.last_normalize_info.row_counts.get(
+            normalized_table_name
+        )
+        if rows_loaded:
+            base_metadata["rows_loaded"] = MetadataValue.int(rows_loaded)
+
+        schema: Optional[str] = None
+        for load_package in load_info_dict.get("load_packages", []):
+            for table in load_package.get("tables", []):
+                if table.get("name") == normalized_table_name:
+                    schema = table.get("schema_name")
+                    break
+            if schema:
+                break
+
+        destination_name: Optional[str] = base_metadata.get("destination_name")
+        table_name = None
+        if destination_name and schema:
+            table_name = ".".join([destination_name, schema, normalized_table_name])
+
+        child_table_names = [
+            name
+            for name in default_schema.data_table_names()
+            if name.startswith(f"{normalized_table_name}__")
+        ]
+        child_table_schemas = {
+            table_name: self._extract_table_schema_metadata(table_name, default_schema)
+            for table_name in child_table_names
+        }
+        table_schema = self._extract_table_schema_metadata(normalized_table_name, default_schema)
+
+        base_metadata = {
+            **child_table_schemas,
+            **base_metadata,
+            **TableMetadataSet(
+                column_schema=table_schema,
+                table_name=table_name,
+            ),
+        }
+
+        return base_metadata
+
+    @public
+    def run(
+        self,
+        context: Union[OpExecutionContext, AssetExecutionContext],
+        dlt_source: Optional[DltSource] = None,
+        dlt_pipeline: Optional[Pipeline] = None,
+        dagster_dlt_translator: Optional[DagsterDltTranslator] = None,
+        **kwargs,
+    ) -> DltEventIterator[DltEventType]:
+        """Runs the dlt pipeline with subset support.
+
+        Args:
+            context (Union[OpExecutionContext, AssetExecutionContext]): Asset or op execution context
+            dlt_source (Optional[DltSource]): optional dlt source if resource is used from an `@op`
+            dlt_pipeline (Optional[Pipeline]): optional dlt pipeline if resource is used from an `@op`
+            dagster_dlt_translator (Optional[DagsterDltTranslator]): optional dlt translator if resource is used from an `@op`
+            **kwargs (dict[str, Any]): Keyword args passed to pipeline `run` method
+
+        Returns:
+            DltEventIterator[DltEventType]: An iterator of MaterializeResult or AssetMaterialization
+
+        """
+        # This resource can be used in both `asset` and `op` definitions. In the context of an asset
+        # execution, we retrieve the dlt source, pipeline, and translator from the asset metadata
+        # as a fallback mechanism. We give preference to explicit parameters to make it easy to
+        # customize execution, e.g., when using partitions.
+        if isinstance(context, AssetExecutionContext):
+            metadata_by_key = context.assets_def.metadata_by_key
+            first_asset_metadata = next(iter(metadata_by_key.values()))
+
+            dlt_source = check.inst(
+                dlt_source or first_asset_metadata.get(META_KEY_SOURCE), DltSource
+            )
+            dlt_pipeline = check.inst(
+                dlt_pipeline or first_asset_metadata.get(META_KEY_PIPELINE), Pipeline
+            )
+            dagster_dlt_translator = check.inst(
+                dagster_dlt_translator or first_asset_metadata.get(META_KEY_TRANSLATOR),
+                DagsterDltTranslator,
+            )
+
+        dlt_source = check.not_none(
+            dlt_source, "dlt_source is a required parameter in an op context"
+        )
+        dlt_pipeline = check.not_none(
+            dlt_pipeline, "dlt_pipeline is a required parameter in an op context"
+        )
+
+        # Default to base translator if undefined
+        dagster_dlt_translator = dagster_dlt_translator or DagsterDltTranslator()
+        return DltEventIterator(
+            self._run(
+                context=context,
+                dlt_source=dlt_source,
+                dlt_pipeline=dlt_pipeline,
+                dagster_dlt_translator=dagster_dlt_translator,
+                **kwargs,
+            ),
+            context=context,
+            dlt_pipeline=dlt_pipeline,
+        )
+
+    def _run(
+        self,
+        context: Union[OpExecutionContext, AssetExecutionContext],
+        dlt_source: DltSource,
+        dlt_pipeline: Pipeline,
+        dagster_dlt_translator: DagsterDltTranslator,
+        **kwargs,
+    ) -> Iterator[DltEventType]:
+        """Runs the dlt pipeline with subset support.
+
+        Args:
+            context (Union[OpExecutionContext, AssetExecutionContext]): Asset or op execution context
+            dlt_source (Optional[DltSource]): optional dlt source if resource is used from an `@op`
+            dlt_pipeline (Optional[Pipeline]): optional dlt pipeline if resource is used from an `@op`
+            dagster_dlt_translator (Optional[DagsterDltTranslator]): optional dlt translator if resource is used from an `@op`
+            **kwargs (dict[str, Any]): Keyword args passed to pipeline `run` method
+
+        Returns:
+            DltEventIterator[DltEventType]: An iterator of MaterializeResult or AssetMaterialization
+
+        """
+        asset_key_dlt_source_resource_mapping = {
+            dagster_dlt_translator.get_asset_key(dlt_source_resource): dlt_source_resource
+            for dlt_source_resource in dlt_source.selected_resources.values()
+        }
+
+        # Filter sources by asset key sub-selection
+        if context.is_subset:
+            asset_key_dlt_source_resource_mapping = {
+                asset_key: asset_dlt_source_resource
+                for (
+                    asset_key,
+                    asset_dlt_source_resource,
+                ) in asset_key_dlt_source_resource_mapping.items()
+                if asset_key in context.selected_asset_keys
+            }
+            dlt_source = dlt_source.with_resources(
+                *[
+                    dlt_source_resource.name
+                    for dlt_source_resource in asset_key_dlt_source_resource_mapping.values()
+                    if dlt_source_resource
+                ]
+            )
+
+        load_info = dlt_pipeline.run(dlt_source, **kwargs)
+
+        load_info.raise_on_failed_jobs()
+
+        has_asset_def: bool = bool(context and context.has_assets_def)
+
+        for (
+            asset_key,
+            dlt_source_resource,
+        ) in asset_key_dlt_source_resource_mapping.items():
+            metadata = self.extract_resource_metadata(
+                context, dlt_source_resource, load_info, dlt_pipeline
+            )
+
+            if has_asset_def:
+                yield MaterializeResult(asset_key=asset_key, metadata=metadata)
+
+            else:
+                yield AssetMaterialization(asset_key=asset_key, metadata=metadata)

dagster-dlt-0.25.9/dagster_dlt/translator.py

@@ -0,0 +1,168 @@
+from collections.abc import Iterable, Mapping, Sequence
+from dataclasses import dataclass
+from typing import Any, Optional
+
+from dagster import AssetKey, AutoMaterializePolicy, AutomationCondition
+from dagster._annotations import public
+from dlt.common.destination import Destination
+from dlt.extract.resource import DltResource
+
+
+@dataclass
+class DagsterDltTranslator:
+    @public
+    def get_asset_key(self, resource: DltResource) -> AssetKey:
+        """Defines asset key for a given dlt resource key and dataset name.
+
+        This method can be overriden to provide custom asset key for a dlt resource.
+
+        Args:
+            resource (DltResource): dlt resource
+
+        Returns:
+            AssetKey of Dagster asset derived from dlt resource
+
+        """
+        return AssetKey(f"dlt_{resource.source_name}_{resource.name}")
+
+    @public
+    def get_auto_materialize_policy(self, resource: DltResource) -> Optional[AutoMaterializePolicy]:
+        """Defines resource specific auto materialize policy.
+
+        This method can be overriden to provide custom auto materialize policy for a dlt resource.
+
+        Args:
+            resource (DltResource): dlt resource
+
+        Returns:
+            Optional[AutoMaterializePolicy]: The automaterialize policy for a resource
+
+        """
+        return None
+
+    @public
+    def get_automation_condition(self, resource: DltResource) -> Optional[AutomationCondition]:
+        """Defines resource specific automation condition.
+
+        This method can be overridden to provide custom automation condition for a dlt resource.
+
+        Args:
+            resource (DltResource): dlt resource
+
+        Returns:
+            Optional[AutomationCondition]: The automation condition for a resource
+
+        """
+        auto_materialize_policy = self.get_auto_materialize_policy(resource)
+        return (
+            auto_materialize_policy.to_automation_condition() if auto_materialize_policy else None
+        )
+
+    @public
+    def get_deps_asset_keys(self, resource: DltResource) -> Iterable[AssetKey]:
+        """Defines upstream asset dependencies given a dlt resource.
+
+        Defaults to a concatenation of `resource.source_name` and `resource.name`.
+
+        Args:
+            resource (DltResource): dlt resource
+
+        Returns:
+            Iterable[AssetKey]: The Dagster asset keys upstream of `dlt_resource_key`.
+
+        """
+        if resource.is_transformer:
+            pipe = resource._pipe  # noqa: SLF001
+            while pipe.has_parent:
+                pipe = pipe.parent
+            return [AssetKey(f"{resource.source_name}_{pipe.name}")]
+        return [AssetKey(f"{resource.source_name}_{resource.name}")]
+
+    @public
+    def get_description(self, resource: DltResource) -> Optional[str]:
+        """A method that takes in a dlt resource returns the Dagster description of the resource.
+
+        This method can be overriden to provide a custom description for a dlt resource.
+
+        Args:
+            resource (DltResource): dlt resource
+
+        Returns:
+            Optional[str]: The Dagster description for the dlt resource.
+        """
+        pipe = resource._pipe  # noqa: SLF001
+        # If the function underlying the resource is a single callable,
+        # return the docstring of the callable.
+        if len(pipe.steps) == 1 and callable(pipe.steps[0]):
+            return pipe.steps[0].__doc__
+        return None
+
+    @public
+    def get_group_name(self, resource: DltResource) -> Optional[str]:
+        """A method that takes in a dlt resource and returns the Dagster group name of the resource.
+
+        This method can be overriden to provide a custom group name for a dlt resource.
+
+        Args:
+            resource (DltResource): dlt resource
+
+        Returns:
+            Optional[str]: A Dagster group name for the dlt resource.
+        """
+        return None
+
+    @public
+    def get_metadata(self, resource: DltResource) -> Mapping[str, Any]:
+        """Defines resource specific metadata.
+
+        Args:
+            resource (DltResource): dlt resource
+
+        Returns:
+            Mapping[str, Any]: The custom metadata entries for this resource.
+        """
+        return {}
+
+    @public
+    def get_owners(self, resource: DltResource) -> Optional[Sequence[str]]:
+        """A method that takes in a dlt resource and returns the Dagster owners of the resource.
+
+        This method can be overriden to provide custom owners for a dlt resource.
+
+        Args:
+            resource (DltResource): dlt resource
+
+        Returns:
+            Optional[Sequence[str]]: A sequence of Dagster owners for the dlt resource.
+        """
+        return None
+
+    @public
+    def get_tags(self, resource: DltResource) -> Mapping[str, str]:
+        """A method that takes in a dlt resource and returns the Dagster tags of the structure.
+
+        This method can be overriden to provide custom tags for a dlt resource.
+
+        Args:
+            resource (DltResource): dlt resource
+
+        Returns:
+            Optional[Mapping[str, str]]: A dictionary representing the Dagster tags for the
+                dlt resource.
+        """
+        return {}
+
+    @public
+    def get_kinds(self, resource: DltResource, destination: Destination) -> set[str]:
+        """A method that takes in a dlt resource and returns the kinds which should be
+        attached. Defaults to the destination type and "dlt".
+
+        This method can be overriden to provide custom kinds for a dlt resource.
+
+        Args:
+            resource (DltResource): dlt resource
+
+        Returns:
+            Set[str]: The kinds of the asset.
+        """
+        return {"dlt", destination.destination_name}

dagster-dlt-0.25.9/dagster_dlt/version.py

@@ -0,0 +1 @@
+__version__ = "0.25.9"

dagster-dlt-0.25.9/dagster_dlt.egg-info/PKG-INFO

@@ -0,0 +1,17 @@
+Metadata-Version: 2.1
+Name: dagster-dlt
+Version: 0.25.9
+Summary: Package for performing ETL/ELT tasks with dlt in Dagster.
+Home-page: https://github.com/dagster-io/dagster/tree/master/python_modules/libraries/dagster-dlt
+Author: Dagster Labs
+Author-email: hello@dagsterlabs.com
+License: Apache-2.0
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: License :: OSI Approved :: Apache Software License
+Classifier: Operating System :: OS Independent
+Requires-Python: >=3.9,<3.13
+Provides-Extra: test
+License-File: LICENSE

dagster-dlt-0.25.9/dagster_dlt.egg-info/SOURCES.txt

@@ -0,0 +1,20 @@
+LICENSE
+MANIFEST.in
+README.md
+integration.yaml
+setup.cfg
+setup.py
+dagster_dlt/__init__.py
+dagster_dlt/asset_decorator.py
+dagster_dlt/constants.py
+dagster_dlt/dlt_event_iterator.py
+dagster_dlt/py.typed
+dagster_dlt/resource.py
+dagster_dlt/translator.py
+dagster_dlt/version.py
+dagster_dlt.egg-info/PKG-INFO
+dagster_dlt.egg-info/SOURCES.txt
+dagster_dlt.egg-info/dependency_links.txt
+dagster_dlt.egg-info/not-zip-safe
+dagster_dlt.egg-info/requires.txt
+dagster_dlt.egg-info/top_level.txt

dagster-dlt-0.25.9/dagster_dlt.egg-info/dependency_links.txt

@@ -0,0 +1 @@
+

dagster-dlt-0.25.9/dagster_dlt.egg-info/not-zip-safe

@@ -0,0 +1 @@
+

dagster-dlt-0.25.9/dagster_dlt.egg-info/requires.txt

@@ -0,0 +1,5 @@
+dagster==1.9.9
+dlt>=0.4
+
+[test]
+duckdb

dagster-dlt-0.25.9/dagster_dlt.egg-info/top_level.txt

@@ -0,0 +1 @@
+dagster_dlt

dagster-dlt-0.25.9/integration.yaml

@@ -0,0 +1,5 @@
+name: Dagster dlt
+suspended: false
+owner: Dagster
+maintainer_type: dagster
+maintainer_email: hello@dagsterlabs.com

dagster-dlt-0.25.9/setup.cfg

@@ -0,0 +1,14 @@
+[metadata]
+license_files = LICENSE
+
+[check-manifest]
+ignore = 
+	.coveragerc
+	tox.ini
+	pytest.ini
+	dagster_dlt_tests/**
+
+[egg_info]
+tag_build = 
+tag_date = 0
+

dagster-dlt-0.25.9/setup.py

@@ -0,0 +1,43 @@
+from pathlib import Path
+
+from setuptools import find_packages, setup
+
+
+def get_version() -> str:
+    version: dict[str, str] = {}
+    with open(Path(__file__).parent / "dagster_dlt/version.py", encoding="utf8") as fp:
+        exec(fp.read(), version)
+
+    return version["__version__"]
+
+
+ver = get_version()
+# dont pin dev installs to avoid pip dep resolver issues
+pin = "" if ver == "1!0+dev" else f"=={ver}"
+setup(
+    name="dagster-dlt",
+    version=ver,
+    author="Dagster Labs",
+    author_email="hello@dagsterlabs.com",
+    license="Apache-2.0",
+    description="Package for performing ETL/ELT tasks with dlt in Dagster.",
+    url="https://github.com/dagster-io/dagster/tree/master/python_modules/libraries/dagster-dlt",
+    classifiers=[
+        "Programming Language :: Python :: 3.9",
+        "Programming Language :: Python :: 3.10",
+        "Programming Language :: Python :: 3.11",
+        "Programming Language :: Python :: 3.12",
+        "License :: OSI Approved :: Apache Software License",
+        "Operating System :: OS Independent",
+    ],
+    packages=find_packages(exclude=["dagster_dlt_tests*"]),
+    include_package_data=True,
+    python_requires=">=3.9,<3.13",
+    install_requires=["dagster==1.9.9", "dlt>=0.4"],
+    zip_safe=False,
+    extras_require={
+        "test": [
+            "duckdb",
+        ]
+    },
+)