dagster-census 0.14.21rc0__py3-none-any.whl → 0.28.10__py3-none-any.whl

dagster_census/__init__.py

@@ -1,15 +1,15 @@
-from dagster.core.utils import check_dagster_package_version
+from dagster_shared.libraries import DagsterLibraryRegistry
 
-from .ops import census_trigger_sync_op
-from .resources import CensusResource, census_resource
-from .types import CensusOutput
-from .version import __version__
+from dagster_census.components.census_component import CensusComponent as CensusComponent
+from dagster_census.ops import census_trigger_sync_op
+from dagster_census.resources import CensusResource
+from dagster_census.types import CensusOutput
+from dagster_census.version import __version__
 
-check_dagster_package_version("dagster-census", __version__)
+DagsterLibraryRegistry.register("dagster-census", __version__)
 
 __all__ = [
-    "CensusResource",
     "CensusOutput",
-    "census_resource",
+    "CensusResource",
     "census_trigger_sync_op",
 ]

dagster_census/components/census_component.py

@@ -0,0 +1,219 @@
+from collections.abc import Callable, Iterable, Sequence
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Annotated, Optional, Union
+
+import dagster as dg
+import pydantic
+from dagster._annotations import public
+from dagster._core.definitions.metadata import TableMetadataSet
+from dagster._utils.names import clean_name
+from dagster.components.component.state_backed_component import StateBackedComponent
+from dagster.components.resolved.base import resolve_fields
+from dagster.components.utils.defs_state import (
+    DefsStateConfig,
+    DefsStateConfigArgs,
+    ResolvedDefsStateConfig,
+)
+from dagster_shared import check
+from dagster_shared.serdes.serdes import deserialize_value
+
+from dagster_census.components.census_scaffolder import CensusComponentScaffolder
+from dagster_census.resources import CensusResource
+from dagster_census.translator import (
+    CensusMetadataSet,
+    CensusSync,
+    CensusWorkspaceData,
+    generate_table_schema,
+)
+
+
+class CensusSyncSelectorByName(dg.Resolvable, dg.Model):
+    by_name: Annotated[
+        Sequence[str],
+        pydantic.Field(..., description="A list of sync names to include in the collection."),
+    ]
+
+
+class CensusSyncSelectorById(dg.Resolvable, dg.Model):
+    by_id: Annotated[
+        Sequence[Union[int, str]],  # Allow strings for use-cases like '{{ env.ENV_VAR }}'
+        pydantic.Field(..., description="A list of sync IDs to include in the collection."),
+    ]
+
+
+def resolve_sync_selector(
+    context: dg.ResolutionContext, model
+) -> Optional[Callable[[CensusSync], bool]]:
+    if isinstance(model, str):
+        resolved = context.resolve_value(model)
+        resolved = check.callable_param(resolved, "unknown")  # pyright: ignore[reportArgumentType]
+        return resolved
+    if isinstance(model, CensusSyncSelectorByName.model()):
+        resolved = resolve_fields(model, CensusSyncSelectorByName.model(), context)
+        return lambda sync: sync.name in resolved["by_name"]
+    elif isinstance(model, CensusSyncSelectorById.model()):
+        resolved = resolve_fields(model, CensusSyncSelectorById.model(), context)
+        return lambda sync: sync.id in resolved["by_id"]
+    else:
+        check.failed(f"Unknown sync target type: {type(model)}")
+
+
+@dataclass
+class CensusResourceArgs(dg.Resolvable, dg.Model):
+    """The fields are analogous to the fields at `CensusResource`."""
+
+    api_key: Annotated[str, pydantic.Field(...)]
+    request_max_retries: Annotated[int, pydantic.Field(None)]
+    request_retry_delay: Annotated[float, pydantic.Field(None)]
+    request_timeout: Annotated[int, pydantic.Field(None)]
+
+
+def resolve_census_workspace(
+    context: dg.ResolutionContext, model: CensusResourceArgs
+) -> CensusResource:
+    return CensusResource(**resolve_fields(model, CensusResourceArgs, context))
+
+
+@public
+@dataclass
+@dg.scaffold_with(CensusComponentScaffolder)
+class CensusComponent(StateBackedComponent, dg.Resolvable):
+    """Loads Census syncs from a Census workspace as Dagster assets.
+    Materializing these assets will trigger the Census sync, enabling
+    you to schedule Census syncs using Dagster.
+
+    Example:
+
+        .. code-block:: yaml
+
+            # defs.yaml
+
+            type: dagster_census.CensusComponent
+            attributes:
+              workspace:
+                api_key: "{{ env.CENSUS_API_KEY }}"
+              sync_selector:
+                by_name:
+                  - my_first_sync
+                  - my_second_sync
+    """
+
+    workspace: Annotated[
+        CensusResource,
+        dg.Resolver(resolve_census_workspace, model_field_type=CensusResourceArgs),
+    ]
+
+    sync_selector: Annotated[
+        Optional[Callable[[CensusSync], bool]],
+        dg.Resolver(
+            resolve_sync_selector,
+            model_field_type=Union[
+                str, CensusSyncSelectorByName.model(), CensusSyncSelectorById.model()
+            ],
+            description="Function used to select Census syncs to pull into Dagster.",
+        ),
+    ] = None
+
+    defs_state: ResolvedDefsStateConfig = field(
+        default_factory=DefsStateConfigArgs.local_filesystem
+    )
+
+    @property
+    def defs_state_config(self) -> DefsStateConfig:
+        default_key = f"{self.__class__.__name__}"
+        return DefsStateConfig.from_args(self.defs_state, default_key=default_key)
+
+    def write_state_to_path(self, state_path: Path) -> None:
+        state = self.workspace.fetch_census_workspace_data()
+        state_path.write_text(dg.serialize_value(state))
+
+    @public
+    def get_asset_spec(self, sync: CensusSync) -> dg.AssetSpec:
+        metadata = {
+            **TableMetadataSet(
+                column_schema=generate_table_schema(sync.mappings),
+                table_name=sync.name,
+            ),
+            **CensusMetadataSet(
+                sync_id=sync.id,
+                sync_name=sync.name,
+                source_id=sync.source_id,
+                destination_id=sync.destination_id,
+            ),
+        }
+
+        return dg.AssetSpec(
+            key=dg.AssetKey(clean_name(sync.name)),
+            metadata=metadata,
+            description=f"Asset generated from Census sync {sync.id}",
+            kinds={"census"},
+        )
+
+    @public
+    def execute(
+        self, context: dg.AssetExecutionContext, census: CensusResource
+    ) -> Iterable[Union[dg.AssetMaterialization, dg.MaterializeResult]]:
+        """Executes a Census sync for the selected sync.
+
+        This method can be overridden in a subclass to customize the sync execution behavior,
+        such as adding custom logging or handling sync results differently.
+
+        Args:
+            context: The asset execution context provided by Dagster
+            census: The CensusResource used to trigger and monitor syncs
+
+        Returns:
+            MaterializeResult event from the Census sync
+
+        Example:
+            Override this method to add custom logging during sync execution:
+
+            .. code-block:: python
+
+                from dagster_census import CensusComponent
+                import dagster as dg
+
+                class CustomCensusComponent(CensusComponent):
+                    def execute(self, context, census):
+                        context.log.info(f"Starting Census sync for {context.asset_key}")
+                        result = super().execute(context, census)
+                        context.log.info("Census sync completed successfully")
+                        return result
+        """
+        # Select the first asset-spec (since there is only one)
+        spec = next(iter(context.assets_def.specs))
+
+        census_metadataset = CensusMetadataSet.extract(spec.metadata)
+        census.trigger_sync_and_poll(sync_id=census_metadataset.sync_id)
+        yield dg.AssetMaterialization(asset_key=clean_name(census_metadataset.sync_name))
+
+    def _load_asset_specs(self, state: CensusWorkspaceData) -> Sequence[dg.AssetSpec]:
+        connection_selector_fn = self.sync_selector or (lambda sync: True)
+        return [self.get_asset_spec(sync) for sync in state.syncs if connection_selector_fn(sync)]
+
+    def _get_census_assets_def(self, sync_name: str, spec: dg.AssetSpec) -> dg.AssetsDefinition:
+        @dg.multi_asset(
+            name=f"census_{clean_name(sync_name)}",
+            can_subset=True,
+            specs=[spec],
+        )
+        def _asset(context: dg.AssetExecutionContext):
+            yield from self.execute(context, self.workspace)
+
+        return _asset
+
+    def build_defs_from_state(
+        self, context: dg.ComponentLoadContext, state_path: Optional[Path]
+    ) -> dg.Definitions:
+        if state_path is None:
+            return dg.Definitions()
+
+        state = deserialize_value(state_path.read_text(), CensusWorkspaceData)
+
+        assets = [
+            self._get_census_assets_def(CensusMetadataSet.extract(spec.metadata).sync_name, spec)
+            for spec in self._load_asset_specs(state)
+        ]
+
+        return dg.Definitions(assets=assets)

dagster_census/components/census_scaffolder.py

@@ -0,0 +1,26 @@
+from typing import Optional
+
+from dagster.components.component.component_scaffolder import Scaffolder
+from dagster.components.component_scaffolding import scaffold_component
+from dagster.components.scaffold.scaffold import ScaffoldRequest
+from pydantic import BaseModel
+
+
+class CensusScaffolderParams(BaseModel):
+    api_key: Optional[str] = None
+
+
+class CensusComponentScaffolder(Scaffolder[CensusScaffolderParams]):
+    @classmethod
+    def get_scaffold_params(cls) -> type[CensusScaffolderParams]:
+        return CensusScaffolderParams
+
+    def scaffold(self, request: ScaffoldRequest[CensusScaffolderParams]) -> None:
+        scaffold_component(
+            request,
+            {
+                "workspace": {
+                    "api_key": "{{ env.CENSUS_API_KEY }}",
+                }
+            },
+        )

dagster_census/ops.py

@@ -1,8 +1,9 @@
 from dagster import Array, Bool, Field, In, Noneable, Nothing, Out, Output, op
+from dagster._core.storage.tags import COMPUTE_KIND_TAG
 
-from .resources import DEFAULT_POLL_INTERVAL
-from .types import CensusOutput
-from .utils import generate_materialization
+from dagster_census.resources import DEFAULT_POLL_INTERVAL
+from dagster_census.types import CensusOutput
+from dagster_census.utils import generate_materialization
 
 
 @op(
@@ -10,31 +11,37 @@ from .utils import generate_materialization
     ins={"start_after": In(Nothing)},
     out=Out(
         CensusOutput,
-        description="Parsed json dictionary representing the details of the Census sync after "
-        "the sync successfully completes.",
+        description=(
+            "Parsed json dictionary representing the details of the Census sync after "
+            "the sync successfully completes."
+        ),
     ),
     config_schema={
         "sync_id": Field(
             int,
             is_required=True,
-            description="id of the parent sync.",
+            description="Id of the parent sync.",
         ),
         "force_full_sync": Field(
             config=Bool,
             default_value=False,
-            description="If this trigger request should be a Full Sync. "
-            "Note that some sync configurations such as Append do not support full syncs.",
+            description=(
+                "If this trigger request should be a Full Sync. "
+                "Note that some sync configurations such as Append do not support full syncs."
+            ),
         ),
         "poll_interval": Field(
             float,
             default_value=DEFAULT_POLL_INTERVAL,
-            description="The time (in seconds) that will be waited between successive polls.",
+            description="The time (in seconds) to wait between successive polls.",
         ),
         "poll_timeout": Field(
             Noneable(float),
             default_value=None,
-            description="The maximum time that will waited before this operation is timed out. By "
-            "default, this will never time out.",
+            description=(
+                "The maximum time to wait before this operation is timed out. By "
+                "default, this will never time out."
+            ),
         ),
         "yield_materializations": Field(
             config=Bool,
@@ -53,19 +60,19 @@ from .utils import generate_materialization
             ),
         ),
     },
-    tags={"kind": "census"},
+    tags={COMPUTE_KIND_TAG: "census"},
 )
 def census_trigger_sync_op(context):
-    """
-    Executes a Census sync for a given ``sync_id``, and polls until that sync
-    completes, raising an error if it is unsuccessful. It outputs a CensusOutput which contains
-    the details of the Census sync after the sync successfully completes, as well as details
-    about which tables the sync updates.
+    """Executes a Census sync for a given ``sync_id`` and polls until that sync completes, raising
+    an error if it is unsuccessful.
+
+    It outputs a :py:class:`~dagster_census.CensusOutput` which contains the details of the Census
+    sync after it successfully completes.
 
     It requires the use of the :py:class:`~dagster_census.census_resource`, which allows it to
-    communicate with the census API.
+    communicate with the Census API.
 
-    Examples:
+    **Examples:**
 
     .. code-block:: python
 

dagster_census/resources.py

@@ -2,54 +2,91 @@ import datetime
 import json
 import logging
 import time
-from typing import Any, Dict, Optional
+from collections.abc import Mapping
+from typing import Any, Optional
 
+import pydantic
 import requests
+from dagster import ConfigurableResource, Failure, __version__, get_dagster_logger
+from dagster_shared.utils.cached_method import cached_method
 from requests.auth import HTTPBasicAuth
 from requests.exceptions import RequestException
 
-from dagster import Failure, Field, StringSource, __version__, get_dagster_logger, resource
-
-from .types import CensusOutput
+from dagster_census.translator import CensusSync, CensusWorkspaceData
+from dagster_census.types import CensusOutput
 
 CENSUS_API_BASE = "app.getcensus.com/api"
 CENSUS_VERSION = "v1"
 
 DEFAULT_POLL_INTERVAL = 10
 
+SYNC_RUN_STATUSES = {"completed", "failed", "queued", "skipped", "working"}
 
-class CensusResource:
-    """
-    This class exposes methods on top of the Census REST API.
-    """
 
-    def __init__(
-        self,
-        api_key: str,
-        request_max_retries: int = 3,
-        request_retry_delay: float = 0.25,
-        log: logging.Logger = get_dagster_logger(),
-    ):
-        self.api_key = api_key
+class CensusResource(ConfigurableResource):
+    """This resource allows users to programatically interface with the Census REST API to launch
+    syncs and monitor their progress. This currently implements only a subset of the functionality
+    exposed by the API.
+
+    **Examples:**
 
-        self._request_max_retries = request_max_retries
-        self._request_retry_delay = request_retry_delay
+    .. code-block:: python
 
-        self._log = log
+        import dagster as dg
+        from dagster_census import CensusResource
 
-    @property
-    def _api_key(self):
-        if self.api_key.startswith("secret-token:"):
-            return self.api_key
-        return "secret-token:" + self.api_key
+        census_resource = CensusResource(
+            api_key=dg.EnvVar("CENSUS_API_KEY")
+        )
+
+        @dg.asset
+        def census_sync_asset(census: CensusResource):
+            census.trigger_sync_and_poll(sync_id=123456)
+
+        defs = dg.Definitions(
+            assets=[census_sync_asset],
+            resources={"census": census_resource}
+        )
+    """
+
+    api_key: str = pydantic.Field(..., description="The Census API key")
+    request_max_retries: int = pydantic.Field(
+        default=3,
+        description=(
+            "The maximum number of times requests to the Census API should be retried "
+            "before failing."
+        ),
+    )
+    request_retry_delay: float = pydantic.Field(
+        default=0.25,
+        description="Time (in seconds) to wait between each request retry.",
+    )
+    request_timeout: int = pydantic.Field(
+        default=15,
+        description="Time (in seconds) after which the requests to Census are declared timed out.",
+    )
 
     @property
     def api_base_url(self) -> str:
         return f"https://{CENSUS_API_BASE}/{CENSUS_VERSION}"
 
-    def make_request(self, method: str, endpoint: str, data: str = None) -> Dict[str, Any]:
-        """
-        Creates and sends a request to the desired Census API endpoint.
+    @classmethod
+    def _is_dagster_maintained(cls) -> bool:
+        return True
+
+    @property
+    @cached_method
+    def _log(self) -> logging.Logger:
+        return get_dagster_logger()
+
+    def make_request(
+        self,
+        method: str,
+        endpoint: str,
+        data: Optional[str] = None,
+        page_number: Optional[int] = None,
+    ) -> Mapping[str, Any]:
+        """Creates and sends a request to the desired Census API endpoint.
 
         Args:
             method (str): The http method to use for this request (e.g. "POST", "GET", "PATCH").
@@ -59,35 +96,41 @@ class CensusResource:
         Returns:
             Dict[str, Any]: JSON data from the response to this request
         """
+        url = f"{self.api_base_url}/{endpoint}"
         headers = {
             "User-Agent": f"dagster-census/{__version__}",
             "Content-Type": "application/json;version=2",
         }
 
+        if page_number is not None:
+            params = {"page": page_number}
+        else:
+            params = {}
+
         num_retries = 0
         while True:
             try:
                 response = requests.request(
                     method=method,
-                    url=f"{self.api_base_url}/{endpoint}",
+                    url=url,
                     headers=headers,
-                    auth=HTTPBasicAuth("bearer", self._api_key),
+                    auth=HTTPBasicAuth("bearer", self.api_key),
                     data=data,
+                    params=params,
                 )
                 response.raise_for_status()
                 return response.json()
             except RequestException as e:
                 self._log.error("Request to Census API failed: %s", e)
-                if num_retries == self._request_max_retries:
+                if num_retries == self.request_max_retries:
                     break
                 num_retries += 1
-                time.sleep(self._request_retry_delay)
+                time.sleep(self.request_retry_delay)
 
-        raise Failure("Exceeded max number of retries.")
+        raise Failure(f"Max retries ({self.request_max_retries}) exceeded with url: {url}.")
 
-    def get_sync(self, sync_id: int) -> Dict[str, Any]:
-        """
-        Gets details about a given sync from the Census API.
+    def get_sync(self, sync_id: int) -> Mapping[str, Any]:
+        """Gets details about a given sync from the Census API.
 
         Args:
             sync_id (int): The Census Sync ID.
@@ -97,9 +140,8 @@ class CensusResource:
         """
         return self.make_request(method="GET", endpoint=f"syncs/{sync_id}")
 
-    def get_source(self, source_id: int) -> Dict[str, Any]:
-        """
-        Gets details about a given source from the Census API.
+    def get_source(self, source_id: int) -> Mapping[str, Any]:
+        """Gets details about a given source from the Census API.
 
         Args:
             source_id (int): The Census Source ID.
@@ -109,9 +151,8 @@ class CensusResource:
         """
         return self.make_request(method="GET", endpoint=f"sources/{source_id}")
 
-    def get_destination(self, destination_id: int) -> Dict[str, Any]:
-        """
-        Gets details about a given destination from the Census API.
+    def get_destination(self, destination_id: int) -> Mapping[str, Any]:
+        """Gets details about a given destination from the Census API.
 
         Args:
             destination_id (int): The Census Destination ID.
@@ -121,9 +162,8 @@ class CensusResource:
         """
         return self.make_request(method="GET", endpoint=f"destinations/{destination_id}")
 
-    def get_sync_run(self, sync_run_id: int) -> Dict[str, Any]:
-        """
-        Gets details about a specific sync run from the Census API.
+    def get_sync_run(self, sync_run_id: int) -> Mapping[str, Any]:
+        """Gets details about a specific sync run from the Census API.
 
         Args:
             sync_run_id (int): The Census Sync Run ID.
@@ -138,9 +178,8 @@ class CensusResource:
         sync_run_id: int,
         poll_interval: float = DEFAULT_POLL_INTERVAL,
         poll_timeout: Optional[float] = None,
-    ) -> Dict[str, Any]:
-        """
-        Given a Census sync run, poll until the run is complete
+    ) -> Mapping[str, Any]:
+        """Given a Census sync run, poll until the run is complete.
 
         Args:
             sync_id (int): The Census Sync Run ID.
@@ -159,12 +198,22 @@ class CensusResource:
             response_dict = self.get_sync_run(sync_run_id)
             if "data" not in response_dict.keys():
                 raise ValueError(
-                    f"Getting status of sync failed, please visit Census Logs at {log_url} to see more."
+                    f"Getting status of sync failed, please visit Census Logs at {log_url} to see"
+                    " more."
                 )
 
+            sync_status = response_dict["data"]["status"]
             sync_id = response_dict["data"]["sync_id"]
 
-            if response_dict["status"] == "working":
+            if sync_status not in SYNC_RUN_STATUSES:
+                raise ValueError(
+                    f"Unexpected response status '{sync_status}'; "
+                    f"must be one of {','.join(sorted(SYNC_RUN_STATUSES))}. "
+                    "See Management API docs for more information: "
+                    "https://docs.getcensus.com/basics/developers/api/sync-runs"
+                )
+
+            if sync_status in {"queued", "working"}:
                 self._log.debug(
                     f"Sync {sync_id} still running after {datetime.datetime.now() - poll_start}."
                 )
@@ -174,7 +223,8 @@ class CensusResource:
                 seconds=poll_timeout
             ):
                 raise Failure(
-                    f"Sync for sync '{sync_id}' timed out after {datetime.datetime.now() - poll_start}."
+                    f"Sync for sync '{sync_id}' timed out after"
+                    f" {datetime.datetime.now() - poll_start}."
                 )
 
             break
@@ -186,9 +236,8 @@ class CensusResource:
 
         return response_dict
 
-    def trigger_sync(self, sync_id: int, force_full_sync: bool = False) -> Dict[str, Any]:
-        """
-        Trigger an asynchronous run for a specific sync.
+    def trigger_sync(self, sync_id: int, force_full_sync: bool = False) -> Mapping[str, Any]:
+        """Trigger an asynchronous run for a specific sync.
 
         Args:
             sync_id (int): The Census Sync Run ID.
@@ -209,8 +258,7 @@ class CensusResource:
         poll_interval: float = DEFAULT_POLL_INTERVAL,
         poll_timeout: Optional[float] = None,
     ) -> CensusOutput:
-        """
-        Trigger a run for a specific sync and poll until it has completed
+        """Trigger a run for a specific sync and poll until it has completed.
 
         Args:
             sync_id (int): The Census Sync Run ID.
@@ -243,55 +291,32 @@ class CensusResource:
             destination=destination_details,
         )
 
+    @cached_method
+    def fetch_census_workspace_data(self) -> CensusWorkspaceData:
+        """Retrieves all Census syncs from the workspace and returns it as a CensusWorkspaceData object.
 
-@resource(
-    config_schema={
-        "api_key": Field(
-            StringSource,
-            is_required=True,
-            description="Census API Key.",
-        ),
-        "request_max_retries": Field(
-            int,
-            default_value=3,
-            description="The maximum number of times requests to the Census API should be retried "
-            "before failing.",
-        ),
-        "request_retry_delay": Field(
-            float,
-            default_value=0.25,
-            description="Time (in seconds) to wait between each request retry.",
-        ),
-    },
-    description="This resource helps manage Census connectors",
-)
-def census_resource(context) -> CensusResource:
-    """
-    This resource allows users to programatically interface with the Census REST API to launch
-    syncs and monitor their progress. This currently implements only a subset of the functionality
-    exposed by the API.
-
-    **Examples:**
-
-    .. code-block:: python
-
-        from dagster import job
-        from dagster_census import census_resource
-
-        my_census_resource = census_resource.configured(
-            {
-                "api_key": {"env": "CENSUS_API_KEY"},
-            }
+        Returns:
+            CensusWorkspaceData: A snapshot of the Census workspace's syncs.
+        """
+        all_syncs = []
+        page = self.make_request(method="GET", endpoint="syncs")
+
+        last_page_number = page["pagination"]["last_page"]
+        for sync in page["data"]:
+            all_syncs.append(self._census_sync_struct_from_json(sync))
+
+        for i in range(2, last_page_number + 1):
+            page = self.make_request(method="GET", endpoint="syncs", page_number=i)
+            for sync in page["data"]:
+                all_syncs.append(self._census_sync_struct_from_json(sync))
+
+        return CensusWorkspaceData(syncs=all_syncs)
+
+    def _census_sync_struct_from_json(self, sync: dict[str, Any]) -> CensusSync:
+        return CensusSync(
+            id=sync["id"],
+            name=sync["label"] or sync["resource_identifier"],
+            source_id=sync["source_attributes"]["connection_id"],
+            destination_id=sync["destination_attributes"]["connection_id"],
+            mappings=sync["mappings"],
         )
-
-        @job(resource_defs={"census":my_census_resource})
-        def my_census_job():
-            ...
-
-    """
-    return CensusResource(
-        api_key=context.resource_config["api_key"],
-        request_max_retries=context.resource_config["request_max_retries"],
-        request_retry_delay=context.resource_config["request_retry_delay"],
-        log=context.log,
-    )

dagster_census/translator.py

@@ -0,0 +1,57 @@
+from collections.abc import Mapping
+from typing import Any
+
+import dagster as dg
+from dagster._core.definitions.metadata.metadata_set import NamespacedMetadataSet
+from dagster._record import record
+from dagster_shared.serdes import whitelist_for_serdes
+
+
+@whitelist_for_serdes
+@record
+class CensusSync:
+    """Represents a Census sync, based on data as returned from the API."""
+
+    id: int
+    name: str
+    source_id: int
+    destination_id: int
+    mappings: list[dict[str, Any]]
+
+
+@whitelist_for_serdes
+@record
+class CensusWorkspaceData:
+    """A record representing all content in a Census workspace.
+    Provided as context for the translator so that it can resolve dependencies between content.
+    """
+
+    syncs: list[CensusSync]
+
+    @property
+    def syncs_by_id(self) -> Mapping[int, CensusSync]:
+        """Returns a mapping of sync IDs to CensusSync objects."""
+        return {sync.id: sync for sync in self.syncs}
+
+
+def generate_table_schema(sync_mappings_props: list[dict[str, Any]]) -> dg.TableSchema:
+    return dg.TableSchema(
+        columns=sorted(
+            [
+                dg.TableColumn(name=mapping["to"], type=mapping.get("field_type", "unknown"))
+                for mapping in sync_mappings_props
+            ],
+            key=lambda col: col.name,
+        )
+    )
+
+
+class CensusMetadataSet(NamespacedMetadataSet):
+    sync_id: int
+    sync_name: str
+    source_id: int
+    destination_id: int
+
+    @classmethod
+    def namespace(cls) -> str:
+        return "dagster-census"

dagster_census/types.py

@@ -1,24 +1,24 @@
-from typing import Any, Dict, NamedTuple
+from collections.abc import Mapping
+from typing import Any, NamedTuple
 
 
 class CensusOutput(
     NamedTuple(
         "_CensusOutput",
         [
-            ("sync_run", Dict[str, Any]),
-            ("source", Dict[str, Any]),
-            ("destination", Dict[str, Any]),
+            ("sync_run", Mapping[str, Any]),
+            ("source", Mapping[str, Any]),
+            ("destination", Mapping[str, Any]),
         ],
     )
 ):
-    """
-    Contains recorded information about the state of a Census sync after a sync completes.
+    """Contains recorded information about the state of a Census sync after a sync completes.
 
-    Attributes:
+    Args:
         sync_run (Dict[str, Any]):
-            The details of the specific sync run
+            The details of the specific sync run.
         source (Dict[str, Any]):
-            Information about the source for the Census sync
+            Information about the source for the Census sync.
         destination (Dict[str, Any]):
-            Information about the destination for the Census sync
+            Information about the destination for the Census sync.
     """

dagster_census/utils.py

@@ -1,14 +1,13 @@
-from typing import List
+from collections.abc import Sequence
 
 from dagster import AssetMaterialization
 
-from .types import CensusOutput
+from dagster_census.types import CensusOutput
 
 
 def generate_materialization(
-    census_output: CensusOutput, asset_key_prefix: List[str]
+    census_output: CensusOutput, asset_key_prefix: Sequence[str]
 ) -> AssetMaterialization:
-
     sync_id = census_output.sync_run["sync_id"]
     source_name = census_output.source["name"]
     destination_name = census_output.destination["name"]
@@ -27,10 +26,12 @@ def generate_materialization(
         "full_sync": census_output.sync_run["full_sync"],
     }
 
-    asset_key_name = asset_key_prefix + [f"sync_{sync_id}"]
+    asset_key_name = [*asset_key_prefix, f"sync_{sync_id}"]
 
     return AssetMaterialization(
         asset_key=asset_key_name,
-        description=f"Sync generated for Census sync {sync_id}: {source_name} to {destination_name}",
+        description=(
+            f"Sync generated for Census sync {sync_id}: {source_name} to {destination_name}"
+        ),
         metadata=metadata,
     )

dagster_census/version.py

@@ -1 +1 @@
-__version__ = "0.14.21rc0"
+__version__ = "0.28.10"

dagster_census-0.28.10.dist-info/METADATA

@@ -0,0 +1,27 @@
+Metadata-Version: 2.4
+Name: dagster-census
+Version: 0.28.10
+Summary: Package for integrating Census with Dagster.
+Home-page: https://github.com/dagster-io/dagster/tree/master/python_modules/libraries/dagster-census
+Author: Dagster Labs
+Author-email: hello@dagsterlabs.com
+License: Apache-2.0
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Programming Language :: Python :: 3.14
+Classifier: License :: OSI Approved :: Apache Software License
+Classifier: Operating System :: OS Independent
+Requires-Python: >=3.10,<3.15
+License-File: LICENSE
+Requires-Dist: dagster==1.12.10
+Dynamic: author
+Dynamic: author-email
+Dynamic: classifier
+Dynamic: home-page
+Dynamic: license
+Dynamic: license-file
+Dynamic: requires-dist
+Dynamic: requires-python
+Dynamic: summary

dagster_census-0.28.10.dist-info/RECORD

@@ -0,0 +1,16 @@
+dagster_census/__init__.py,sha256=eIDWjz13trVGLVMDjAdq0LeUkbEiWex1ho2NX8cDFOY,501
+dagster_census/ops.py,sha256=4N53-w0u3PG5kn-q1tW-7Ac-i5SCDFq9ku--g7gDig4,3659
+dagster_census/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+dagster_census/resources.py,sha256=R-McSQLQTOImDVAQ3Tr-ZbvbUMQnbKpW7hkvtuwtS7I,11661
+dagster_census/translator.py,sha256=caSshQ3ZfSsl_ASNlN6p24t-15WDRN29Z1xBg7Utbp4,1510
+dagster_census/types.py,sha256=on1CIKWZMsSuw5h-HkzQPVctMQzQu2L9HgkX5gc9X54,698
+dagster_census/utils.py,sha256=fFsKjn6xlLEel1-nvAwfAaemjSTg2gYxJHY8sK2e1no,1348
+dagster_census/version.py,sha256=110BPTEyn13PDWnHpj0D7VMBpjnNxJ6-MM3ZHvQwyQ0,24
+dagster_census/components/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+dagster_census/components/census_component.py,sha256=vgaVfT0pFVoC8ddhJJQEGJwjfZGsjLiTlXDOeNYW4q0,7970
+dagster_census/components/census_scaffolder.py,sha256=n0YT0Rov6yJsPHAVpbtUrkxxF59z0WLCnRhYEpYtF4A,818
+dagster_census-0.28.10.dist-info/licenses/LICENSE,sha256=4lsMW-RCvfVD4_F57wrmpe3vX1xwUk_OAKKmV_XT7Z0,11348
+dagster_census-0.28.10.dist-info/METADATA,sha256=c2aYCvD4kTlMACHTjkTvgz9klKUJ0mIcPAQ80JGpAho,922
+dagster_census-0.28.10.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+dagster_census-0.28.10.dist-info/top_level.txt,sha256=Lz6T6gBn89ilLGOLr8YIrBTRcWPCjeOtdNk8sdqmUnA,15
+dagster_census-0.28.10.dist-info/RECORD,,

{dagster_census-0.14.21rc0.dist-info → dagster_census-0.28.10.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: bdist_wheel (0.33.6)
+Generator: setuptools (80.9.0)
 Root-Is-Purelib: true
 Tag: py3-none-any
 

dagster_census-0.28.10.dist-info/licenses/LICENSE

@@ -0,0 +1,201 @@
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "{}"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright 2025 Dagster Labs, Inc.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

dagster_census-0.14.21rc0.dist-info/METADATA

@@ -1,19 +0,0 @@
-Metadata-Version: 2.1
-Name: dagster-census
-Version: 0.14.21rc0
-Summary: Package for integrating Census with Dagster.
-Home-page: https://github.com/dagster-io/dagster/tree/master/python_modules/libraries/dagster-census
-Author: Elementl
-Author-email: hello@elementl.com
-License: Apache-2.0
-Platform: UNKNOWN
-Classifier: Programming Language :: Python :: 3.6
-Classifier: Programming Language :: Python :: 3.7
-Classifier: Programming Language :: Python :: 3.8
-Classifier: Programming Language :: Python :: 3.9
-Classifier: License :: OSI Approved :: Apache Software License
-Classifier: Operating System :: OS Independent
-Requires-Dist: dagster (==0.14.21rc0)
-
-UNKNOWN
-

dagster_census-0.14.21rc0.dist-info/RECORD

@@ -1,10 +0,0 @@
-dagster_census/__init__.py,sha256=10sSerK0e8Fu0Qyj_h4NgNKLuBtfl4GRF9jfNJ19X20,394
-dagster_census/ops.py,sha256=qhstHKuMOC4McS1R8lWXZEUbBojbb29ZDHOqiLSTf1o,3507
-dagster_census/resources.py,sha256=BFoKEGuvFyDSqzp37uIh-ITOPDIRkqvwUxlVUxNIGYs,10046
-dagster_census/types.py,sha256=OrhzeBYHLv8K7zvXb_LgeD46UYEjNizWZxWvUV2dh_I,667
-dagster_census/utils.py,sha256=cG7mA4JspbNbsOw681WnexZmcRkvo56vwXF1_MJfUIM,1294
-dagster_census/version.py,sha256=ysovRfcGlcNUPhgQ2ub-CVWFIVbCbkN8tZMyhUkCct0,27
-dagster_census-0.14.21rc0.dist-info/METADATA,sha256=skhOGZ2WdhQF27lB1TbqJXpNhc-sKrXwd-RbYMbJYiQ,664
-dagster_census-0.14.21rc0.dist-info/WHEEL,sha256=p46_5Uhzqz6AzeSosiOnxK-zmFja1i22CrQCjmYe8ec,92
-dagster_census-0.14.21rc0.dist-info/top_level.txt,sha256=Lz6T6gBn89ilLGOLr8YIrBTRcWPCjeOtdNk8sdqmUnA,15
-dagster_census-0.14.21rc0.dist-info/RECORD,,