dagster-polytomic 0.28.12__tar.gz

dagster_polytomic-0.28.12/LICENSE

@@ -0,0 +1,201 @@
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "{}"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright 2025 Dagster Labs, Inc.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

dagster_polytomic-0.28.12/MANIFEST.in

@@ -0,0 +1,3 @@
+include README.md
+include LICENSE
+include dagster_polytomic/py.typed

dagster_polytomic-0.28.12/PKG-INFO

@@ -0,0 +1,28 @@
+Metadata-Version: 2.4
+Name: dagster_polytomic
+Version: 0.28.12
+Summary: Package for integrating Polytomic with Dagster.
+Home-page: https://github.com/dagster-io/dagster/tree/master/python_modules/libraries/dagster-polytomic
+Author: Dagster Labs
+Author-email: hello@dagsterlabs.com
+License: Apache-2.0
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Programming Language :: Python :: 3.14
+Classifier: License :: OSI Approved :: Apache Software License
+Classifier: Operating System :: OS Independent
+Requires-Python: >=3.10,<3.15
+License-File: LICENSE
+Requires-Dist: dagster==1.12.12
+Requires-Dist: polytomic>=1.11.3
+Dynamic: author
+Dynamic: author-email
+Dynamic: classifier
+Dynamic: home-page
+Dynamic: license
+Dynamic: license-file
+Dynamic: requires-dist
+Dynamic: requires-python
+Dynamic: summary

dagster_polytomic-0.28.12/README.md

@@ -0,0 +1,4 @@
+# dagster-polytomic
+
+The docs for `dagster-polytomic` can be found
+[here](https://docs.dagster.io/integrations/libraries/polytomic/dagster-polytomic).

dagster_polytomic-0.28.12/dagster_polytomic/__init__.py

@@ -0,0 +1,6 @@
+from dagster_shared.libraries import DagsterLibraryRegistry
+
+from dagster_polytomic.component import PolytomicComponent as PolytomicComponent
+from dagster_polytomic.version import __version__ as __version__
+
+DagsterLibraryRegistry.register("dagster-polytomic", __version__)

dagster_polytomic-0.28.12/dagster_polytomic/component.py

@@ -0,0 +1,134 @@
+import itertools
+from collections import defaultdict
+from pathlib import Path
+from typing import Optional
+
+import dagster as dg
+from dagster._annotations import preview
+from dagster._core.definitions.metadata.metadata_set import TableMetadataSet
+from dagster._core.errors import DagsterInvalidDefinitionError
+from dagster.components.component.state_backed_component import StateBackedComponent
+from dagster.components.utils.defs_state import (
+    DefsStateConfig,
+    DefsStateConfigArgs,
+    ResolvedDefsStateConfig,
+)
+from pydantic import Field
+
+from dagster_polytomic.objects import PolytomicBulkSyncEnrichedSchema, PolytomicWorkspaceData
+from dagster_polytomic.translation import (
+    _TRANSLATOR_DATA_METADATA_KEY,
+    PolytomicSchemaMetadataSet,
+    PolytomicTranslatorData,
+    ResolvedPolytomicTranslationFn,
+)
+from dagster_polytomic.workspace import PolytomicWorkspace
+
+
+@preview
+class PolytomicComponent(StateBackedComponent, dg.Model, dg.Resolvable):
+    workspace: PolytomicWorkspace = Field(
+        description="Defines configuration for interacting with a Polytomic organization.",
+    )
+    translation: Optional[ResolvedPolytomicTranslationFn] = Field(
+        default=None,
+        description="Defines how to translate a Polytomic object into an AssetSpec object.",
+    )
+    defs_state: ResolvedDefsStateConfig = DefsStateConfigArgs.local_filesystem()
+
+    @property
+    def defs_state_config(self) -> DefsStateConfig:
+        default_key = f"{self.__class__.__name__}[{self.workspace.identity.organization_name}]"
+        return DefsStateConfig.from_args(self.defs_state, default_key=default_key)
+
+    async def write_state_to_path(self, state_path: Path) -> None:
+        """Fetch documents from Polytomic API and write state to path."""
+        state = await self.workspace.fetch_polytomic_state()
+        state_path.write_text(dg.serialize_value(state))
+
+    def load_state_from_path(self, state_path: Path) -> PolytomicWorkspaceData:
+        """Load state from path using Dagster's deserialization system."""
+        return dg.deserialize_value(state_path.read_text(), PolytomicWorkspaceData)
+
+    def _get_default_polytomic_spec(self, data: PolytomicTranslatorData) -> Optional[dg.AssetSpec]:
+        """Core function for converting a Polytomic schema into an AssetSpec object."""
+        if isinstance(data.obj, PolytomicBulkSyncEnrichedSchema):
+            enriched_schema = data.obj
+            return dg.AssetSpec(
+                key=dg.AssetKey(
+                    [enriched_schema.destination_configuration_schema or "", enriched_schema.id]
+                ),
+                group_name=enriched_schema.destination_configuration_schema.replace("-", "_")
+                if enriched_schema.destination_configuration_schema
+                else None,
+                metadata={
+                    _TRANSLATOR_DATA_METADATA_KEY: data,
+                    **PolytomicSchemaMetadataSet.from_enriched_schema(
+                        enriched_schema=enriched_schema
+                    ),
+                    **TableMetadataSet(
+                        table_name=f"{enriched_schema.destination_configuration_schema}.{enriched_schema.id}"
+                        if enriched_schema.destination_configuration_schema
+                        else enriched_schema.id
+                    ),
+                },
+                kinds={"polytomic"},
+            )
+        return None
+
+    def get_asset_spec(self, data: PolytomicTranslatorData) -> Optional[dg.AssetSpec]:
+        """Core function for converting a Polytomic object into an AssetSpec object."""
+        base_asset_spec = self._get_default_polytomic_spec(data)
+        if self.translation and base_asset_spec:
+            return self.translation(base_asset_spec, data)
+        else:
+            return base_asset_spec
+
+    def _build_asset_specs(self, workspace_data: PolytomicWorkspaceData) -> list[dg.AssetSpec]:
+        """Invokes the `get_asset_spec` method on all objects in the provided `workspace_data`.
+        Filters out any cases where the asset_spec is `None`, and provides a helpful error
+        message in cases where keys overlap between different documents.
+        """
+        maybe_specs = [
+            self.get_asset_spec(
+                PolytomicTranslatorData(obj=enriched_schema, workspace_data=workspace_data)
+            )
+            for enriched_schema in itertools.chain.from_iterable(
+                workspace_data.enriched_schemas_by_bulk_sync_id.values()
+            )
+        ]
+
+        specs_by_key: dict[dg.AssetKey, list[dg.AssetSpec]] = defaultdict(list)
+        for spec in filter(None, maybe_specs):
+            specs_by_key[spec.key].append(spec)
+
+        for key, specs in specs_by_key.items():
+            if len(specs) == 1:
+                continue
+
+            ids = [
+                PolytomicSchemaMetadataSet.extract(spec.metadata).bulk_sync_id or spec
+                for spec in specs
+            ]
+            ids_str = "\n\t".join(map(str, ids))
+            raise DagsterInvalidDefinitionError(
+                f"Multiple objects map to the same key {key}:"
+                f"\n\t{ids_str}\n"
+                "Please ensure that each object has a unique name by updating the `translation` function."
+            )
+
+        return list(itertools.chain.from_iterable(specs_by_key.values()))
+
+    def build_defs_from_workspace_data(
+        self, workspace_data: PolytomicWorkspaceData
+    ) -> dg.Definitions:
+        return dg.Definitions(assets=self._build_asset_specs(workspace_data))
+
+    def build_defs_from_state(
+        self, context: dg.ComponentLoadContext, state_path: Optional[Path]
+    ) -> dg.Definitions:
+        if state_path is None:
+            return dg.Definitions()
+
+        state = self.load_state_from_path(state_path)
+        return self.build_defs_from_workspace_data(state)

dagster_polytomic-0.28.12/dagster_polytomic/objects.py

@@ -0,0 +1,235 @@
+from collections import defaultdict
+from functools import cached_property
+from typing import Optional
+
+from dagster import _check as check
+from dagster_shared.record import record
+from dagster_shared.serdes import whitelist_for_serdes
+from polytomic import (
+    BulkField,
+    BulkSchema,
+    BulkSyncResponse,
+    ConnectionResponseSchema,
+    GetIdentityResponseSchema,
+)
+
+
+@whitelist_for_serdes
+@record
+class PolytomicIdentity:
+    """Represents a Polytomic identity."""
+
+    id: str
+    organization_id: str
+    organization_name: str
+
+    @classmethod
+    def from_api_response(cls, response: GetIdentityResponseSchema) -> "PolytomicIdentity":
+        """Create PolytomicIdentity from API response."""
+        return cls(
+            id=check.not_none(response.id, "Identity ID cannot be None"),
+            organization_id=check.not_none(
+                response.organization_id, "Organization ID cannot be None"
+            ),
+            organization_name=check.not_none(
+                response.organization_name, "Organization name cannot be None"
+            ),
+        )
+
+
+@whitelist_for_serdes
+@record
+class PolytomicConnection:
+    """Represents a Polytomic connection."""
+
+    id: str
+    name: Optional[str]
+    type: Optional[str]
+    organization_id: Optional[str]
+    status: Optional[str]
+
+    @classmethod
+    def from_api_response(cls, response: ConnectionResponseSchema) -> "PolytomicConnection":
+        """Create PolytomicConnection from API response."""
+        return cls(
+            id=check.not_none(response.id, "Connection ID cannot be None"),
+            name=response.name,
+            type=response.type.name if response.type else None,
+            organization_id=response.organization_id,
+            status=response.status,
+        )
+
+
+@whitelist_for_serdes
+@record
+class PolytomicBulkSync:
+    """Represents a Polytomic bulk sync."""
+
+    id: str
+    name: Optional[str]
+    active: bool
+    mode: Optional[str]
+    source_connection_id: Optional[str]
+    destination_connection_id: Optional[str]
+    organization_id: Optional[str]
+    destination_configuration_schema: Optional[str]
+
+    @classmethod
+    def from_api_response(cls, response: BulkSyncResponse) -> "PolytomicBulkSync":
+        """Create PolytomicBulkSync from API response."""
+        return cls(
+            id=check.not_none(response.id, "Bulk sync ID cannot be None"),
+            name=response.name,
+            active=response.active or False,
+            mode=response.mode,
+            source_connection_id=response.source_connection_id,
+            destination_connection_id=response.destination_connection_id,
+            organization_id=response.organization_id,
+            destination_configuration_schema=response.destination_configuration.get("schema")
+            if response.destination_configuration
+            else None,
+        )
+
+
+@whitelist_for_serdes
+@record
+class PolytomicBulkField:
+    """Represents a field in a Polytomic bulk schema."""
+
+    id: str
+    enabled: bool
+
+    @classmethod
+    def from_api_response(cls, response: BulkField) -> "PolytomicBulkField":
+        """Create PolytomicBulkField from API field data."""
+        return cls(
+            id=check.not_none(response.id, "Bulk field ID cannot be None"),
+            enabled=response.enabled or False,
+        )
+
+
+@whitelist_for_serdes
+@record
+class PolytomicBulkSyncSchema:
+    """Represents a schema in a Polytomic bulk sync."""
+
+    id: str
+    enabled: bool
+    output_name: Optional[str]
+    partition_key: Optional[str]
+    tracking_field: Optional[str]
+    fields: list[PolytomicBulkField]
+
+    @classmethod
+    def from_api_response(cls, response: BulkSchema) -> "PolytomicBulkSyncSchema":
+        """Create PolytomicBulkSyncSchema from API response."""
+        fields = []
+        if response.fields:
+            fields = [PolytomicBulkField.from_api_response(field) for field in response.fields]
+
+        return cls(
+            id=check.not_none(response.id, "Bulk sync schema ID cannot be None"),
+            enabled=response.enabled or False,
+            output_name=response.output_name,
+            partition_key=response.partition_key,
+            tracking_field=response.tracking_field,
+            fields=fields,
+        )
+
+
+@whitelist_for_serdes
+@record
+class PolytomicBulkSyncEnrichedSchema:
+    """Represents an enriched schema in a Polytomic bulk sync."""
+
+    id: str
+    bulk_sync_id: str
+    enabled: bool
+    output_name: Optional[str]
+    partition_key: Optional[str]
+    tracking_field: Optional[str]
+    fields: list[PolytomicBulkField]
+    destination_configuration_schema: Optional[str]
+    source_connection_id: Optional[str]
+    source_connection_name: Optional[str]
+    destination_connection_id: Optional[str]
+    destination_connection_name: Optional[str]
+
+
+@whitelist_for_serdes
+@record
+class PolytomicWorkspaceData:
+    """Serializable container object for recording the state of the Polytomic API at a given point in time.
+
+    Properties:
+        connections: list[PolytomicConnection]
+        bulk_syncs: list[PolytomicBulkSync]
+        schemas_by_bulk_sync_id: dict[str, list[PolytomicBulkSyncSchema]] - Mapping of bulk sync ID to its schemas
+    """
+
+    connections: list[PolytomicConnection]
+    bulk_syncs: list[PolytomicBulkSync]
+    schemas_by_bulk_sync_id: dict[str, list[PolytomicBulkSyncSchema]]
+
+    @cached_property
+    def enriched_schemas_by_bulk_sync_id(self) -> dict[str, list[PolytomicBulkSyncEnrichedSchema]]:
+        enriched_schemas_by_bulk_sync_id = defaultdict(list)
+        for bulk_sync_id, schemas in self.schemas_by_bulk_sync_id.items():
+            bulk_sync = self.get_bulk_sync(bulk_sync_id=bulk_sync_id)
+            source_connection = (
+                self.get_connection(connection_id=bulk_sync.source_connection_id)
+                if bulk_sync and bulk_sync.source_connection_id
+                else None
+            )
+            destination_connection = (
+                self.get_connection(connection_id=bulk_sync.destination_connection_id)
+                if bulk_sync and bulk_sync.destination_connection_id
+                else None
+            )
+
+            for schema in schemas:
+                enriched_schemas_by_bulk_sync_id[bulk_sync_id].append(
+                    PolytomicBulkSyncEnrichedSchema(
+                        id=schema.id,
+                        bulk_sync_id=bulk_sync_id,
+                        enabled=schema.enabled,
+                        output_name=schema.output_name,
+                        partition_key=schema.partition_key,
+                        tracking_field=schema.tracking_field,
+                        fields=schema.fields,
+                        destination_configuration_schema=bulk_sync.destination_configuration_schema
+                        if bulk_sync
+                        else None,
+                        source_connection_id=bulk_sync.source_connection_id if bulk_sync else None,
+                        source_connection_name=source_connection.name
+                        if source_connection
+                        else None,
+                        destination_connection_id=bulk_sync.destination_connection_id
+                        if bulk_sync
+                        else None,
+                        destination_connection_name=destination_connection.name
+                        if destination_connection
+                        else None,
+                    )
+                )
+        return enriched_schemas_by_bulk_sync_id
+
+    @cached_property
+    def _connections_by_id(self) -> dict[str, PolytomicConnection]:
+        return {connection.id: connection for connection in self.connections}
+
+    @cached_property
+    def _bulk_syncs_by_id(self) -> dict[str, PolytomicBulkSync]:
+        return {bulk_sync.id: bulk_sync for bulk_sync in self.bulk_syncs}
+
+    def get_connection(self, connection_id: str) -> Optional[PolytomicConnection]:
+        """Get a connection by ID."""
+        return self._connections_by_id.get(connection_id)
+
+    def get_bulk_sync(self, bulk_sync_id: str) -> Optional[PolytomicBulkSync]:
+        """Get a bulk sync by ID."""
+        return self._bulk_syncs_by_id.get(bulk_sync_id)
+
+    def get_bulk_sync_schemas(self, bulk_sync_id: str) -> list[PolytomicBulkSyncSchema]:
+        """Get schemas for a specific bulk sync."""
+        return self.schemas_by_bulk_sync_id.get(bulk_sync_id, [])

dagster_polytomic-0.28.12/dagster_polytomic/py.typed

@@ -0,0 +1 @@
+partial

dagster_polytomic-0.28.12/dagster_polytomic/translation.py

@@ -0,0 +1,126 @@
+from typing import Annotated, Optional, TypeAlias, Union
+
+from dagster._core.definitions.assets.definition.asset_spec import AssetSpec
+from dagster._core.definitions.metadata.metadata_set import NamespacedMetadataSet
+from dagster.components import Resolvable, Resolver
+from dagster.components.resolved.base import resolve_fields
+from dagster.components.resolved.context import ResolutionContext
+from dagster.components.resolved.core_models import AssetSpecUpdateKwargs
+from dagster.components.utils import TranslatorResolvingInfo
+from dagster.components.utils.translation import TranslationFn, TranslationFnResolver
+from dagster_shared.record import record
+from typing_extensions import Self
+
+from dagster_polytomic.objects import PolytomicBulkSyncEnrichedSchema, PolytomicWorkspaceData
+
+_TRANSLATOR_DATA_METADATA_KEY = ".dagster-polytomic/translator_data"
+
+
+class PolytomicSchemaMetadataSet(NamespacedMetadataSet):
+    """Represents metadata that is captured from a Polytomic schema."""
+
+    id: str
+    bulk_sync_id: str
+    output_name: Optional[str] = None
+    partition_key: Optional[str] = None
+    tracking_field: Optional[str] = None
+    source_connection_id: Optional[str] = None
+    source_connection_name: Optional[str] = None
+    destination_connection_id: Optional[str] = None
+    destination_connection_name: Optional[str] = None
+
+    @classmethod
+    def from_enriched_schema(cls, enriched_schema: PolytomicBulkSyncEnrichedSchema) -> Self:
+        return cls(
+            id=enriched_schema.id,
+            bulk_sync_id=enriched_schema.bulk_sync_id,
+            output_name=enriched_schema.output_name,
+            partition_key=enriched_schema.partition_key,
+            tracking_field=enriched_schema.tracking_field,
+            source_connection_id=enriched_schema.source_connection_id,
+            source_connection_name=enriched_schema.source_connection_name,
+            destination_connection_id=enriched_schema.destination_connection_id,
+            destination_connection_name=enriched_schema.destination_connection_name,
+        )
+
+    @classmethod
+    def namespace(cls) -> str:
+        return "dagster-polytomic"
+
+
+@record
+class PolytomicTranslatorData:
+    """Container class for data required to translate an object in an
+    Polytomic workspace into a Dagster definition.
+
+    Properties:
+        obj (PolytomicBulkSyncEnrichedSchema): The object to translate.
+        workspace_data (PolytomicWorkspaceData): Global workspace data.
+    """
+
+    obj: PolytomicBulkSyncEnrichedSchema
+    workspace_data: PolytomicWorkspaceData
+
+
+def _resolve_multilayer_translation(context: ResolutionContext, model):
+    """The Polytomic translation schema supports defining global transforms
+    as well as per-object-type transforms. This resolver composes the
+    per-object-type transforms with the global transforms.
+    """
+    info = TranslatorResolvingInfo(
+        asset_attributes=model,
+        resolution_context=context,
+        model_key="translation",
+    )
+
+    def _translation_fn(base_asset_spec: AssetSpec, data: PolytomicTranslatorData):
+        processed_spec = info.get_asset_spec(
+            base_asset_spec,
+            {
+                "data": data,
+                "spec": base_asset_spec,
+            },
+        )
+
+        nested_translation_fns = resolve_fields(
+            model=model,
+            resolved_cls=PolytomicTranslationArgs,
+            context=context.with_scope(
+                **{
+                    "data": data,
+                    "spec": processed_spec,
+                }
+            ),
+        )
+        for_schema = nested_translation_fns.get("for_schema")
+
+        if isinstance(data.obj, PolytomicBulkSyncEnrichedSchema) and for_schema:
+            return for_schema(processed_spec, data)
+
+        return processed_spec
+
+    return _translation_fn
+
+
+PolytomicTranslationFn: TypeAlias = TranslationFn[PolytomicTranslatorData]
+
+ResolvedTargetedPolytomicTranslationFn = Annotated[
+    PolytomicTranslationFn,
+    TranslationFnResolver[PolytomicTranslatorData](lambda data: {"data": data}),
+]
+
+
+@record
+class PolytomicTranslationArgs(AssetSpecUpdateKwargs, Resolvable):
+    """Model used to allow per-object-type translation of a Polytomic object."""
+
+    for_schema: Optional[ResolvedTargetedPolytomicTranslationFn] = None
+
+
+ResolvedPolytomicTranslationFn = Annotated[
+    PolytomicTranslationFn,
+    Resolver(
+        _resolve_multilayer_translation,
+        model_field_type=Union[str, PolytomicTranslationArgs.model()],
+    ),
+]

dagster_polytomic-0.28.12/dagster_polytomic/version.py

@@ -0,0 +1 @@
+__version__ = "0.28.12"

dagster_polytomic-0.28.12/dagster_polytomic/workspace.py

@@ -0,0 +1,92 @@
+import asyncio
+from functools import cached_property
+
+import dagster as dg
+from dagster import _check as check
+from polytomic import AsyncPolytomic, Polytomic
+from pydantic import Field
+
+from dagster_polytomic.objects import (
+    PolytomicBulkSync,
+    PolytomicBulkSyncSchema,
+    PolytomicConnection,
+    PolytomicIdentity,
+    PolytomicWorkspaceData,
+)
+
+POLYTOMIC_CLIENT_VERSION = "2024-02-08"
+
+
+class PolytomicWorkspace(dg.Resolvable, dg.Model):
+    """Handles all interactions with the Polytomic API to fetch and manage state."""
+
+    token: str = Field(
+        description="The API key to your Polytomic organization.",
+        examples=['"{{ env.POLYTOMIC_API_KEY }}"'],
+        repr=False,
+    )
+
+    @cached_property
+    def async_client(self) -> AsyncPolytomic:
+        return AsyncPolytomic(
+            version=POLYTOMIC_CLIENT_VERSION,
+            token=self.token,
+        )
+
+    @cached_property
+    def client(self) -> Polytomic:
+        return Polytomic(
+            version=POLYTOMIC_CLIENT_VERSION,
+            token=self.token,
+        )
+
+    @cached_property
+    def identity(self) -> PolytomicIdentity:
+        data = self.client.identity.get().data
+        return PolytomicIdentity.from_api_response(
+            check.not_none(data, "Identity data cannot be None")
+        )
+
+    async def _fetch_connections(self) -> list[PolytomicConnection]:
+        """Fetch all connections."""
+        response = await self.async_client.connections.list()
+        data = response.data or []
+        return [PolytomicConnection.from_api_response(conn) for conn in data]
+
+    async def _fetch_bulk_syncs(self) -> list[PolytomicBulkSync]:
+        """Fetch all bulk syncs."""
+        response = await self.async_client.bulk_sync.list()
+        data = response.data or []
+        return [PolytomicBulkSync.from_api_response(sync) for sync in data]
+
+    async def _fetch_bulk_sync_schemas(self, bulk_sync_id: str) -> list[PolytomicBulkSyncSchema]:
+        """Fetch all schemas for a specific bulk sync."""
+        response = await self.async_client.bulk_sync.schemas.list(id=bulk_sync_id)
+        data = response.data or []
+        return [
+            PolytomicBulkSyncSchema.from_api_response(schema) for schema in data if schema.enabled
+        ]
+
+    async def fetch_polytomic_state(self) -> PolytomicWorkspaceData:
+        """Fetch all connections, bulks syncs and schemas from the Polytomic API.
+
+        This is the main public method for getting complete Polytomic state.
+        """
+        connections = await self._fetch_connections()
+        bulk_syncs = await self._fetch_bulk_syncs()
+
+        # Fetch all schemas in parallel using asyncio.gather
+        schema_results = await asyncio.gather(
+            *[self._fetch_bulk_sync_schemas(bulk_sync_id=bulk_sync.id) for bulk_sync in bulk_syncs]
+        )
+
+        # Build the schemas dictionary
+        schemas = {
+            bulk_sync.id: schema_list for bulk_sync, schema_list in zip(bulk_syncs, schema_results)
+        }
+
+        return PolytomicWorkspaceData(
+            connections=connections,
+            bulk_syncs=bulk_syncs,
+            schemas_by_bulk_sync_id=schemas,
+        )

dagster_polytomic-0.28.12/dagster_polytomic.egg-info/PKG-INFO

@@ -0,0 +1,28 @@
+Metadata-Version: 2.4
+Name: dagster_polytomic
+Version: 0.28.12
+Summary: Package for integrating Polytomic with Dagster.
+Home-page: https://github.com/dagster-io/dagster/tree/master/python_modules/libraries/dagster-polytomic
+Author: Dagster Labs
+Author-email: hello@dagsterlabs.com
+License: Apache-2.0
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Programming Language :: Python :: 3.14
+Classifier: License :: OSI Approved :: Apache Software License
+Classifier: Operating System :: OS Independent
+Requires-Python: >=3.10,<3.15
+License-File: LICENSE
+Requires-Dist: dagster==1.12.12
+Requires-Dist: polytomic>=1.11.3
+Dynamic: author
+Dynamic: author-email
+Dynamic: classifier
+Dynamic: home-page
+Dynamic: license
+Dynamic: license-file
+Dynamic: requires-dist
+Dynamic: requires-python
+Dynamic: summary

dagster_polytomic-0.28.12/dagster_polytomic.egg-info/SOURCES.txt

@@ -0,0 +1,18 @@
+LICENSE
+MANIFEST.in
+README.md
+setup.cfg
+setup.py
+dagster_polytomic/__init__.py
+dagster_polytomic/component.py
+dagster_polytomic/objects.py
+dagster_polytomic/py.typed
+dagster_polytomic/translation.py
+dagster_polytomic/version.py
+dagster_polytomic/workspace.py
+dagster_polytomic.egg-info/PKG-INFO
+dagster_polytomic.egg-info/SOURCES.txt
+dagster_polytomic.egg-info/dependency_links.txt
+dagster_polytomic.egg-info/not-zip-safe
+dagster_polytomic.egg-info/requires.txt
+dagster_polytomic.egg-info/top_level.txt

dagster_polytomic-0.28.12/dagster_polytomic.egg-info/dependency_links.txt

@@ -0,0 +1 @@
+

dagster_polytomic-0.28.12/dagster_polytomic.egg-info/not-zip-safe

@@ -0,0 +1 @@
+

dagster_polytomic-0.28.12/dagster_polytomic.egg-info/requires.txt

@@ -0,0 +1,2 @@
+dagster==1.12.12
+polytomic>=1.11.3

dagster_polytomic-0.28.12/dagster_polytomic.egg-info/top_level.txt

@@ -0,0 +1 @@
+dagster_polytomic

dagster_polytomic-0.28.12/setup.cfg

@@ -0,0 +1,14 @@
+[metadata]
+license_files = LICENSE
+
+[check-manifest]
+ignore = 
+	.coveragerc
+	tox.ini
+	pytest.ini
+	dagster_polytomic_tests/**
+
+[egg_info]
+tag_build = 
+tag_date = 0
+

dagster_polytomic-0.28.12/setup.py

@@ -0,0 +1,39 @@
+from pathlib import Path
+
+from setuptools import find_packages, setup
+
+
+def get_version() -> str:
+    version: dict[str, str] = {}
+    with open(Path(__file__).parent / "dagster_polytomic/version.py", encoding="utf8") as fp:
+        exec(fp.read(), version)
+
+    return version["__version__"]
+
+
+ver = get_version()
+# dont pin dev installs to avoid pip dep resolver issues
+pin = "" if ver == "1!0+dev" else f"=={ver}"
+setup(
+    name="dagster_polytomic",
+    version=ver,
+    author="Dagster Labs",
+    author_email="hello@dagsterlabs.com",
+    license="Apache-2.0",
+    description="Package for integrating Polytomic with Dagster.",
+    url="https://github.com/dagster-io/dagster/tree/master/python_modules/libraries/dagster-polytomic",
+    classifiers=[
+        "Programming Language :: Python :: 3.10",
+        "Programming Language :: Python :: 3.11",
+        "Programming Language :: Python :: 3.12",
+        "Programming Language :: Python :: 3.13",
+        "Programming Language :: Python :: 3.14",
+        "License :: OSI Approved :: Apache Software License",
+        "Operating System :: OS Independent",
+    ],
+    packages=find_packages(exclude=["dagster_polytomic_tests*"]),
+    include_package_data=True,
+    python_requires=">=3.10,<3.15",
+    install_requires=["dagster==1.12.12", "polytomic>=1.11.3"],
+    zip_safe=False,
+)