dagster-omni 0.27.9__tar.gz → 0.27.11__tar.gz

{dagster_omni-0.27.9/dagster_omni.egg-info → dagster_omni-0.27.11}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dagster_omni
-Version: 0.27.9
+Version: 0.27.11
 Summary: Package for integrating Omni with Dagster.
 Home-page: https://github.com/dagster-io/dagster/tree/master/python_modules/libraries/dagster-omni
 Author: Dagster Labs
@@ -14,7 +14,7 @@ Classifier: Programming Language :: Python :: 3.13
 Classifier: License :: OSI Approved :: Apache Software License
 Classifier: Operating System :: OS Independent
 License-File: LICENSE
-Requires-Dist: dagster==1.11.9
+Requires-Dist: dagster==1.11.11
 Requires-Dist: aiohttp
 Dynamic: author
 Dynamic: author-email

{dagster_omni-0.27.9 → dagster_omni-0.27.11}/dagster_omni/component.py

@@ -1,64 +1,30 @@
 import itertools
 from collections import defaultdict
 from pathlib import Path
-from typing import Optional, Union
+from typing import Optional
 
 import dagster as dg
 from dagster._annotations import preview
-from dagster._core.definitions.metadata.metadata_set import NamespacedMetadataSet
-from dagster._core.definitions.metadata.metadata_value import UrlMetadataValue
 from dagster._core.errors import DagsterInvalidDefinitionError
 from dagster.components.component.state_backed_component import StateBackedComponent
-from dagster.components.utils.translation import ResolvedTranslationFn
-from dagster_shared.record import record
 from pydantic import Field
-from typing_extensions import Self
 
 from dagster_omni.objects import OmniDocument, OmniQuery, OmniWorkspaceData
+from dagster_omni.translation import (
+    TRANSLATOR_DATA_METADATA_KEY,
+    OmniDocumentMetadataSet,
+    OmniTranslatorData,
+    ResolvedOmniTranslationFn,
+)
 from dagster_omni.workspace import OmniWorkspace
 
-_TRANSLATOR_DATA_METADATA_KEY = ".dagster-omni/translator_data"
-
-
-class OmniDocumentMetadataSet(NamespacedMetadataSet):
-    url: Optional[UrlMetadataValue] = None
-    document_name: str
-    document_type: str
-
-    @classmethod
-    def from_document(cls, workspace: OmniWorkspace, document: OmniDocument) -> Self:
-        url_str = f"{workspace.base_url.rstrip('/')}/dashboards/{document.identifier}"
-        return cls(
-            url=UrlMetadataValue(url_str) if document.has_dashboard else None,
-            document_name=document.name,
-            document_type=document.type,
-        )
-
-    @classmethod
-    def namespace(cls) -> str:
-        return "dagster-omni"
-
-
-@record
-class OmniTranslatorData:
-    """Container class for data required to translate an object in an
-    Omni workspace into a Dagster definition.
-
-    Properties:
-        obj (Union[OmniDocument, OmniQuery]): The object to translate.
-        workspace_data (OmniWorkspaceData): Global workspace data.
-    """
-
-    obj: Union[OmniDocument, OmniQuery]
-    workspace_data: OmniWorkspaceData
-
 
 @preview
 class OmniComponent(StateBackedComponent, dg.Model, dg.Resolvable):
     workspace: OmniWorkspace = Field(
         description="Defines configuration for interacting with an Omni instance.",
     )
-    translation: Optional[ResolvedTranslationFn[OmniTranslatorData]] = Field(
+    translation: Optional[ResolvedOmniTranslationFn] = Field(
         default=None,
         description="Defines how to translate an Omni object into an AssetSpec object.",
     )
@@ -72,49 +38,58 @@ class OmniComponent(StateBackedComponent, dg.Model, dg.Resolvable):
         """Load state from path using Dagster's deserialization system."""
         return dg.deserialize_value(state_path.read_text(), OmniWorkspaceData)
 
-    def _get_default_asset_spec(self, data: OmniTranslatorData) -> dg.AssetSpec:
+    def _get_default_omni_spec(
+        self, context: dg.ComponentLoadContext, data: OmniTranslatorData, workspace: OmniWorkspace
+    ) -> Optional[dg.AssetSpec]:
         """Core function for converting an Omni document into an AssetSpec object."""
         if isinstance(data.obj, OmniDocument):
             doc = data.obj
-            deps = [
+            maybe_deps = [
                 self.get_asset_spec(
-                    OmniTranslatorData(obj=query, workspace_data=data.workspace_data)
+                    context, OmniTranslatorData(obj=query, workspace_data=data.workspace_data)
                 )
-                for query in doc.queries
+                for query in data.obj.queries
             ]
 
             prefix = doc.folder.path.split("/") if doc.folder else []
+            user = data.workspace_data.get_user(doc.owner.id)
+            owner_email = user.primary_email if user else None
+
             return dg.AssetSpec(
                 key=dg.AssetKey([*prefix, doc.name]),
                 group_name=prefix[0].replace("-", "_") if prefix else None,
                 tags={label.name: "" for label in doc.labels},
-                deps=deps,
+                deps=list(filter(None, maybe_deps)),
                 metadata={
-                    **OmniDocumentMetadataSet.from_document(self.workspace, doc),
-                    _TRANSLATOR_DATA_METADATA_KEY: data,
+                    **OmniDocumentMetadataSet.from_document(workspace, doc),
+                    TRANSLATOR_DATA_METADATA_KEY: data,
                 },
                 kinds={"omni"},
+                owners=[owner_email] if owner_email else None,
             )
-        elif isinstance(data.obj, OmniQuery):
+        if isinstance(data.obj, OmniQuery):
             return dg.AssetSpec(key=dg.AssetKey([data.obj.query_config.table]))
-        else:
-            raise ValueError(f"Unsupported object type: {type(data.obj)}")
+        return None
 
-    def get_asset_spec(self, data: OmniTranslatorData) -> dg.AssetSpec:
+    def get_asset_spec(
+        self, context: dg.ComponentLoadContext, data: OmniTranslatorData
+    ) -> Optional[dg.AssetSpec]:
         """Core function for converting an Omni document into an AssetSpec object."""
-        base_asset_spec = self._get_default_asset_spec(data)
-        if self.translation:
+        base_asset_spec = self._get_default_omni_spec(context, data, self.workspace)
+        if self.translation and base_asset_spec:
             return self.translation(base_asset_spec, data)
         else:
             return base_asset_spec
 
-    def _build_asset_specs(self, workspace_data: OmniWorkspaceData) -> list[dg.AssetSpec]:
+    def _build_asset_specs(
+        self, context: dg.ComponentLoadContext, workspace_data: OmniWorkspaceData
+    ) -> list[dg.AssetSpec]:
         """Invokes the `get_asset_spec` method on all objects in the provided `workspace_data`.
         Filters out any cases where the asset_spec is `None`, and provides a helpful error
         message in cases where keys overlap between different documents.
         """
         maybe_specs = [
-            self.get_asset_spec(OmniTranslatorData(obj=doc, workspace_data=workspace_data))
+            self.get_asset_spec(context, OmniTranslatorData(obj=doc, workspace_data=workspace_data))
             for doc in workspace_data.documents
         ]
 
@@ -136,8 +111,10 @@ class OmniComponent(StateBackedComponent, dg.Model, dg.Resolvable):
 
         return list(itertools.chain.from_iterable(specs_by_key.values()))
 
-    def build_defs_from_workspace_data(self, workspace_data: OmniWorkspaceData) -> dg.Definitions:
-        return dg.Definitions(assets=self._build_asset_specs(workspace_data))
+    def build_defs_from_workspace_data(
+        self, context: dg.ComponentLoadContext, workspace_data: OmniWorkspaceData
+    ) -> dg.Definitions:
+        return dg.Definitions(assets=self._build_asset_specs(context, workspace_data))
 
     def build_defs_from_state(
         self, context: dg.ComponentLoadContext, state_path: Optional[Path]
@@ -146,4 +123,4 @@ class OmniComponent(StateBackedComponent, dg.Model, dg.Resolvable):
             return dg.Definitions()
 
         state = self.load_state_from_path(state_path)
-        return self.build_defs_from_workspace_data(state)
+        return self.build_defs_from_workspace_data(context, state)

{dagster_omni-0.27.9 → dagster_omni-0.27.11}/dagster_omni/objects.py

@@ -1,3 +1,4 @@
+from functools import cached_property
 from typing import Any, Optional
 
 from dagster_shared.record import record
@@ -57,6 +58,8 @@ class OmniDocument:
     folder: Optional[OmniFolder]
     labels: list[OmniLabel]
     queries: list["OmniQuery"]
+    favorites: Optional[int] = None
+    views: Optional[int] = None
 
     @classmethod
     def from_json(cls, data: dict[str, Any], queries: list["OmniQuery"]) -> "OmniDocument":
@@ -81,6 +84,8 @@ class OmniDocument:
             folder=folder,
             labels=labels,
             queries=queries,
+            favorites=data.get("_count", {}).get("favorites", None),
+            views=data.get("_count", {}).get("views", None),
         )
 
 
@@ -118,6 +123,42 @@ class OmniQuery:
         )
 
 
+@whitelist_for_serdes
+@record
+class OmniUser:
+    """Represents an Omni user with all user information in a single class."""
+
+    id: str
+    name: Optional[str]
+    display_name: str
+    user_name: str
+    active: bool
+    primary_email: Optional[str]
+    groups: list[str]
+    created: str
+    last_modified: str
+
+    @classmethod
+    def from_json(cls, data: dict[str, Any]) -> "OmniUser":
+        """Create OmniUser from JSON response data."""
+        primary_email = next(
+            (email["value"] for email in data.get("emails", []) if email["primary"]), None
+        )
+        groups = [group["display"] for group in data.get("groups", [])]
+
+        return cls(
+            id=data["id"],
+            name=data.get("displayName") or data.get("userName") or "",
+            display_name=data.get("displayName", ""),
+            user_name=data.get("userName", ""),
+            active=data.get("active", True),
+            primary_email=primary_email,
+            groups=groups,
+            created=data.get("meta", {}).get("created", ""),
+            last_modified=data.get("meta", {}).get("lastModified", ""),
+        )
+
+
 @whitelist_for_serdes
 @record
 class OmniWorkspaceData:
@@ -125,6 +166,15 @@ class OmniWorkspaceData:
 
     Properties:
         documents: list[OmniDocument]
+        users: list[OmniUser]
     """
 
     documents: list[OmniDocument]
+    users: list[OmniUser]
+
+    @cached_property
+    def _users_by_id(self) -> dict[str, OmniUser]:
+        return {user.id: user for user in self.users}
+
+    def get_user(self, user_id: str) -> Optional[OmniUser]:
+        return self._users_by_id.get(user_id)

dagster_omni-0.27.11/dagster_omni/translation.py

@@ -0,0 +1,143 @@
+from typing import Annotated, Optional, Union
+
+import dateutil
+from dagster._core.definitions.assets.definition.asset_spec import AssetSpec
+from dagster._core.definitions.metadata.metadata_set import NamespacedMetadataSet
+from dagster._core.definitions.metadata.metadata_value import (
+    TimestampMetadataValue,
+    UrlMetadataValue,
+)
+from dagster.components import Resolvable, Resolver
+from dagster.components.resolved.base import resolve_fields
+from dagster.components.resolved.context import ResolutionContext
+from dagster.components.resolved.core_models import AssetSpecKeyUpdateKwargs, AssetSpecUpdateKwargs
+from dagster.components.utils import TranslatorResolvingInfo
+from dagster.components.utils.translation import TranslationFn, TranslationFnResolver
+from dagster_shared.record import record
+from typing_extensions import Self, TypeAlias
+
+from dagster_omni.objects import OmniDocument, OmniQuery, OmniWorkspaceData
+from dagster_omni.workspace import OmniWorkspace
+
+TRANSLATOR_DATA_METADATA_KEY = ".dagster-omni/translator_data"
+
+
+class OmniDocumentMetadataSet(NamespacedMetadataSet):
+    """Represents metadata that is captured from an Omni document."""
+
+    url: Optional[UrlMetadataValue] = None
+    owner_name: str
+    document_name: str
+    document_type: str
+    updated_at: TimestampMetadataValue
+    favorites: Optional[int] = None
+    views: Optional[int] = None
+
+    @classmethod
+    def from_document(cls, workspace: OmniWorkspace, document: OmniDocument) -> Self:
+        url_str = f"{workspace.base_url.rstrip('/')}/dashboards/{document.identifier}"
+
+        return cls(
+            url=UrlMetadataValue(url_str) if document.has_dashboard else None,
+            document_name=document.name,
+            document_type=document.type,
+            updated_at=TimestampMetadataValue(
+                dateutil.parser.parse(document.updated_at).timestamp()
+            ),
+            owner_name=document.owner.name,
+            favorites=document.favorites,
+            views=document.views,
+        )
+
+    @classmethod
+    def namespace(cls) -> str:
+        return "dagster-omni"
+
+
+@record
+class OmniTranslatorData:
+    """Container class for data required to translate an object in an
+    Omni workspace into a Dagster definition.
+
+    Properties:
+        obj (Union[OmniDocument, OmniQuery]): The object to translate.
+        workspace_data (OmniWorkspaceData): Global workspace data.
+    """
+
+    obj: Union[OmniDocument, OmniQuery]
+    workspace_data: OmniWorkspaceData
+
+
+def _resolve_multilayer_translation(context: ResolutionContext, model):
+    """The Omni translation schema supports defining global transforms
+    as well as per-object-type transforms. This resolver composes the
+    per-object-type transforms with the global transforms.
+    """
+    info = TranslatorResolvingInfo(
+        asset_attributes=model,
+        resolution_context=context,
+        model_key="translation",
+    )
+
+    def _translation_fn(base_asset_spec: AssetSpec, data: OmniTranslatorData):
+        processed_spec = info.get_asset_spec(
+            base_asset_spec,
+            {
+                "data": data,
+                "spec": base_asset_spec,
+            },
+        )
+
+        nested_translation_fns = resolve_fields(
+            model=model,
+            resolved_cls=OmniTranslationArgs,
+            context=context.with_scope(
+                **{
+                    "data": data,
+                    "spec": processed_spec,
+                }
+            ),
+        )
+        for_document = nested_translation_fns.get("for_document")
+        for_query = nested_translation_fns.get("for_query")
+
+        if isinstance(data.obj, OmniDocument) and for_document:
+            return for_document(processed_spec, data)
+        if isinstance(data.obj, OmniQuery) and for_query:
+            return for_query(processed_spec, data)
+
+        return processed_spec
+
+    return _translation_fn
+
+
+OmniTranslationFn: TypeAlias = TranslationFn[OmniTranslatorData]
+
+ResolvedTargetedOmniTranslationFn = Annotated[
+    OmniTranslationFn,
+    TranslationFnResolver[OmniTranslatorData](lambda data: {"data": data}),
+]
+
+ResolvedTargetedKeyOnlyOmniTranslationFn = Annotated[
+    OmniTranslationFn,
+    TranslationFnResolver[OmniTranslatorData](
+        lambda data: {"data": data}, model_field_type=AssetSpecKeyUpdateKwargs.model()
+    ),
+]
+
+
+@record
+class OmniTranslationArgs(AssetSpecUpdateKwargs, Resolvable):
+    """Model used to allow per-object-type translation of an Omni object."""
+
+    for_document: Optional[ResolvedTargetedOmniTranslationFn] = None
+    for_query: Optional[ResolvedTargetedKeyOnlyOmniTranslationFn] = None
+
+
+ResolvedOmniTranslationFn = Annotated[
+    OmniTranslationFn,
+    Resolver(
+        _resolve_multilayer_translation,
+        model_field_type=Union[str, OmniTranslationArgs.model()],
+    ),
+]

dagster_omni-0.27.11/dagster_omni/version.py

@@ -0,0 +1 @@
+__version__ = "0.27.11"

{dagster_omni-0.27.9 → dagster_omni-0.27.11}/dagster_omni/workspace.py

@@ -8,7 +8,7 @@ from aiohttp.client_exceptions import ClientResponseError
 from dagster._utils.backoff import async_backoff, exponential_delay_generator
 from pydantic import Field
 
-from dagster_omni.objects import OmniDocument, OmniQuery, OmniWorkspaceData
+from dagster_omni.objects import OmniDocument, OmniQuery, OmniUser, OmniWorkspaceData
 
 
 class OmniWorkspace(dg.Resolvable, dg.Model):
@@ -50,7 +50,7 @@ class OmniWorkspace(dg.Resolvable, dg.Model):
         return isinstance(exc, aiohttp.ClientError)
 
     def _build_url(self, endpoint: str) -> str:
-        return f"{self.base_url.rstrip('/')}/api/v1/{endpoint.lstrip('/')}"
+        return f"{self.base_url.rstrip('/')}/{endpoint.lstrip('/')}"
 
     async def make_request(
         self,
@@ -79,7 +79,7 @@ class OmniWorkspace(dg.Resolvable, dg.Model):
 
     async def _fetch_document_queries(self, document_identifier: str) -> list[OmniQuery]:
         """Fetch all queries for a specific document."""
-        endpoint = f"documents/{document_identifier}/queries"
+        endpoint = f"api/v1/documents/{document_identifier}/queries"
         try:
             response = await self.make_request(endpoint)
             return [OmniQuery.from_json(query_data) for query_data in response.get("queries", [])]
@@ -96,7 +96,7 @@ class OmniWorkspace(dg.Resolvable, dg.Model):
 
     async def _fetch_documents(self) -> list[OmniDocument]:
         """Fetch all documents from the Omni API with their queries embedded."""
-        base_params = {"pageSize": "100"}
+        base_params = {"pageSize": "100", "include": "_count"}
         documents = []
         next_cursor = None
 
@@ -105,7 +105,7 @@ class OmniWorkspace(dg.Resolvable, dg.Model):
             if next_cursor:
                 params["cursor"] = next_cursor
 
-            response = await self.make_request("documents", params)
+            response = await self.make_request("api/v1/documents", params)
 
             # Fan out the requests to fetch queries for each document in parallel
             coroutines = [
@@ -120,10 +120,37 @@ class OmniWorkspace(dg.Resolvable, dg.Model):
 
         return documents
 
+    async def _fetch_users(self) -> list[OmniUser]:
+        """Fetch all users from the Omni SCIM API."""
+        base_params = {"count": "100"}
+        users = []
+        start_index = 1
+
+        while True:
+            params = base_params.copy()
+            params["startIndex"] = str(start_index)
+
+            response = await self.make_request("api/scim/v2/users", params)
+
+            user_resources = response.get("Resources", [])
+            if not user_resources:
+                break
+
+            users.extend([OmniUser.from_json(user_data) for user_data in user_resources])
+
+            # Check if we've received fewer users than requested, indicating we're done
+            if len(user_resources) < int(base_params["count"]):
+                break
+
+            start_index += len(user_resources)
+
+        return users
+
     async def fetch_omni_state(self) -> OmniWorkspaceData:
-        """Fetch all documents from the Omni API with queries embedded.
+        """Fetch all documents and users from the Omni API.
 
         This is the main public method for getting complete Omni state.
         """
-        documents = await self._fetch_documents()
-        return OmniWorkspaceData(documents=documents)
+        # Fetch documents and users concurrently
+        documents, users = await asyncio.gather(self._fetch_documents(), self._fetch_users())
+        return OmniWorkspaceData(documents=documents, users=users)

{dagster_omni-0.27.9 → dagster_omni-0.27.11/dagster_omni.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dagster_omni
-Version: 0.27.9
+Version: 0.27.11
 Summary: Package for integrating Omni with Dagster.
 Home-page: https://github.com/dagster-io/dagster/tree/master/python_modules/libraries/dagster-omni
 Author: Dagster Labs
@@ -14,7 +14,7 @@ Classifier: Programming Language :: Python :: 3.13
 Classifier: License :: OSI Approved :: Apache Software License
 Classifier: Operating System :: OS Independent
 License-File: LICENSE
-Requires-Dist: dagster==1.11.9
+Requires-Dist: dagster==1.11.11
 Requires-Dist: aiohttp
 Dynamic: author
 Dynamic: author-email

{dagster_omni-0.27.9 → dagster_omni-0.27.11}/dagster_omni.egg-info/SOURCES.txt

@@ -7,6 +7,7 @@ dagster_omni/__init__.py
 dagster_omni/component.py
 dagster_omni/objects.py
 dagster_omni/py.typed
+dagster_omni/translation.py
 dagster_omni/version.py
 dagster_omni/workspace.py
 dagster_omni.egg-info/PKG-INFO

dagster_omni-0.27.11/dagster_omni.egg-info/requires.txt

@@ -0,0 +1,2 @@
+dagster==1.11.11
+aiohttp

{dagster_omni-0.27.9 → dagster_omni-0.27.11}/setup.py

@@ -34,7 +34,7 @@ setup(
     packages=find_packages(exclude=["dagster_omni_tests*"]),
     include_package_data=True,
     install_requires=[
-        "dagster==1.11.9",
+        "dagster==1.11.11",
         "aiohttp",
     ],
     zip_safe=False,

dagster_omni-0.27.9/dagster_omni/version.py

@@ -1 +0,0 @@
-__version__ = "0.27.9"

dagster_omni-0.27.9/dagster_omni.egg-info/requires.txt

@@ -1,2 +0,0 @@
-dagster==1.11.9
-aiohttp