acryl-datahub 1.2.0.10rc1__py3-none-any.whl → 1.2.0.10rc2__py3-none-any.whl

datahub/ingestion/source/superset.py

@@ -282,6 +282,7 @@ def get_filter_name(filter_obj):
 )
 @capability(SourceCapability.DOMAINS, "Enabled by `domain` config to assign domain_key")
 @capability(SourceCapability.LINEAGE_COARSE, "Supported by default")
+@capability(SourceCapability.TAGS, "Supported by default")
 class SupersetSource(StatefulIngestionSourceBase):
     """
     This plugin extracts the following:
@@ -521,6 +522,11 @@ class SupersetSource(StatefulIngestionSourceBase):
         )
         dashboard_snapshot.aspects.append(owners_info)
 
+        superset_tags = self._extract_and_map_tags(dashboard_data.get("tags", []))
+        tags = self._merge_tags_with_existing(dashboard_urn, superset_tags)
+        if tags:
+            dashboard_snapshot.aspects.append(tags)
+
         return dashboard_snapshot
 
     def _process_dashboard(self, dashboard_data: Any) -> Iterable[MetadataWorkUnit]:
@@ -919,6 +925,12 @@ class SupersetSource(StatefulIngestionSourceBase):
             lastModified=last_modified,
         )
         chart_snapshot.aspects.append(owners_info)
+
+        superset_tags = self._extract_and_map_tags(chart_data.get("tags", []))
+        tags = self._merge_tags_with_existing(chart_urn, superset_tags)
+        if tags:
+            chart_snapshot.aspects.append(tags)
+
         yield MetadataWorkUnit(
             id=chart_urn, mce=MetadataChangeEvent(proposedSnapshot=chart_snapshot)
         )
@@ -1288,17 +1300,18 @@ class SupersetSource(StatefulIngestionSourceBase):
             externalUrl=dataset_url,
             lastModified=TimeStamp(time=modified_ts),
         )
-        global_tags = GlobalTagsClass(tags=[TagAssociationClass(tag=tag_urn)])
-
-        aspects_items: List[Any] = []
-        aspects_items.extend(
-            [
-                self.gen_schema_metadata(dataset_response),
-                dataset_info,
-                upstream_lineage,
-                global_tags,
-            ]
-        )
+
+        dataset_tags = GlobalTagsClass(tags=[TagAssociationClass(tag=tag_urn)])
+        tags = self._merge_tags_with_existing(datasource_urn, dataset_tags)
+
+        aspects_items: List[Any] = [
+            self.gen_schema_metadata(dataset_response),
+            dataset_info,
+            upstream_lineage,
+        ]
+
+        if tags:
+            aspects_items.append(tags)
 
         dataset_snapshot = DatasetSnapshot(
             urn=datasource_urn,
@@ -1320,6 +1333,75 @@ class SupersetSource(StatefulIngestionSourceBase):
 
         return dataset_snapshot
 
+    def _extract_and_map_tags(
+        self, raw_tags: List[Dict[str, Any]]
+    ) -> Optional[GlobalTagsClass]:
+        """Extract and map Superset tags to DataHub GlobalTagsClass.
+
+        Filters out system-generated tags (type != 1) and only processes user-defined tags
+        from the Superset API response.
+
+        Args:
+            raw_tags: List of tag dictionaries from Superset API
+
+        Returns:
+            GlobalTagsClass with user-defined tags, or None if no tags found
+        """
+        user_tags = [
+            tag.get("name", "")
+            for tag in raw_tags
+            if tag.get("type") == 1 and tag.get("name")
+        ]
+
+        if not user_tags:
+            return None
+
+        tag_urns = [builder.make_tag_urn(tag) for tag in user_tags]
+        return GlobalTagsClass(
+            tags=[TagAssociationClass(tag=tag_urn) for tag_urn in tag_urns]
+        )
+
+    def _merge_tags_with_existing(
+        self, entity_urn: str, new_tags: Optional[GlobalTagsClass]
+    ) -> Optional[GlobalTagsClass]:
+        """Merge new tags with existing ones from DataHub to preserve manually added tags.
+
+        This method ensures that tags manually added via DataHub UI are not overwritten
+        during ingestion. It fetches existing tags from the graph and merges them with
+        new tags from the source system, avoiding duplicates.
+
+        Args:
+            entity_urn: URN of the entity to check for existing tags
+            new_tags: New tags to add as GlobalTagsClass object
+
+        Returns:
+            GlobalTagsClass with merged tags preserving existing ones, or None if no tags
+        """
+        if not new_tags or not new_tags.tags:
+            return None
+
+        # Fetch existing tags from DataHub
+        existing_global_tags = None
+        if self.ctx.graph:
+            existing_global_tags = self.ctx.graph.get_aspect(
+                entity_urn=entity_urn, aspect_type=GlobalTagsClass
+            )
+
+        # Merge existing tags with new ones, avoiding duplicates
+        all_tags = []
+        existing_tag_urns = set()
+
+        if existing_global_tags and existing_global_tags.tags:
+            all_tags.extend(existing_global_tags.tags)
+            existing_tag_urns = {tag.tag for tag in existing_global_tags.tags}
+
+        # Add new tags that don't already exist
+        for new_tag in new_tags.tags:
+            if new_tag.tag not in existing_tag_urns:
+                all_tags.append(new_tag)
+
+        return GlobalTagsClass(tags=all_tags) if all_tags else None
+
     def _process_dataset(self, dataset_data: Any) -> Iterable[MetadataWorkUnit]:
         dataset_name = ""
         try:

datahub/metadata/_internal_schema_classes.py

@@ -15271,7 +15271,7 @@ class DataHubIngestionSourceSourceTypeClass(object):
     
     
 class LogicalParentClass(_Aspect):
-    # No docs available.
+    """Relates a physical asset to a logical model."""
 
 
     ASPECT_NAME = 'logicalParent'
@@ -24921,6 +24921,7 @@ class GlobalSettingsInfoClass(_Aspect):
 
     def __init__(self,
         sso: Union[None, "SsoSettingsClass"]=None,
+        oauth: Union[None, "OAuthSettingsClass"]=None,
         views: Union[None, "GlobalViewsSettingsClass"]=None,
         docPropagation: Optional[Union["DocPropagationFeatureSettingsClass", None]]=None,
         homePage: Union[None, "GlobalHomePageSettingsClass"]=None,
@@ -24929,6 +24930,7 @@ class GlobalSettingsInfoClass(_Aspect):
         super().__init__()
         
         self.sso = sso
+        self.oauth = oauth
         self.views = views
         if docPropagation is None:
             # default: {'configVersion': None, 'config': None, 'enabled': True, 'columnPropagationEnabled': True}
@@ -24940,6 +24942,7 @@ class GlobalSettingsInfoClass(_Aspect):
     
     def _restore_defaults(self) -> None:
         self.sso = self.RECORD_SCHEMA.fields_dict["sso"].default
+        self.oauth = self.RECORD_SCHEMA.fields_dict["oauth"].default
         self.views = self.RECORD_SCHEMA.fields_dict["views"].default
         self.docPropagation = _json_converter.from_json_object(self.RECORD_SCHEMA.fields_dict["docPropagation"].default, writers_schema=self.RECORD_SCHEMA.fields_dict["docPropagation"].type)
         self.homePage = self.RECORD_SCHEMA.fields_dict["homePage"].default
@@ -24956,6 +24959,16 @@ class GlobalSettingsInfoClass(_Aspect):
         self._inner_dict['sso'] = value
     
     
+    @property
+    def oauth(self) -> Union[None, "OAuthSettingsClass"]:
+        """Settings related to the oauth authentication provider"""
+        return self._inner_dict.get('oauth')  # type: ignore
+    
+    @oauth.setter
+    def oauth(self, value: Union[None, "OAuthSettingsClass"]) -> None:
+        self._inner_dict['oauth'] = value
+    
+    
     @property
     def views(self) -> Union[None, "GlobalViewsSettingsClass"]:
         """Settings related to the Views Feature"""
@@ -25021,6 +25034,145 @@ class GlobalViewsSettingsClass(DictWrapper):
         self._inner_dict['defaultView'] = value
     
     
+class OAuthProviderClass(DictWrapper):
+    """An OAuth Provider. This provides information required to validate inbound
+    requests with OAuth 2.0 bearer tokens."""
+    
+    RECORD_SCHEMA = get_schema_type("com.linkedin.pegasus2avro.settings.global.OAuthProvider")
+    def __init__(self,
+        enabled: bool,
+        name: str,
+        issuer: str,
+        audience: str,
+        jwksUri: Union[None, str]=None,
+        algorithm: Optional[str]=None,
+        userIdClaim: Optional[str]=None,
+    ):
+        super().__init__()
+        
+        self.enabled = enabled
+        self.name = name
+        self.jwksUri = jwksUri
+        self.issuer = issuer
+        self.audience = audience
+        if algorithm is None:
+            # default: 'RS256'
+            self.algorithm = self.RECORD_SCHEMA.fields_dict["algorithm"].default
+        else:
+            self.algorithm = algorithm
+        if userIdClaim is None:
+            # default: 'sub'
+            self.userIdClaim = self.RECORD_SCHEMA.fields_dict["userIdClaim"].default
+        else:
+            self.userIdClaim = userIdClaim
+    
+    def _restore_defaults(self) -> None:
+        self.enabled = bool()
+        self.name = str()
+        self.jwksUri = self.RECORD_SCHEMA.fields_dict["jwksUri"].default
+        self.issuer = str()
+        self.audience = str()
+        self.algorithm = self.RECORD_SCHEMA.fields_dict["algorithm"].default
+        self.userIdClaim = self.RECORD_SCHEMA.fields_dict["userIdClaim"].default
+    
+    
+    @property
+    def enabled(self) -> bool:
+        """Whether this OAuth provider is enabled."""
+        return self._inner_dict.get('enabled')  # type: ignore
+    
+    @enabled.setter
+    def enabled(self, value: bool) -> None:
+        self._inner_dict['enabled'] = value
+    
+    
+    @property
+    def name(self) -> str:
+        """The name of this OAuth provider. This is used for display purposes only."""
+        return self._inner_dict.get('name')  # type: ignore
+    
+    @name.setter
+    def name(self, value: str) -> None:
+        self._inner_dict['name'] = value
+    
+    
+    @property
+    def jwksUri(self) -> Union[None, str]:
+        """The URI of the JSON Web Key Set (JWKS) endpoint for this OAuth provider."""
+        return self._inner_dict.get('jwksUri')  # type: ignore
+    
+    @jwksUri.setter
+    def jwksUri(self, value: Union[None, str]) -> None:
+        self._inner_dict['jwksUri'] = value
+    
+    
+    @property
+    def issuer(self) -> str:
+        """The expected issuer (iss) claim in the JWTs issued by this OAuth provider."""
+        return self._inner_dict.get('issuer')  # type: ignore
+    
+    @issuer.setter
+    def issuer(self, value: str) -> None:
+        self._inner_dict['issuer'] = value
+    
+    
+    @property
+    def audience(self) -> str:
+        """The expected audience (aud) claim in the JWTs issued by this OAuth provider."""
+        return self._inner_dict.get('audience')  # type: ignore
+    
+    @audience.setter
+    def audience(self, value: str) -> None:
+        self._inner_dict['audience'] = value
+    
+    
+    @property
+    def algorithm(self) -> str:
+        """The JWT signing algorithm required for this provider.
+    Prevents algorithm confusion attacks. Common values: RS256, RS384, RS512, PS256, ES256"""
+        return self._inner_dict.get('algorithm')  # type: ignore
+    
+    @algorithm.setter
+    def algorithm(self, value: str) -> None:
+        self._inner_dict['algorithm'] = value
+    
+    
+    @property
+    def userIdClaim(self) -> str:
+        """The JWT claim to use as the user identifier for this provider.
+    Different providers use different claims (sub, email, preferred_username, etc.)"""
+        return self._inner_dict.get('userIdClaim')  # type: ignore
+    
+    @userIdClaim.setter
+    def userIdClaim(self, value: str) -> None:
+        self._inner_dict['userIdClaim'] = value
+    
+    
+class OAuthSettingsClass(DictWrapper):
+    """Trust oauth providers to use for authentication."""
+    
+    RECORD_SCHEMA = get_schema_type("com.linkedin.pegasus2avro.settings.global.OAuthSettings")
+    def __init__(self,
+        providers: List["OAuthProviderClass"],
+    ):
+        super().__init__()
+        
+        self.providers = providers
+    
+    def _restore_defaults(self) -> None:
+        self.providers = list()
+    
+    
+    @property
+    def providers(self) -> List["OAuthProviderClass"]:
+        """Trusted OAuth Providers"""
+        return self._inner_dict.get('providers')  # type: ignore
+    
+    @providers.setter
+    def providers(self, value: List["OAuthProviderClass"]) -> None:
+        self._inner_dict['providers'] = value
+    
+    
 class OidcSettingsClass(DictWrapper):
     """Settings for OIDC SSO integration."""
     
@@ -27636,6 +27788,8 @@ __SCHEMA_TYPES = {
     'com.linkedin.pegasus2avro.settings.global.GlobalHomePageSettings': GlobalHomePageSettingsClass,
     'com.linkedin.pegasus2avro.settings.global.GlobalSettingsInfo': GlobalSettingsInfoClass,
     'com.linkedin.pegasus2avro.settings.global.GlobalViewsSettings': GlobalViewsSettingsClass,
+    'com.linkedin.pegasus2avro.settings.global.OAuthProvider': OAuthProviderClass,
+    'com.linkedin.pegasus2avro.settings.global.OAuthSettings': OAuthSettingsClass,
     'com.linkedin.pegasus2avro.settings.global.OidcSettings': OidcSettingsClass,
     'com.linkedin.pegasus2avro.settings.global.SsoSettings': SsoSettingsClass,
     'com.linkedin.pegasus2avro.step.DataHubStepStateProperties': DataHubStepStatePropertiesClass,
@@ -28153,6 +28307,8 @@ __SCHEMA_TYPES = {
     'GlobalHomePageSettings': GlobalHomePageSettingsClass,
     'GlobalSettingsInfo': GlobalSettingsInfoClass,
     'GlobalViewsSettings': GlobalViewsSettingsClass,
+    'OAuthProvider': OAuthProviderClass,
+    'OAuthSettings': OAuthSettingsClass,
     'OidcSettings': OidcSettingsClass,
     'SsoSettings': SsoSettingsClass,
     'DataHubStepStateProperties': DataHubStepStatePropertiesClass,

datahub/metadata/com/linkedin/pegasus2avro/settings/global/__init__.py

@@ -12,6 +12,8 @@ from ......schema_classes import DocPropagationFeatureSettingsClass
 from ......schema_classes import GlobalHomePageSettingsClass
 from ......schema_classes import GlobalSettingsInfoClass
 from ......schema_classes import GlobalViewsSettingsClass
+from ......schema_classes import OAuthProviderClass
+from ......schema_classes import OAuthSettingsClass
 from ......schema_classes import OidcSettingsClass
 from ......schema_classes import SsoSettingsClass
 
@@ -21,6 +23,8 @@ DocPropagationFeatureSettings = DocPropagationFeatureSettingsClass
 GlobalHomePageSettings = GlobalHomePageSettingsClass
 GlobalSettingsInfo = GlobalSettingsInfoClass
 GlobalViewsSettings = GlobalViewsSettingsClass
+OAuthProvider = OAuthProviderClass
+OAuthSettings = OAuthSettingsClass
 OidcSettings = OidcSettingsClass
 SsoSettings = SsoSettingsClass
 

datahub/metadata/schema.avsc

@@ -10607,7 +10607,8 @@
         "name": "parent",
         "default": null
       }
-    ]
+    ],
+    "doc": "Relates a physical asset to a logical model."
   },
   {
     "type": "record",
@@ -11887,6 +11888,78 @@
         "default": null,
         "doc": "SSO integrations between DataHub and identity providers"
       },
+      {
+        "type": [
+          "null",
+          {
+            "type": "record",
+            "name": "OAuthSettings",
+            "namespace": "com.linkedin.pegasus2avro.settings.global",
+            "fields": [
+              {
+                "type": {
+                  "type": "array",
+                  "items": {
+                    "type": "record",
+                    "name": "OAuthProvider",
+                    "namespace": "com.linkedin.pegasus2avro.settings.global",
+                    "fields": [
+                      {
+                        "type": "boolean",
+                        "name": "enabled",
+                        "doc": "Whether this OAuth provider is enabled."
+                      },
+                      {
+                        "type": "string",
+                        "name": "name",
+                        "doc": "The name of this OAuth provider. This is used for display purposes only."
+                      },
+                      {
+                        "type": [
+                          "null",
+                          "string"
+                        ],
+                        "name": "jwksUri",
+                        "default": null,
+                        "doc": "The URI of the JSON Web Key Set (JWKS) endpoint for this OAuth provider."
+                      },
+                      {
+                        "type": "string",
+                        "name": "issuer",
+                        "doc": "The expected issuer (iss) claim in the JWTs issued by this OAuth provider."
+                      },
+                      {
+                        "type": "string",
+                        "name": "audience",
+                        "doc": "The expected audience (aud) claim in the JWTs issued by this OAuth provider."
+                      },
+                      {
+                        "type": "string",
+                        "name": "algorithm",
+                        "default": "RS256",
+                        "doc": "The JWT signing algorithm required for this provider.\nPrevents algorithm confusion attacks. Common values: RS256, RS384, RS512, PS256, ES256"
+                      },
+                      {
+                        "type": "string",
+                        "name": "userIdClaim",
+                        "default": "sub",
+                        "doc": "The JWT claim to use as the user identifier for this provider.\nDifferent providers use different claims (sub, email, preferred_username, etc.)"
+                      }
+                    ],
+                    "doc": "An OAuth Provider. This provides information required to validate inbound\nrequests with OAuth 2.0 bearer tokens."
+                  }
+                },
+                "name": "providers",
+                "doc": "Trusted OAuth Providers"
+              }
+            ],
+            "doc": "Trust oauth providers to use for authentication."
+          }
+        ],
+        "name": "oauth",
+        "default": null,
+        "doc": "Settings related to the oauth authentication provider"
+      },
       {
         "type": [
           "null",

datahub/metadata/schemas/GlobalSettingsInfo.avsc

@@ -198,6 +198,78 @@
       "default": null,
       "doc": "SSO integrations between DataHub and identity providers"
     },
+    {
+      "type": [
+        "null",
+        {
+          "type": "record",
+          "name": "OAuthSettings",
+          "namespace": "com.linkedin.pegasus2avro.settings.global",
+          "fields": [
+            {
+              "type": {
+                "type": "array",
+                "items": {
+                  "type": "record",
+                  "name": "OAuthProvider",
+                  "namespace": "com.linkedin.pegasus2avro.settings.global",
+                  "fields": [
+                    {
+                      "type": "boolean",
+                      "name": "enabled",
+                      "doc": "Whether this OAuth provider is enabled."
+                    },
+                    {
+                      "type": "string",
+                      "name": "name",
+                      "doc": "The name of this OAuth provider. This is used for display purposes only."
+                    },
+                    {
+                      "type": [
+                        "null",
+                        "string"
+                      ],
+                      "name": "jwksUri",
+                      "default": null,
+                      "doc": "The URI of the JSON Web Key Set (JWKS) endpoint for this OAuth provider."
+                    },
+                    {
+                      "type": "string",
+                      "name": "issuer",
+                      "doc": "The expected issuer (iss) claim in the JWTs issued by this OAuth provider."
+                    },
+                    {
+                      "type": "string",
+                      "name": "audience",
+                      "doc": "The expected audience (aud) claim in the JWTs issued by this OAuth provider."
+                    },
+                    {
+                      "type": "string",
+                      "name": "algorithm",
+                      "default": "RS256",
+                      "doc": "The JWT signing algorithm required for this provider.\nPrevents algorithm confusion attacks. Common values: RS256, RS384, RS512, PS256, ES256"
+                    },
+                    {
+                      "type": "string",
+                      "name": "userIdClaim",
+                      "default": "sub",
+                      "doc": "The JWT claim to use as the user identifier for this provider.\nDifferent providers use different claims (sub, email, preferred_username, etc.)"
+                    }
+                  ],
+                  "doc": "An OAuth Provider. This provides information required to validate inbound\nrequests with OAuth 2.0 bearer tokens."
+                }
+              },
+              "name": "providers",
+              "doc": "Trusted OAuth Providers"
+            }
+          ],
+          "doc": "Trust oauth providers to use for authentication."
+        }
+      ],
+      "name": "oauth",
+      "default": null,
+      "doc": "Settings related to the oauth authentication provider"
+    },
     {
       "type": [
         "null",

datahub/metadata/schemas/LogicalParent.avsc

@@ -140,5 +140,6 @@
       "name": "parent",
       "default": null
     }
-  ]
+  ],
+  "doc": "Relates a physical asset to a logical model."
 }