acryl-datahub 1.0.0rc2__py3-none-any.whl → 1.0.0rc4__py3-none-any.whl

datahub/ingestion/source/mode.py

@@ -759,9 +759,9 @@ class ModeSource(StatefulIngestionSourceBase):
                 return platform, database
         else:
             self.report.report_warning(
-                title="Failed to create Data Platform Urn",
-                message=f"Cannot create datasource urn for datasource id: "
-                f"{data_source_id}",
+                title="Unable to construct upstream lineage",
+                message="We did not find a data source / connection with a matching ID, meaning that we do not know the platform/database to use in lineage.",
+                context=f"Data Source ID: {data_source_id}",
             )
         return None, None
 

datahub/ingestion/source/nifi.py

@@ -488,7 +488,7 @@ class NifiSource(Source):
     def get_report(self) -> SourceReport:
         return self.report
 
-    def update_flow(self, pg_flow_dto: Dict, recursion_level: int = 0) -> None:  # noqa: C901
+    def update_flow(self, pg_flow_dto: Dict, recursion_level: int = 0) -> None:
         """
         Update self.nifi_flow with contents of the input process group `pg_flow_dto`
         """
@@ -894,7 +894,7 @@ class NifiSource(Source):
         if not delete_response.ok:
             logger.error("failed to delete provenance ", provenance_uri)
 
-    def construct_workunits(self) -> Iterable[MetadataWorkUnit]:  # noqa: C901
+    def construct_workunits(self) -> Iterable[MetadataWorkUnit]:
         rootpg = self.nifi_flow.root_process_group
         flow_name = rootpg.name  # self.config.site_name
         flow_urn = self.make_flow_urn()

datahub/ingestion/source/openapi.py

@@ -270,7 +270,7 @@ class APISource(Source, ABC):
         mce = MetadataChangeEvent(proposedSnapshot=dataset_snapshot)
         return ApiWorkUnit(id=dataset_name, mce=mce)
 
-    def get_workunits_internal(self) -> Iterable[ApiWorkUnit]:  # noqa: C901
+    def get_workunits_internal(self) -> Iterable[ApiWorkUnit]:
         config = self.config
 
         sw_dict = self.config.get_swagger()

datahub/ingestion/source/openapi_parser.py

@@ -111,7 +111,7 @@ def check_sw_version(sw_dict: dict) -> None:
         )
 
 
-def get_endpoints(sw_dict: dict) -> dict:  # noqa: C901
+def get_endpoints(sw_dict: dict) -> dict:
     """
     Get all the URLs, together with their description and the tags
     """

datahub/ingestion/source/powerbi_report_server/report_server_domain.py

@@ -33,7 +33,7 @@ class CatalogItem(BaseModel):
     )
 
     @validator("display_name", always=True)
-    def validate_diplay_name(cls, value, values):  # noqa: N805
+    def validate_diplay_name(cls, value, values):
         if values["created_by"]:
             return values["created_by"].split("\\")[-1]
         return ""

datahub/ingestion/source/sql/athena.py

@@ -55,7 +55,7 @@ try:
 except ImportError:
     _F = typing.TypeVar("_F", bound=typing.Callable[..., typing.Any])
 
-    def override(f: _F, /) -> _F:  # noqa: F811
+    def override(f: _F, /) -> _F:
         return f
 
 
@@ -104,7 +104,7 @@ class CustomAthenaRestDialect(AthenaRestDialect):
             return "\n".join([r for r in res])
 
     @typing.no_type_check
-    def _get_column_type(self, type_: Union[str, Dict[str, Any]]) -> TypeEngine:  # noqa: C901
+    def _get_column_type(self, type_: Union[str, Dict[str, Any]]) -> TypeEngine:
         """Derives the data type of the Athena column.
 
         This method is overwritten to extend the behavior of PyAthena.

datahub/ingestion/source/sql/hive_metastore.py

@@ -67,7 +67,7 @@ TableKey = namedtuple("TableKey", ["schema", "table"])
 
 
 class HiveMetastoreConfigMode(StrEnum):
-    hive: str = "hive"  # noqa: F811
+    hive: str = "hive"
     presto: str = "presto"
     presto_on_hive: str = "presto-on-hive"
     trino: str = "trino"

datahub/ingestion/source/sql/mssql/source.py

@@ -401,7 +401,7 @@ class SQLServerSource(SQLAlchemySource):
                 data_job.add_property(name=data_name, value=str(data_value))
             yield from self.construct_job_workunits(data_job)
 
-    def loop_stored_procedures(  # noqa: C901
+    def loop_stored_procedures(
         self,
         inspector: Inspector,
         schema: str,

datahub/ingestion/source/sql/sql_common.py

@@ -635,7 +635,7 @@ class SQLAlchemySource(StatefulIngestionSourceBase, TestableSource):
 
         return None
 
-    def loop_tables(  # noqa: C901
+    def loop_tables(
         self,
         inspector: Inspector,
         schema: str,

datahub/ingestion/source/sql/teradata.py

@@ -649,7 +649,7 @@ ORDER by DataBaseName, TableName;
             )
 
             # Disabling the below because the cached view definition is not the view definition the column in tablesv actually holds the last statement executed against the object... not necessarily the view definition
-            # setattr(  # noqa: B010
+            # setattr(
             #   TeradataDialect,
             #    "get_view_definition",
             #   lambda self, connection, view_name, schema=None, **kw: optimized_get_view_definition(
@@ -746,7 +746,7 @@ ORDER by DataBaseName, TableName;
         else:
             raise Exception("Unable to get database name from Sqlalchemy inspector")
 
-    def cached_loop_tables(  # noqa: C901
+    def cached_loop_tables(
         self,
         inspector: Inspector,
         schema: str,
@@ -782,7 +782,7 @@ ORDER by DataBaseName, TableName;
                 break
         return description, properties, location
 
-    def cached_loop_views(  # noqa: C901
+    def cached_loop_views(
         self,
         inspector: Inspector,
         schema: str,

datahub/ingestion/source/tableau/tableau.py

@@ -2190,6 +2190,10 @@ class TableauSiteSource:
                 dataset_snapshot.aspects.append(browse_paths)
             else:
                 logger.debug(f"Browse path not set for Custom SQL table {csql_id}")
+                logger.warning(
+                    f"Skipping Custom SQL table {csql_id} due to filtered downstream"
+                )
+                continue
 
             dataset_properties = DatasetPropertiesClass(
                 name=csql.get(c.NAME),
@@ -2628,6 +2632,15 @@ class TableauSiteSource:
             datasource_info = datasource
 
         browse_path = self._get_project_browse_path_name(datasource)
+        if (
+            not is_embedded_ds
+            and self._get_published_datasource_project_luid(datasource) is None
+        ):
+            logger.warning(
+                f"Skip ingesting published datasource {datasource.get(c.NAME)} because of filtered project"
+            )
+            return
+
         logger.debug(f"datasource {datasource.get(c.NAME)} browse-path {browse_path}")
         datasource_id = datasource[c.ID]
         datasource_urn = builder.make_dataset_urn_with_platform_instance(
@@ -2851,6 +2864,11 @@ class TableauSiteSource:
             query_filter=tables_filter,
             page_size=self.config.effective_database_table_page_size,
         ):
+            if tableau_database_table_id_to_urn_map.get(tableau_table[c.ID]) is None:
+                logger.warning(
+                    f"Skipping table {tableau_table[c.ID]} due to filtered out published datasource"
+                )
+                continue
             database_table = self.database_tables[
                 tableau_database_table_id_to_urn_map[tableau_table[c.ID]]
             ]
@@ -2905,6 +2923,7 @@ class TableauSiteSource:
             dataset_snapshot.aspects.append(browse_paths)
         else:
             logger.debug(f"Browse path not set for table {database_table.urn}")
+            return
 
         schema_metadata = self.get_schema_metadata_for_table(
             tableau_columns, database_table.parsed_columns

datahub/ingestion/source/unity/source.py

@@ -464,7 +464,17 @@ class UnityCatalogSource(StatefulIngestionSourceBase, TestableSource):
 
             with self.report.new_stage(f"Ingest schema {schema.id}"):
                 yield from self.gen_schema_containers(schema)
-                yield from self.process_tables(schema)
+                try:
+                    yield from self.process_tables(schema)
+                except Exception as e:
+                    logger.exception(f"Error parsing schema {schema}")
+                    self.report.report_warning(
+                        message="Missed schema because of parsing issues",
+                        context=str(schema),
+                        title="Error parsing schema",
+                        exc=e,
+                    )
+                    continue
 
                 self.report.schemas.processed(schema.id)
 

datahub/metadata/schema.avsc

@@ -3504,7 +3504,9 @@
                   "fieldName": "glossaryTerms",
                   "fieldType": "URN",
                   "filterNameOverride": "Glossary Term",
-                  "hasValuesFieldName": "hasGlossaryTerms"
+                  "hasValuesFieldName": "hasGlossaryTerms",
+                  "includeSystemModifiedAt": true,
+                  "systemModifiedAtFieldName": "termsModifiedAt"
                 },
                 "java": {
                   "class": "com.linkedin.pegasus2avro.common.urn.GlossaryTermUrn"
@@ -10339,7 +10341,9 @@
           "/terms/*/urn": {
             "boostScore": 0.5,
             "fieldName": "editedFieldGlossaryTerms",
-            "fieldType": "URN"
+            "fieldType": "URN",
+            "includeSystemModifiedAt": true,
+            "systemModifiedAtFieldName": "schemaFieldTermsModifiedAt"
           }
         },
         "type": [
@@ -10477,6 +10481,12 @@
             "namespace": "com.linkedin.pegasus2avro.businessattribute",
             "fields": [
               {
+                "Searchable": {
+                  "fieldName": "schemaFieldBusinessAttribute",
+                  "includeSystemModifiedAt": true,
+                  "queryByDefault": false,
+                  "systemModifiedAtFieldName": "schemaFieldBusinessAttributeModifiedAt"
+                },
                 "java": {
                   "class": "com.linkedin.pegasus2avro.common.urn.BusinessAttributeUrn"
                 },
@@ -15077,6 +15087,7 @@
       },
       {
         "Searchable": {
+          "boostScore": 10.0,
           "enableAutocomplete": true,
           "fieldNameAliases": [
             "_entityName"
@@ -16546,7 +16557,9 @@
                                     "/terms/*/urn": {
                                       "boostScore": 0.5,
                                       "fieldName": "editedFieldGlossaryTerms",
-                                      "fieldType": "URN"
+                                      "fieldType": "URN",
+                                      "includeSystemModifiedAt": true,
+                                      "systemModifiedAtFieldName": "schemaFieldTermsModifiedAt"
                                     }
                                   },
                                   "type": [
@@ -19423,6 +19436,13 @@
         "doc": "The display name of the property. This is the name that will be shown in the UI and can be used to look up the property id."
       },
       {
+        "UrnValidation": {
+          "entityTypes": [
+            "dataType"
+          ],
+          "exist": true,
+          "strict": true
+        },
         "java": {
           "class": "com.linkedin.pegasus2avro.common.urn.Urn"
         },
@@ -19511,6 +19531,13 @@
             "fieldName": "entityTypes"
           }
         },
+        "UrnValidation": {
+          "entityTypes": [
+            "entityType"
+          ],
+          "exist": true,
+          "strict": true
+        },
         "Urn": "Urn",
         "urn_is_array": true,
         "type": {

datahub/metadata/schemas/AssertionInfo.avsc

@@ -2010,7 +2010,9 @@
                                               "fieldName": "glossaryTerms",
                                               "fieldType": "URN",
                                               "filterNameOverride": "Glossary Term",
-                                              "hasValuesFieldName": "hasGlossaryTerms"
+                                              "hasValuesFieldName": "hasGlossaryTerms",
+                                              "includeSystemModifiedAt": true,
+                                              "systemModifiedAtFieldName": "termsModifiedAt"
                                             },
                                             "java": {
                                               "class": "com.linkedin.pegasus2avro.common.urn.GlossaryTermUrn"

datahub/metadata/schemas/BusinessAttributeInfo.avsc

@@ -221,7 +221,9 @@
         "/terms/*/urn": {
           "boostScore": 0.5,
           "fieldName": "editedFieldGlossaryTerms",
-          "fieldType": "URN"
+          "fieldType": "URN",
+          "includeSystemModifiedAt": true,
+          "systemModifiedAtFieldName": "schemaFieldTermsModifiedAt"
         }
       },
       "type": [
@@ -254,7 +256,9 @@
                         "fieldName": "glossaryTerms",
                         "fieldType": "URN",
                         "filterNameOverride": "Glossary Term",
-                        "hasValuesFieldName": "hasGlossaryTerms"
+                        "hasValuesFieldName": "hasGlossaryTerms",
+                        "includeSystemModifiedAt": true,
+                        "systemModifiedAtFieldName": "termsModifiedAt"
                       },
                       "java": {
                         "class": "com.linkedin.pegasus2avro.common.urn.GlossaryTermUrn"

datahub/metadata/schemas/BusinessAttributes.avsc

@@ -31,6 +31,12 @@
           "namespace": "com.linkedin.pegasus2avro.businessattribute",
           "fields": [
             {
+              "Searchable": {
+                "fieldName": "schemaFieldBusinessAttribute",
+                "includeSystemModifiedAt": true,
+                "queryByDefault": false,
+                "systemModifiedAtFieldName": "schemaFieldBusinessAttributeModifiedAt"
+              },
               "java": {
                 "class": "com.linkedin.pegasus2avro.common.urn.BusinessAttributeUrn"
               },

datahub/metadata/schemas/ChartInfo.avsc

@@ -39,6 +39,7 @@
     },
     {
       "Searchable": {
+        "boostScore": 10.0,
         "enableAutocomplete": true,
         "fieldNameAliases": [
           "_entityName"

datahub/metadata/schemas/EditableSchemaMetadata.avsc

@@ -303,7 +303,9 @@
                 "/terms/*/urn": {
                   "boostScore": 0.5,
                   "fieldName": "editedFieldGlossaryTerms",
-                  "fieldType": "URN"
+                  "fieldType": "URN",
+                  "includeSystemModifiedAt": true,
+                  "systemModifiedAtFieldName": "schemaFieldTermsModifiedAt"
                 }
               },
               "type": [
@@ -336,7 +338,9 @@
                                 "fieldName": "glossaryTerms",
                                 "fieldType": "URN",
                                 "filterNameOverride": "Glossary Term",
-                                "hasValuesFieldName": "hasGlossaryTerms"
+                                "hasValuesFieldName": "hasGlossaryTerms",
+                                "includeSystemModifiedAt": true,
+                                "systemModifiedAtFieldName": "termsModifiedAt"
                               },
                               "java": {
                                 "class": "com.linkedin.pegasus2avro.common.urn.GlossaryTermUrn"

datahub/metadata/schemas/GlossaryTerms.avsc

@@ -26,7 +26,9 @@
                 "fieldName": "glossaryTerms",
                 "fieldType": "URN",
                 "filterNameOverride": "Glossary Term",
-                "hasValuesFieldName": "hasGlossaryTerms"
+                "hasValuesFieldName": "hasGlossaryTerms",
+                "includeSystemModifiedAt": true,
+                "systemModifiedAtFieldName": "termsModifiedAt"
               },
               "java": {
                 "class": "com.linkedin.pegasus2avro.common.urn.GlossaryTermUrn"

datahub/metadata/schemas/InputFields.avsc

@@ -553,7 +553,9 @@
                                         "fieldName": "glossaryTerms",
                                         "fieldType": "URN",
                                         "filterNameOverride": "Glossary Term",
-                                        "hasValuesFieldName": "hasGlossaryTerms"
+                                        "hasValuesFieldName": "hasGlossaryTerms",
+                                        "includeSystemModifiedAt": true,
+                                        "systemModifiedAtFieldName": "termsModifiedAt"
                                       },
                                       "java": {
                                         "class": "com.linkedin.pegasus2avro.common.urn.GlossaryTermUrn"

datahub/metadata/schemas/MetadataChangeEvent.avsc

@@ -183,6 +183,7 @@
                       },
                       {
                         "Searchable": {
+                          "boostScore": 10.0,
                           "enableAutocomplete": true,
                           "fieldNameAliases": [
                             "_entityName"
@@ -994,7 +995,9 @@
                                   "fieldName": "glossaryTerms",
                                   "fieldType": "URN",
                                   "filterNameOverride": "Glossary Term",
-                                  "hasValuesFieldName": "hasGlossaryTerms"
+                                  "hasValuesFieldName": "hasGlossaryTerms",
+                                  "includeSystemModifiedAt": true,
+                                  "systemModifiedAtFieldName": "termsModifiedAt"
                                 },
                                 "java": {
                                   "class": "com.linkedin.pegasus2avro.common.urn.GlossaryTermUrn"
@@ -4644,7 +4647,9 @@
                                   "/terms/*/urn": {
                                     "boostScore": 0.5,
                                     "fieldName": "editedFieldGlossaryTerms",
-                                    "fieldType": "URN"
+                                    "fieldType": "URN",
+                                    "includeSystemModifiedAt": true,
+                                    "systemModifiedAtFieldName": "schemaFieldTermsModifiedAt"
                                   }
                                 },
                                 "type": [

datahub/metadata/schemas/SchemaMetadata.avsc

@@ -777,7 +777,9 @@
                                 "fieldName": "glossaryTerms",
                                 "fieldType": "URN",
                                 "filterNameOverride": "Glossary Term",
-                                "hasValuesFieldName": "hasGlossaryTerms"
+                                "hasValuesFieldName": "hasGlossaryTerms",
+                                "includeSystemModifiedAt": true,
+                                "systemModifiedAtFieldName": "termsModifiedAt"
                               },
                               "java": {
                                 "class": "com.linkedin.pegasus2avro.common.urn.GlossaryTermUrn"

datahub/metadata/schemas/StructuredPropertyDefinition.avsc

@@ -23,6 +23,13 @@
       "doc": "The display name of the property. This is the name that will be shown in the UI and can be used to look up the property id."
     },
     {
+      "UrnValidation": {
+        "entityTypes": [
+          "dataType"
+        ],
+        "exist": true,
+        "strict": true
+      },
       "java": {
         "class": "com.linkedin.pegasus2avro.common.urn.Urn"
       },
@@ -111,6 +118,13 @@
           "fieldName": "entityTypes"
         }
       },
+      "UrnValidation": {
+        "entityTypes": [
+          "entityType"
+        ],
+        "exist": true,
+        "strict": true
+      },
       "type": {
         "type": "array",
         "items": "string"

datahub/sql_parsing/schema_resolver.py

@@ -13,7 +13,7 @@ from datahub.ingestion.graph.client import DataHubGraph
 from datahub.ingestion.source.bigquery_v2.bigquery_audit import BigqueryTableIdentifier
 from datahub.metadata.schema_classes import SchemaFieldClass, SchemaMetadataClass
 from datahub.metadata.urns import DataPlatformUrn
-from datahub.sql_parsing._models import _TableName as _TableName  # noqa: I250
+from datahub.sql_parsing._models import _TableName as _TableName
 from datahub.sql_parsing.sql_parsing_common import PLATFORMS_WITH_CASE_SENSITIVE_TABLES
 from datahub.utilities.file_backed_collections import ConnectionWrapper, FileBackedDict
 from datahub.utilities.urns.field_paths import get_simple_field_path_from_v2_field_path

datahub/sql_parsing/sqlglot_lineage.py

@@ -473,7 +473,7 @@ def _create_table_ddl_cll(
     return column_lineage
 
 
-def _select_statement_cll(  # noqa: C901
+def _select_statement_cll(
     statement: _SupportedColumnLineageTypes,
     dialect: sqlglot.Dialect,
     root_scope: sqlglot.optimizer.Scope,

datahub/upgrade/upgrade.py

@@ -293,9 +293,9 @@ def is_client_server_compatible(client: VersionStats, server: VersionStats) -> i
         return server.version.micro - client.version.micro
 
 
-def _maybe_print_upgrade_message(  # noqa: C901
+def _maybe_print_upgrade_message(
     version_stats: Optional[DataHubVersionStats],
-) -> None:  # noqa: C901
+) -> None:
     days_before_cli_stale = 7
     days_before_quickstart_stale = 7
 

datahub/utilities/ingest_utils.py

@@ -0,0 +1,106 @@
+import json
+import logging
+from typing import Optional
+
+import click
+
+from datahub.configuration.common import ConfigModel
+from datahub.configuration.config_loader import load_config_file
+from datahub.emitter.mce_builder import datahub_guid
+
+logger = logging.getLogger(__name__)
+
+
+def _make_ingestion_urn(name: str) -> str:
+    guid = datahub_guid(
+        {
+            "name": name,
+        }
+    )
+    return f"urn:li:dataHubIngestionSource:deploy-{guid}"
+
+
+class DeployOptions(ConfigModel):
+    name: str
+    schedule: Optional[str] = None
+    time_zone: str = "UTC"
+    cli_version: Optional[str] = None
+    executor_id: str = "default"
+
+
+def deploy_source_vars(
+    name: Optional[str],
+    config: str,
+    urn: Optional[str],
+    executor_id: str,
+    cli_version: Optional[str],
+    schedule: Optional[str],
+    time_zone: str,
+    extra_pip: Optional[str],
+    debug: bool = False,
+) -> dict:
+    pipeline_config = load_config_file(
+        config,
+        allow_stdin=True,
+        allow_remote=True,
+        resolve_env_vars=False,
+    )
+
+    deploy_options_raw = pipeline_config.pop("deployment", None)
+    if deploy_options_raw is not None:
+        deploy_options = DeployOptions.parse_obj(deploy_options_raw)
+
+        if name:
+            logger.info(f"Overriding deployment name {deploy_options.name} with {name}")
+            deploy_options.name = name
+    else:
+        if not name:
+            raise click.UsageError(
+                "Either --name must be set or deployment_name specified in the config"
+            )
+        deploy_options = DeployOptions(name=name)
+
+    # Use remaining CLI args to override deploy_options
+    if schedule:
+        deploy_options.schedule = schedule
+    if time_zone:
+        deploy_options.time_zone = time_zone
+    if cli_version:
+        deploy_options.cli_version = cli_version
+    if executor_id:
+        deploy_options.executor_id = executor_id
+
+    logger.info(f"Using {repr(deploy_options)}")
+
+    if not urn:
+        # When urn/name is not specified, we will generate a unique urn based on the deployment name.
+        urn = _make_ingestion_urn(deploy_options.name)
+        logger.info(f"Using recipe urn: {urn}")
+
+    variables: dict = {
+        "urn": urn,
+        "input": {
+            "name": deploy_options.name,
+            "type": pipeline_config["source"]["type"],
+            "config": {
+                "recipe": json.dumps(pipeline_config),
+                "executorId": deploy_options.executor_id,
+                "debugMode": debug,
+                "version": deploy_options.cli_version,
+            },
+        },
+    }
+
+    if deploy_options.schedule is not None:
+        variables["input"]["schedule"] = {
+            "interval": deploy_options.schedule,
+            "timezone": deploy_options.time_zone,
+        }
+    if extra_pip is not None:
+        extra_args_list = (
+            variables.get("input", {}).get("config", {}).get("extraArgs", [])
+        )
+        extra_args_list.append({"key": "extra_pip_requirements", "value": extra_pip})
+        variables["input"]["config"]["extraArgs"] = extra_args_list
+
+    return variables

datahub/utilities/mapping.py

@@ -171,7 +171,7 @@ class OperationProcessor:
         self.owner_source_type = owner_source_type
         self.match_nested_props = match_nested_props
 
-    def process(self, raw_props: Mapping[str, Any]) -> Dict[str, Any]:  # noqa: C901
+    def process(self, raw_props: Mapping[str, Any]) -> Dict[str, Any]:
         # Defining the following local variables -
         # operations_map - the final resulting map when operations are processed.
         # Against each operation the values to be applied are stored.