acryl-datahub 0.15.0.1rc5__py3-none-any.whl → 0.15.0.1rc7__py3-none-any.whl

datahub/metadata/schemas/MetadataChangeEvent.avsc

@@ -5109,6 +5109,21 @@
                         "default": null,
                         "doc": "URL where the reference exist"
                       },
+                      {
+                        "Searchable": {
+                          "boostScore": 10.0,
+                          "enableAutocomplete": true,
+                          "fieldType": "WORD_GRAM",
+                          "queryByDefault": true
+                        },
+                        "type": [
+                          "null",
+                          "string"
+                        ],
+                        "name": "name",
+                        "default": null,
+                        "doc": "Display name of the MLModel"
+                      },
                       {
                         "Searchable": {
                           "fieldType": "TEXT",
@@ -5123,6 +5138,7 @@
                         "doc": "Documentation of the MLModel"
                       },
                       {
+                        "deprecated": true,
                         "type": [
                           "null",
                           "long"
@@ -5131,6 +5147,24 @@
                         "default": null,
                         "doc": "Date when the MLModel was developed"
                       },
+                      {
+                        "type": [
+                          "null",
+                          "com.linkedin.pegasus2avro.common.TimeStamp"
+                        ],
+                        "name": "created",
+                        "default": null,
+                        "doc": "Audit stamp containing who created this and when"
+                      },
+                      {
+                        "type": [
+                          "null",
+                          "com.linkedin.pegasus2avro.common.TimeStamp"
+                        ],
+                        "name": "lastModified",
+                        "default": null,
+                        "doc": "Date when the MLModel was last modified"
+                      },
                       {
                         "type": [
                           "null",
@@ -5363,7 +5397,8 @@
                         "Relationship": {
                           "/*": {
                             "entityTypes": [
-                              "dataJob"
+                              "dataJob",
+                              "dataProcessInstance"
                             ],
                             "isLineage": true,
                             "name": "TrainedBy"
@@ -5378,7 +5413,7 @@
                         ],
                         "name": "trainingJobs",
                         "default": null,
-                        "doc": "List of jobs (if any) used to train the model",
+                        "doc": "List of jobs (if any) used to train the model. Visible in Lineage. Note that ML Models can also be specified as the output of a specific Data Process Instances (runs) via the DataProcessInstanceOutputs aspect.",
                         "Urn": "Urn",
                         "urn_is_array": true
                       },
@@ -6804,6 +6839,21 @@
                         "default": {},
                         "doc": "Custom property bag."
                       },
+                      {
+                        "Searchable": {
+                          "boostScore": 10.0,
+                          "enableAutocomplete": true,
+                          "fieldType": "WORD_GRAM",
+                          "queryByDefault": true
+                        },
+                        "type": [
+                          "null",
+                          "string"
+                        ],
+                        "name": "name",
+                        "default": null,
+                        "doc": "Display name of the MLModelGroup"
+                      },
                       {
                         "Searchable": {
                           "fieldType": "TEXT",
@@ -6818,6 +6868,7 @@
                         "doc": "Documentation of the MLModelGroup"
                       },
                       {
+                        "deprecated": true,
                         "type": [
                           "null",
                           "long"
@@ -6826,6 +6877,47 @@
                         "default": null,
                         "doc": "Date when the MLModelGroup was developed"
                       },
+                      {
+                        "type": [
+                          "null",
+                          "com.linkedin.pegasus2avro.common.TimeStamp"
+                        ],
+                        "name": "created",
+                        "default": null,
+                        "doc": "Time and Actor who created the MLModelGroup"
+                      },
+                      {
+                        "type": [
+                          "null",
+                          "com.linkedin.pegasus2avro.common.TimeStamp"
+                        ],
+                        "name": "lastModified",
+                        "default": null,
+                        "doc": "Date when the MLModelGroup was last modified"
+                      },
+                      {
+                        "Relationship": {
+                          "/*": {
+                            "entityTypes": [
+                              "dataJob"
+                            ],
+                            "isLineage": true,
+                            "name": "TrainedBy"
+                          }
+                        },
+                        "type": [
+                          "null",
+                          {
+                            "type": "array",
+                            "items": "string"
+                          }
+                        ],
+                        "name": "trainingJobs",
+                        "default": null,
+                        "doc": "List of jobs (if any) used to train the model group. Visible in Lineage.",
+                        "Urn": "Urn",
+                        "urn_is_array": true
+                      },
                       {
                         "type": [
                           "null",

datahub/specific/dataproduct.py

@@ -131,7 +131,7 @@ class DataProductPatchBuilder(MetadataPatchProposal):
         self._add_patch(
             DataProductProperties.ASPECT_NAME,
             "add",
-            path=f"/assets/{asset_urn}",
+            path=f"/assets/{self.quote(asset_urn)}",
             value=DataProductAssociation(destinationUrn=asset_urn),
         )
         return self
@@ -140,7 +140,7 @@ class DataProductPatchBuilder(MetadataPatchProposal):
         self._add_patch(
             DataProductProperties.ASPECT_NAME,
             "remove",
-            path=f"/assets/{asset_urn}",
+            path=f"/assets/{self.quote(asset_urn)}",
             value={},
         )
         return self

datahub/sql_parsing/sqlglot_lineage.py

@@ -66,6 +66,7 @@ SQL_LINEAGE_TIMEOUT_ENABLED = get_boolean_env_variable(
     "SQL_LINEAGE_TIMEOUT_ENABLED", True
 )
 SQL_LINEAGE_TIMEOUT_SECONDS = 10
+SQL_PARSER_TRACE = get_boolean_env_variable("DATAHUB_SQL_PARSER_TRACE", False)
 
 
 # These rules are a subset of the rules in sqlglot.optimizer.optimizer.RULES.
@@ -365,10 +366,11 @@ def _prepare_query_columns(
 
             return node
 
-        # logger.debug(
-        #     "Prior to case normalization sql %s",
-        #     statement.sql(pretty=True, dialect=dialect),
-        # )
+        if SQL_PARSER_TRACE:
+            logger.debug(
+                "Prior to case normalization sql %s",
+                statement.sql(pretty=True, dialect=dialect),
+            )
         statement = statement.transform(_sqlglot_force_column_normalizer, copy=False)
         # logger.debug(
         #     "Sql after casing normalization %s",
@@ -562,7 +564,7 @@ def _select_statement_cll(  # noqa: C901
                 )
             )
 
-        # TODO: Also extract referenced columns (aka auxillary / non-SELECT lineage)
+        # TODO: Also extract referenced columns (aka auxiliary / non-SELECT lineage)
     except (sqlglot.errors.OptimizeError, ValueError, IndexError) as e:
         raise SqlUnderstandingError(
             f"sqlglot failed to compute some lineage: {e}"
@@ -1022,6 +1024,14 @@ def _sqlglot_lineage_inner(
     logger.debug(
         f"Resolved {total_schemas_resolved} of {total_tables_discovered} table schemas"
     )
+    if SQL_PARSER_TRACE:
+        for qualified_table, schema_info in table_name_schema_mapping.items():
+            logger.debug(
+                "Table name %s resolved to %s with schema %s",
+                qualified_table,
+                table_name_urn_mapping[qualified_table],
+                schema_info,
+            )
 
     column_lineage: Optional[List[_ColumnLineageInfo]] = None
     try: