acryl-datahub 1.2.0.3rc2__py3-none-any.whl → 1.2.0.4rc1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of acryl-datahub might be problematic. Click here for more details.
- {acryl_datahub-1.2.0.3rc2.dist-info → acryl_datahub-1.2.0.4rc1.dist-info}/METADATA +2522 -2522
- {acryl_datahub-1.2.0.3rc2.dist-info → acryl_datahub-1.2.0.4rc1.dist-info}/RECORD +32 -32
- datahub/_version.py +1 -1
- datahub/api/entities/external/external_tag.py +6 -4
- datahub/api/entities/external/lake_formation_external_entites.py +50 -49
- datahub/api/entities/external/restricted_text.py +107 -182
- datahub/api/entities/external/unity_catalog_external_entites.py +51 -52
- datahub/ingestion/api/source.py +81 -7
- datahub/ingestion/autogenerated/capability_summary.json +47 -19
- datahub/ingestion/source/abs/source.py +9 -0
- datahub/ingestion/source/aws/glue.py +18 -2
- datahub/ingestion/source/aws/tag_entities.py +2 -2
- datahub/ingestion/source/datahub/datahub_source.py +8 -1
- datahub/ingestion/source/delta_lake/source.py +8 -1
- datahub/ingestion/source/dremio/dremio_source.py +19 -2
- datahub/ingestion/source/fivetran/fivetran.py +9 -3
- datahub/ingestion/source/ge_data_profiler.py +8 -0
- datahub/ingestion/source/looker/looker_liquid_tag.py +56 -5
- datahub/ingestion/source/mock_data/datahub_mock_data.py +26 -10
- datahub/ingestion/source/powerbi/powerbi.py +4 -1
- datahub/ingestion/source/redshift/redshift.py +1 -0
- datahub/ingestion/source/salesforce.py +8 -0
- datahub/ingestion/source/sql/hive_metastore.py +8 -0
- datahub/ingestion/source/sql/teradata.py +8 -1
- datahub/ingestion/source/sql/trino.py +9 -0
- datahub/ingestion/source/unity/tag_entities.py +3 -3
- datahub/sdk/entity_client.py +22 -7
- datahub/utilities/mapping.py +29 -2
- {acryl_datahub-1.2.0.3rc2.dist-info → acryl_datahub-1.2.0.4rc1.dist-info}/WHEEL +0 -0
- {acryl_datahub-1.2.0.3rc2.dist-info → acryl_datahub-1.2.0.4rc1.dist-info}/entry_points.txt +0 -0
- {acryl_datahub-1.2.0.3rc2.dist-info → acryl_datahub-1.2.0.4rc1.dist-info}/licenses/LICENSE +0 -0
- {acryl_datahub-1.2.0.3rc2.dist-info → acryl_datahub-1.2.0.4rc1.dist-info}/top_level.txt +0 -0
|
@@ -1,9 +1,18 @@
|
|
|
1
1
|
{
|
|
2
|
-
"generated_at": "2025-07-
|
|
2
|
+
"generated_at": "2025-07-31T12:54:30.557618+00:00",
|
|
3
3
|
"generated_by": "metadata-ingestion/scripts/capability_summary.py",
|
|
4
4
|
"plugin_details": {
|
|
5
5
|
"abs": {
|
|
6
6
|
"capabilities": [
|
|
7
|
+
{
|
|
8
|
+
"capability": "CONTAINERS",
|
|
9
|
+
"description": "Extract ABS containers and folders",
|
|
10
|
+
"subtype_modifier": [
|
|
11
|
+
"Folder",
|
|
12
|
+
"ABS container"
|
|
13
|
+
],
|
|
14
|
+
"supported": true
|
|
15
|
+
},
|
|
7
16
|
{
|
|
8
17
|
"capability": "DATA_PROFILING",
|
|
9
18
|
"description": "Optionally enabled via configuration",
|
|
@@ -468,7 +477,9 @@
|
|
|
468
477
|
{
|
|
469
478
|
"capability": "CONTAINERS",
|
|
470
479
|
"description": "Enabled by default",
|
|
471
|
-
"subtype_modifier":
|
|
480
|
+
"subtype_modifier": [
|
|
481
|
+
"Database"
|
|
482
|
+
],
|
|
472
483
|
"supported": true
|
|
473
484
|
},
|
|
474
485
|
{
|
|
@@ -531,13 +542,6 @@
|
|
|
531
542
|
"platform_name": "File Based Lineage",
|
|
532
543
|
"support_status": "CERTIFIED"
|
|
533
544
|
},
|
|
534
|
-
"datahub-mock-data": {
|
|
535
|
-
"capabilities": [],
|
|
536
|
-
"classname": "datahub.ingestion.source.mock_data.datahub_mock_data.DataHubMockDataSource",
|
|
537
|
-
"platform_id": "datahubmockdata",
|
|
538
|
-
"platform_name": "DataHubMockData",
|
|
539
|
-
"support_status": "TESTING"
|
|
540
|
-
},
|
|
541
545
|
"dbt": {
|
|
542
546
|
"capabilities": [
|
|
543
547
|
{
|
|
@@ -607,7 +611,9 @@
|
|
|
607
611
|
{
|
|
608
612
|
"capability": "CONTAINERS",
|
|
609
613
|
"description": "Enabled by default",
|
|
610
|
-
"subtype_modifier":
|
|
614
|
+
"subtype_modifier": [
|
|
615
|
+
"Folder"
|
|
616
|
+
],
|
|
611
617
|
"supported": true
|
|
612
618
|
},
|
|
613
619
|
{
|
|
@@ -643,6 +649,14 @@
|
|
|
643
649
|
"subtype_modifier": null,
|
|
644
650
|
"supported": true
|
|
645
651
|
},
|
|
652
|
+
{
|
|
653
|
+
"capability": "LINEAGE_FINE",
|
|
654
|
+
"description": "Extract column-level lineage",
|
|
655
|
+
"subtype_modifier": [
|
|
656
|
+
"Table"
|
|
657
|
+
],
|
|
658
|
+
"supported": true
|
|
659
|
+
},
|
|
646
660
|
{
|
|
647
661
|
"capability": "DATA_PROFILING",
|
|
648
662
|
"description": "Optionally enabled via configuration",
|
|
@@ -688,7 +702,9 @@
|
|
|
688
702
|
{
|
|
689
703
|
"capability": "LINEAGE_COARSE",
|
|
690
704
|
"description": "Enabled by default",
|
|
691
|
-
"subtype_modifier":
|
|
705
|
+
"subtype_modifier": [
|
|
706
|
+
"Table"
|
|
707
|
+
],
|
|
692
708
|
"supported": true
|
|
693
709
|
}
|
|
694
710
|
],
|
|
@@ -1229,8 +1245,7 @@
|
|
|
1229
1245
|
"capability": "CONTAINERS",
|
|
1230
1246
|
"description": "Enabled by default",
|
|
1231
1247
|
"subtype_modifier": [
|
|
1232
|
-
"
|
|
1233
|
-
"Schema"
|
|
1248
|
+
"Catalog"
|
|
1234
1249
|
],
|
|
1235
1250
|
"supported": true
|
|
1236
1251
|
},
|
|
@@ -2387,8 +2402,9 @@
|
|
|
2387
2402
|
},
|
|
2388
2403
|
{
|
|
2389
2404
|
"capability": "LINEAGE_COARSE",
|
|
2390
|
-
"description": "
|
|
2405
|
+
"description": "Extract table-level lineage",
|
|
2391
2406
|
"subtype_modifier": [
|
|
2407
|
+
"Table",
|
|
2392
2408
|
"View"
|
|
2393
2409
|
],
|
|
2394
2410
|
"supported": true
|
|
@@ -2411,8 +2427,7 @@
|
|
|
2411
2427
|
"capability": "CONTAINERS",
|
|
2412
2428
|
"description": "Enabled by default",
|
|
2413
2429
|
"subtype_modifier": [
|
|
2414
|
-
"
|
|
2415
|
-
"Schema"
|
|
2430
|
+
"Catalog"
|
|
2416
2431
|
],
|
|
2417
2432
|
"supported": true
|
|
2418
2433
|
},
|
|
@@ -2598,7 +2613,8 @@
|
|
|
2598
2613
|
"capability": "CONTAINERS",
|
|
2599
2614
|
"description": "Enabled by default",
|
|
2600
2615
|
"subtype_modifier": [
|
|
2601
|
-
"Database"
|
|
2616
|
+
"Database",
|
|
2617
|
+
"Schema"
|
|
2602
2618
|
],
|
|
2603
2619
|
"supported": true
|
|
2604
2620
|
},
|
|
@@ -2812,6 +2828,15 @@
|
|
|
2812
2828
|
"description": "Enabled by default",
|
|
2813
2829
|
"subtype_modifier": null,
|
|
2814
2830
|
"supported": true
|
|
2831
|
+
},
|
|
2832
|
+
{
|
|
2833
|
+
"capability": "LINEAGE_COARSE",
|
|
2834
|
+
"description": "Extract table-level lineage for Salesforce objects",
|
|
2835
|
+
"subtype_modifier": [
|
|
2836
|
+
"Custom Object",
|
|
2837
|
+
"Object"
|
|
2838
|
+
],
|
|
2839
|
+
"supported": true
|
|
2815
2840
|
}
|
|
2816
2841
|
],
|
|
2817
2842
|
"classname": "datahub.ingestion.source.salesforce.SalesforceSource",
|
|
@@ -3207,7 +3232,9 @@
|
|
|
3207
3232
|
{
|
|
3208
3233
|
"capability": "CONTAINERS",
|
|
3209
3234
|
"description": "Enabled by default",
|
|
3210
|
-
"subtype_modifier":
|
|
3235
|
+
"subtype_modifier": [
|
|
3236
|
+
"Database"
|
|
3237
|
+
],
|
|
3211
3238
|
"supported": true
|
|
3212
3239
|
},
|
|
3213
3240
|
{
|
|
@@ -3339,8 +3366,9 @@
|
|
|
3339
3366
|
},
|
|
3340
3367
|
{
|
|
3341
3368
|
"capability": "LINEAGE_COARSE",
|
|
3342
|
-
"description": "
|
|
3369
|
+
"description": "Extract table-level lineage",
|
|
3343
3370
|
"subtype_modifier": [
|
|
3371
|
+
"Table",
|
|
3344
3372
|
"View"
|
|
3345
3373
|
],
|
|
3346
3374
|
"supported": true
|
|
@@ -44,6 +44,7 @@ from datahub.ingestion.source.azure.abs_utils import (
|
|
|
44
44
|
get_key_prefix,
|
|
45
45
|
strip_abs_prefix,
|
|
46
46
|
)
|
|
47
|
+
from datahub.ingestion.source.common.subtypes import SourceCapabilityModifier
|
|
47
48
|
from datahub.ingestion.source.data_lake_common.data_lake_utils import (
|
|
48
49
|
ContainerWUCreator,
|
|
49
50
|
add_partition_columns_to_schema,
|
|
@@ -128,6 +129,14 @@ class TableData:
|
|
|
128
129
|
@support_status(SupportStatus.INCUBATING)
|
|
129
130
|
@capability(SourceCapability.DATA_PROFILING, "Optionally enabled via configuration")
|
|
130
131
|
@capability(SourceCapability.TAGS, "Can extract ABS object/container tags if enabled")
|
|
132
|
+
@capability(
|
|
133
|
+
SourceCapability.CONTAINERS,
|
|
134
|
+
"Extract ABS containers and folders",
|
|
135
|
+
subtype_modifier=[
|
|
136
|
+
SourceCapabilityModifier.FOLDER,
|
|
137
|
+
SourceCapabilityModifier.ABS_CONTAINER,
|
|
138
|
+
],
|
|
139
|
+
)
|
|
131
140
|
class ABSSource(StatefulIngestionSourceBase):
|
|
132
141
|
source_config: DataLakeSourceConfig
|
|
133
142
|
report: DataLakeSourceReport
|
|
@@ -395,7 +395,7 @@ class GlueSource(StatefulIngestionSourceBase):
|
|
|
395
395
|
t = LakeFormationTag(
|
|
396
396
|
key=tag_key,
|
|
397
397
|
value=tag_value,
|
|
398
|
-
|
|
398
|
+
catalog=catalog_id,
|
|
399
399
|
)
|
|
400
400
|
tags.append(t)
|
|
401
401
|
return tags
|
|
@@ -438,7 +438,7 @@ class GlueSource(StatefulIngestionSourceBase):
|
|
|
438
438
|
t = LakeFormationTag(
|
|
439
439
|
key=tag_key,
|
|
440
440
|
value=tag_value,
|
|
441
|
-
|
|
441
|
+
catalog=catalog_id,
|
|
442
442
|
)
|
|
443
443
|
tags.append(t)
|
|
444
444
|
return tags
|
|
@@ -522,6 +522,14 @@ class GlueSource(StatefulIngestionSourceBase):
|
|
|
522
522
|
bucket = url.netloc
|
|
523
523
|
key = url.path[1:]
|
|
524
524
|
|
|
525
|
+
# validate that we have a non-empty key
|
|
526
|
+
if not key:
|
|
527
|
+
self.report.num_job_script_location_invalid += 1
|
|
528
|
+
logger.warning(
|
|
529
|
+
f"Error parsing DAG for Glue job. The script {script_path} is not a valid S3 path for flow urn: {flow_urn}."
|
|
530
|
+
)
|
|
531
|
+
return None
|
|
532
|
+
|
|
525
533
|
# download the script contents
|
|
526
534
|
# see https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/s3.html#S3.Client.get_object
|
|
527
535
|
try:
|
|
@@ -533,6 +541,14 @@ class GlueSource(StatefulIngestionSourceBase):
|
|
|
533
541
|
)
|
|
534
542
|
self.report.num_job_script_failed_download += 1
|
|
535
543
|
return None
|
|
544
|
+
except botocore.exceptions.ParamValidationError as e:
|
|
545
|
+
self.report_warning(
|
|
546
|
+
flow_urn,
|
|
547
|
+
f"Invalid S3 path for Glue job script {script_path}: {e}",
|
|
548
|
+
)
|
|
549
|
+
self.report.num_job_script_location_invalid += 1
|
|
550
|
+
return None
|
|
551
|
+
|
|
536
552
|
script = obj["Body"].read().decode("utf-8")
|
|
537
553
|
|
|
538
554
|
try:
|
|
@@ -88,8 +88,8 @@ class LakeFormationTagPlatformResourceId(BaseModel, ExternalEntityId):
|
|
|
88
88
|
return existing_platform_resource
|
|
89
89
|
|
|
90
90
|
return LakeFormationTagPlatformResourceId(
|
|
91
|
-
tag_key=tag.key,
|
|
92
|
-
tag_value=tag.value if tag.value is not None else None,
|
|
91
|
+
tag_key=str(tag.key),
|
|
92
|
+
tag_value=str(tag.value) if tag.value is not None else None,
|
|
93
93
|
platform_instance=platform_instance,
|
|
94
94
|
exists_in_lake_formation=exists_in_lake_formation,
|
|
95
95
|
catalog=catalog,
|
|
@@ -19,6 +19,7 @@ from datahub.ingestion.api.source_helpers import (
|
|
|
19
19
|
auto_workunit_reporter,
|
|
20
20
|
)
|
|
21
21
|
from datahub.ingestion.api.workunit import MetadataWorkUnit
|
|
22
|
+
from datahub.ingestion.source.common.subtypes import SourceCapabilityModifier
|
|
22
23
|
from datahub.ingestion.source.datahub.config import DataHubSourceConfig
|
|
23
24
|
from datahub.ingestion.source.datahub.datahub_api_reader import DataHubApiReader
|
|
24
25
|
from datahub.ingestion.source.datahub.datahub_database_reader import (
|
|
@@ -39,7 +40,13 @@ logger = logging.getLogger(__name__)
|
|
|
39
40
|
@platform_name("DataHub")
|
|
40
41
|
@config_class(DataHubSourceConfig)
|
|
41
42
|
@support_status(SupportStatus.TESTING)
|
|
42
|
-
@capability(
|
|
43
|
+
@capability(
|
|
44
|
+
SourceCapability.CONTAINERS,
|
|
45
|
+
"Enabled by default",
|
|
46
|
+
subtype_modifier=[
|
|
47
|
+
SourceCapabilityModifier.DATABASE,
|
|
48
|
+
],
|
|
49
|
+
)
|
|
43
50
|
class DataHubSource(StatefulIngestionSourceBase):
|
|
44
51
|
platform: str = "datahub"
|
|
45
52
|
|
|
@@ -29,6 +29,7 @@ from datahub.ingestion.source.aws.s3_util import (
|
|
|
29
29
|
get_key_prefix,
|
|
30
30
|
strip_s3_prefix,
|
|
31
31
|
)
|
|
32
|
+
from datahub.ingestion.source.common.subtypes import SourceCapabilityModifier
|
|
32
33
|
from datahub.ingestion.source.data_lake_common.data_lake_utils import ContainerWUCreator
|
|
33
34
|
from datahub.ingestion.source.delta_lake.config import DeltaLakeSourceConfig
|
|
34
35
|
from datahub.ingestion.source.delta_lake.delta_lake_utils import (
|
|
@@ -85,7 +86,13 @@ OPERATION_STATEMENT_TYPES = {
|
|
|
85
86
|
@config_class(DeltaLakeSourceConfig)
|
|
86
87
|
@support_status(SupportStatus.INCUBATING)
|
|
87
88
|
@capability(SourceCapability.TAGS, "Can extract S3 object/bucket tags if enabled")
|
|
88
|
-
@capability(
|
|
89
|
+
@capability(
|
|
90
|
+
SourceCapability.CONTAINERS,
|
|
91
|
+
"Enabled by default",
|
|
92
|
+
subtype_modifier=[
|
|
93
|
+
SourceCapabilityModifier.FOLDER,
|
|
94
|
+
],
|
|
95
|
+
)
|
|
89
96
|
class DeltaLakeSource(StatefulIngestionSourceBase):
|
|
90
97
|
"""
|
|
91
98
|
This plugin extracts:
|
|
@@ -22,6 +22,7 @@ from datahub.ingestion.api.source import (
|
|
|
22
22
|
SourceReport,
|
|
23
23
|
)
|
|
24
24
|
from datahub.ingestion.api.workunit import MetadataWorkUnit
|
|
25
|
+
from datahub.ingestion.source.common.subtypes import SourceCapabilityModifier
|
|
25
26
|
from datahub.ingestion.source.dremio.dremio_api import (
|
|
26
27
|
DremioAPIOperations,
|
|
27
28
|
DremioEdition,
|
|
@@ -86,11 +87,27 @@ class DremioSourceMapEntry:
|
|
|
86
87
|
@platform_name("Dremio")
|
|
87
88
|
@config_class(DremioSourceConfig)
|
|
88
89
|
@support_status(SupportStatus.CERTIFIED)
|
|
89
|
-
@capability(
|
|
90
|
+
@capability(
|
|
91
|
+
SourceCapability.CONTAINERS,
|
|
92
|
+
"Enabled by default",
|
|
93
|
+
)
|
|
90
94
|
@capability(SourceCapability.DATA_PROFILING, "Optionally enabled via configuration")
|
|
91
95
|
@capability(SourceCapability.DESCRIPTIONS, "Enabled by default")
|
|
92
96
|
@capability(SourceCapability.DOMAINS, "Supported via the `domain` config field")
|
|
93
|
-
@capability(
|
|
97
|
+
@capability(
|
|
98
|
+
SourceCapability.LINEAGE_COARSE,
|
|
99
|
+
"Enabled by default",
|
|
100
|
+
subtype_modifier=[
|
|
101
|
+
SourceCapabilityModifier.TABLE,
|
|
102
|
+
],
|
|
103
|
+
)
|
|
104
|
+
@capability(
|
|
105
|
+
SourceCapability.LINEAGE_FINE,
|
|
106
|
+
"Extract column-level lineage",
|
|
107
|
+
subtype_modifier=[
|
|
108
|
+
SourceCapabilityModifier.TABLE,
|
|
109
|
+
],
|
|
110
|
+
)
|
|
94
111
|
@capability(SourceCapability.OWNERSHIP, "Enabled by default")
|
|
95
112
|
@capability(SourceCapability.PLATFORM_INSTANCE, "Enabled by default")
|
|
96
113
|
@capability(SourceCapability.USAGE_STATS, "Enabled by default to get usage stats")
|
|
@@ -16,7 +16,11 @@ from datahub.ingestion.api.decorators import (
|
|
|
16
16
|
platform_name,
|
|
17
17
|
support_status,
|
|
18
18
|
)
|
|
19
|
-
from datahub.ingestion.api.source import
|
|
19
|
+
from datahub.ingestion.api.source import (
|
|
20
|
+
MetadataWorkUnitProcessor,
|
|
21
|
+
SourceReport,
|
|
22
|
+
StructuredLogCategory,
|
|
23
|
+
)
|
|
20
24
|
from datahub.ingestion.api.workunit import MetadataWorkUnit
|
|
21
25
|
from datahub.ingestion.source.fivetran.config import (
|
|
22
26
|
KNOWN_DATA_PLATFORM_MAPPING,
|
|
@@ -96,8 +100,10 @@ class FivetranSource(StatefulIngestionSourceBase):
|
|
|
96
100
|
self.report.info(
|
|
97
101
|
title="Guessing source platform for lineage",
|
|
98
102
|
message="We encountered a connector type that we don't fully support yet. "
|
|
99
|
-
"We will attempt to guess the platform based on the connector type."
|
|
100
|
-
|
|
103
|
+
"We will attempt to guess the platform based on the connector type. "
|
|
104
|
+
"Note that we use connector_id as the key not connector_name which you may see in the UI of Fivetran. ",
|
|
105
|
+
context=f"connector_name: {connector.connector_name} (connector_id: {connector.connector_id}, connector_type: {connector.connector_type})",
|
|
106
|
+
log_category=StructuredLogCategory.LINEAGE,
|
|
101
107
|
)
|
|
102
108
|
source_details.platform = connector.connector_type
|
|
103
109
|
|
|
@@ -216,6 +216,14 @@ def get_column_unique_count_dh_patch(self: SqlAlchemyDataset, column: str) -> in
|
|
|
216
216
|
)
|
|
217
217
|
).scalar()
|
|
218
218
|
)
|
|
219
|
+
elif self.engine.dialect.name.lower() == DATABRICKS:
|
|
220
|
+
return convert_to_json_serializable(
|
|
221
|
+
self.engine.execute(
|
|
222
|
+
sa.select(sa.func.approx_count_distinct(sa.column(column))).select_from(
|
|
223
|
+
self._table
|
|
224
|
+
)
|
|
225
|
+
).scalar()
|
|
226
|
+
)
|
|
219
227
|
return convert_to_json_serializable(
|
|
220
228
|
self.engine.execute(
|
|
221
229
|
sa.select([sa.func.count(sa.func.distinct(sa.column(column)))]).select_from(
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
from functools import lru_cache
|
|
2
|
-
from typing import ClassVar, Optional, TextIO
|
|
2
|
+
from typing import ClassVar, Optional, TextIO, Type
|
|
3
3
|
|
|
4
4
|
from liquid import Environment
|
|
5
5
|
from liquid.ast import Node
|
|
@@ -20,16 +20,27 @@ class CustomTagException(Exception):
|
|
|
20
20
|
class ConditionNode(Node):
|
|
21
21
|
def __init__(self, tok: Token, sql_or_lookml_reference: str, filter_name: str):
|
|
22
22
|
self.tok = tok
|
|
23
|
-
|
|
24
23
|
self.sql_or_lookml_reference = sql_or_lookml_reference
|
|
25
|
-
|
|
26
24
|
self.filter_name = filter_name
|
|
27
25
|
|
|
28
26
|
def render_to_output(self, context: Context, buffer: TextIO) -> Optional[bool]:
|
|
29
27
|
# This implementation will make sure that sql parse work correctly if looker condition tag
|
|
30
28
|
# is used in lookml sql field
|
|
31
29
|
buffer.write(f"{self.sql_or_lookml_reference}='dummy_value'")
|
|
30
|
+
return True
|
|
32
31
|
|
|
32
|
+
|
|
33
|
+
class IncrementConditionNode(Node):
|
|
34
|
+
def __init__(self, tok: Token, sql_or_lookml_reference: str):
|
|
35
|
+
self.tok = tok
|
|
36
|
+
self.sql_or_lookml_reference = sql_or_lookml_reference
|
|
37
|
+
|
|
38
|
+
def render_to_output(self, context: Context, buffer: TextIO) -> Optional[bool]:
|
|
39
|
+
# For incrementcondition, we need to generate a condition that would be used
|
|
40
|
+
# in incremental PDT updates. This typically involves date/time comparisons.
|
|
41
|
+
# We'll render it as a date comparison with a placeholder value
|
|
42
|
+
# See details in Looker documentation for incrementcondition tag -> cloud.google.com/looker/docs/reference/param-view-increment-key
|
|
43
|
+
buffer.write(f"{self.sql_or_lookml_reference} > '2023-01-01'")
|
|
33
44
|
return True
|
|
34
45
|
|
|
35
46
|
|
|
@@ -44,7 +55,6 @@ class ConditionTag(Tag):
|
|
|
44
55
|
This class render the below tag as order.region='ap-south-1' if order_region is provided in config.liquid_variables
|
|
45
56
|
as order_region: 'ap-south-1'
|
|
46
57
|
{% condition order_region %} order.region {% endcondition %}
|
|
47
|
-
|
|
48
58
|
"""
|
|
49
59
|
|
|
50
60
|
TAG_START: ClassVar[str] = "condition"
|
|
@@ -79,7 +89,48 @@ class ConditionTag(Tag):
|
|
|
79
89
|
)
|
|
80
90
|
|
|
81
91
|
|
|
82
|
-
|
|
92
|
+
class IncrementConditionTag(Tag):
|
|
93
|
+
"""
|
|
94
|
+
IncrementConditionTag is the equivalent implementation of looker's custom liquid tag "incrementcondition".
|
|
95
|
+
Refer doc: https://cloud.google.com/looker/docs/incremental-pdts#using_the_incrementcondition_tag
|
|
96
|
+
|
|
97
|
+
This tag is used for incremental PDTs to determine which records should be updated.
|
|
98
|
+
It typically works with date/time fields to filter data that has changed since the last update.
|
|
99
|
+
|
|
100
|
+
Example usage in Looker:
|
|
101
|
+
{% incrementcondition created_at %} order.created_at {% endincrementcondition %}
|
|
102
|
+
|
|
103
|
+
This would generate SQL like: order.created_at > '2023-01-01 00:00:00'
|
|
104
|
+
"""
|
|
105
|
+
|
|
106
|
+
TAG_START: ClassVar[str] = "incrementcondition"
|
|
107
|
+
TAG_END: ClassVar[str] = "endincrementcondition"
|
|
108
|
+
name: str = "incrementcondition"
|
|
109
|
+
|
|
110
|
+
def __init__(self, env: Environment):
|
|
111
|
+
super().__init__(env)
|
|
112
|
+
self.parser = get_parser(self.env)
|
|
113
|
+
|
|
114
|
+
def parse(self, stream: TokenStream) -> Node:
|
|
115
|
+
expect(stream, TOKEN_TAG, value=IncrementConditionTag.TAG_START)
|
|
116
|
+
|
|
117
|
+
start_token = stream.current
|
|
118
|
+
|
|
119
|
+
stream.next_token()
|
|
120
|
+
expect(stream, TOKEN_LITERAL)
|
|
121
|
+
sql_or_lookml_reference: str = stream.current.value.strip()
|
|
122
|
+
|
|
123
|
+
stream.next_token()
|
|
124
|
+
expect(stream, TOKEN_TAG, value=IncrementConditionTag.TAG_END)
|
|
125
|
+
|
|
126
|
+
return IncrementConditionNode(
|
|
127
|
+
tok=start_token,
|
|
128
|
+
sql_or_lookml_reference=sql_or_lookml_reference,
|
|
129
|
+
)
|
|
130
|
+
|
|
131
|
+
|
|
132
|
+
# Updated custom_tags list to include both tags
|
|
133
|
+
custom_tags: list[Type[Tag]] = [ConditionTag, IncrementConditionTag]
|
|
83
134
|
|
|
84
135
|
|
|
85
136
|
@string_filter
|
|
@@ -13,7 +13,7 @@ from datahub.ingestion.api.decorators import (
|
|
|
13
13
|
platform_name,
|
|
14
14
|
support_status,
|
|
15
15
|
)
|
|
16
|
-
from datahub.ingestion.api.source import Source, SourceReport
|
|
16
|
+
from datahub.ingestion.api.source import Source, SourceReport, StructuredLogCategory
|
|
17
17
|
from datahub.ingestion.api.workunit import MetadataWorkUnit
|
|
18
18
|
from datahub.ingestion.source.common.subtypes import DatasetSubTypes
|
|
19
19
|
from datahub.ingestion.source.mock_data.datahub_mock_data_report import (
|
|
@@ -35,6 +35,8 @@ from datahub.utilities.str_enum import StrEnum
|
|
|
35
35
|
|
|
36
36
|
logger = logging.getLogger(__name__)
|
|
37
37
|
|
|
38
|
+
PLATFORM_NAME = "fake"
|
|
39
|
+
|
|
38
40
|
|
|
39
41
|
class SubTypePattern(StrEnum):
|
|
40
42
|
ALTERNATING = "alternating"
|
|
@@ -137,6 +139,10 @@ class DataHubMockDataConfig(ConfigModel):
|
|
|
137
139
|
default=0,
|
|
138
140
|
description="Number of warnings to add in report for testing",
|
|
139
141
|
)
|
|
142
|
+
num_info: int = Field(
|
|
143
|
+
default=0,
|
|
144
|
+
description="Number of info to add in report for testing",
|
|
145
|
+
)
|
|
140
146
|
|
|
141
147
|
gen_1: LineageConfigGen1 = Field(
|
|
142
148
|
default_factory=LineageConfigGen1,
|
|
@@ -144,7 +150,7 @@ class DataHubMockDataConfig(ConfigModel):
|
|
|
144
150
|
)
|
|
145
151
|
|
|
146
152
|
|
|
147
|
-
@platform_name(
|
|
153
|
+
@platform_name(PLATFORM_NAME)
|
|
148
154
|
@config_class(DataHubMockDataConfig)
|
|
149
155
|
@support_status(SupportStatus.TESTING)
|
|
150
156
|
class DataHubMockDataSource(Source):
|
|
@@ -159,6 +165,9 @@ class DataHubMockDataSource(Source):
|
|
|
159
165
|
self.report = DataHubMockDataReport()
|
|
160
166
|
|
|
161
167
|
def get_workunits(self) -> Iterable[MetadataWorkUnit]:
|
|
168
|
+
# We don't want any implicit aspects to be produced
|
|
169
|
+
# so we are not using get_workunits_internal
|
|
170
|
+
|
|
162
171
|
if self.config.throw_uncaught_exceptions:
|
|
163
172
|
raise Exception("This is a test exception")
|
|
164
173
|
|
|
@@ -176,10 +185,17 @@ class DataHubMockDataSource(Source):
|
|
|
176
185
|
message="This is test warning",
|
|
177
186
|
title="Test Warning",
|
|
178
187
|
context=f"This is test warning {i}",
|
|
188
|
+
log_category=StructuredLogCategory.LINEAGE,
|
|
189
|
+
)
|
|
190
|
+
|
|
191
|
+
if self.config.num_info > 0:
|
|
192
|
+
for i in range(self.config.num_info):
|
|
193
|
+
self.report.info(
|
|
194
|
+
message="This is test info",
|
|
195
|
+
title="Test Info",
|
|
196
|
+
context=f"This is test info {i}",
|
|
179
197
|
)
|
|
180
198
|
|
|
181
|
-
# We don't want any implicit aspects to be produced
|
|
182
|
-
# so we are not using get_workunits_internal
|
|
183
199
|
if self.config.gen_1.enabled:
|
|
184
200
|
for wu in self._data_gen_1():
|
|
185
201
|
if self.report.first_urn_seen is None:
|
|
@@ -309,7 +325,7 @@ class DataHubMockDataSource(Source):
|
|
|
309
325
|
table_level, table_index, subtype_pattern, subtype_types, level_subtypes
|
|
310
326
|
)
|
|
311
327
|
|
|
312
|
-
urn = make_dataset_urn(platform=
|
|
328
|
+
urn = make_dataset_urn(platform=PLATFORM_NAME, name=table_name)
|
|
313
329
|
mcp = MetadataChangeProposalWrapper(
|
|
314
330
|
entityUrn=urn,
|
|
315
331
|
entityType="dataset",
|
|
@@ -433,7 +449,7 @@ class DataHubMockDataSource(Source):
|
|
|
433
449
|
|
|
434
450
|
def _get_status_aspect(self, table: str) -> MetadataWorkUnit:
|
|
435
451
|
urn = make_dataset_urn(
|
|
436
|
-
platform=
|
|
452
|
+
platform=PLATFORM_NAME,
|
|
437
453
|
name=table,
|
|
438
454
|
)
|
|
439
455
|
mcp = MetadataChangeProposalWrapper(
|
|
@@ -448,7 +464,7 @@ class DataHubMockDataSource(Source):
|
|
|
448
464
|
) -> MetadataWorkUnit:
|
|
449
465
|
mcp = MetadataChangeProposalWrapper(
|
|
450
466
|
entityUrn=make_dataset_urn(
|
|
451
|
-
platform=
|
|
467
|
+
platform=PLATFORM_NAME,
|
|
452
468
|
name=downstream_table,
|
|
453
469
|
),
|
|
454
470
|
entityType="dataset",
|
|
@@ -456,7 +472,7 @@ class DataHubMockDataSource(Source):
|
|
|
456
472
|
upstreams=[
|
|
457
473
|
UpstreamClass(
|
|
458
474
|
dataset=make_dataset_urn(
|
|
459
|
-
platform=
|
|
475
|
+
platform=PLATFORM_NAME,
|
|
460
476
|
name=upstream_table,
|
|
461
477
|
),
|
|
462
478
|
type=DatasetLineageTypeClass.TRANSFORMED,
|
|
@@ -468,7 +484,7 @@ class DataHubMockDataSource(Source):
|
|
|
468
484
|
|
|
469
485
|
def _get_profile_aspect(self, table: str) -> MetadataWorkUnit:
|
|
470
486
|
urn = make_dataset_urn(
|
|
471
|
-
platform=
|
|
487
|
+
platform=PLATFORM_NAME,
|
|
472
488
|
name=table,
|
|
473
489
|
)
|
|
474
490
|
mcp = MetadataChangeProposalWrapper(
|
|
@@ -485,7 +501,7 @@ class DataHubMockDataSource(Source):
|
|
|
485
501
|
|
|
486
502
|
def _get_usage_aspect(self, table: str) -> MetadataWorkUnit:
|
|
487
503
|
urn = make_dataset_urn(
|
|
488
|
-
platform=
|
|
504
|
+
platform=PLATFORM_NAME,
|
|
489
505
|
name=table,
|
|
490
506
|
)
|
|
491
507
|
mcp = MetadataChangeProposalWrapper(
|
|
@@ -1226,7 +1226,10 @@ class Mapper:
|
|
|
1226
1226
|
@platform_name("PowerBI")
|
|
1227
1227
|
@config_class(PowerBiDashboardSourceConfig)
|
|
1228
1228
|
@support_status(SupportStatus.CERTIFIED)
|
|
1229
|
-
@capability(
|
|
1229
|
+
@capability(
|
|
1230
|
+
SourceCapability.CONTAINERS,
|
|
1231
|
+
"Enabled by default",
|
|
1232
|
+
)
|
|
1230
1233
|
@capability(SourceCapability.DESCRIPTIONS, "Enabled by default")
|
|
1231
1234
|
@capability(SourceCapability.OWNERSHIP, "Enabled by default")
|
|
1232
1235
|
@capability(SourceCapability.PLATFORM_INSTANCE, "Enabled by default")
|
|
@@ -132,6 +132,7 @@ logger: logging.Logger = logging.getLogger(__name__)
|
|
|
132
132
|
"Enabled by default",
|
|
133
133
|
subtype_modifier=[
|
|
134
134
|
SourceCapabilityModifier.DATABASE,
|
|
135
|
+
SourceCapabilityModifier.SCHEMA,
|
|
135
136
|
],
|
|
136
137
|
)
|
|
137
138
|
@capability(SourceCapability.DOMAINS, "Supported via the `domain` config field")
|
|
@@ -549,6 +549,14 @@ class SalesforceApi:
|
|
|
549
549
|
capability_name=SourceCapability.TAGS,
|
|
550
550
|
description="Enabled by default",
|
|
551
551
|
)
|
|
552
|
+
@capability(
|
|
553
|
+
capability_name=SourceCapability.LINEAGE_COARSE,
|
|
554
|
+
description="Extract table-level lineage for Salesforce objects",
|
|
555
|
+
subtype_modifier=[
|
|
556
|
+
SourceCapabilityModifier.SALESFORCE_CUSTOM_OBJECT,
|
|
557
|
+
SourceCapabilityModifier.SALESFORCE_STANDARD_OBJECT,
|
|
558
|
+
],
|
|
559
|
+
)
|
|
552
560
|
class SalesforceSource(StatefulIngestionSourceBase):
|
|
553
561
|
def __init__(self, config: SalesforceConfig, ctx: PipelineContext) -> None:
|
|
554
562
|
super().__init__(config, ctx)
|
|
@@ -27,6 +27,7 @@ from datahub.ingestion.api.workunit import MetadataWorkUnit
|
|
|
27
27
|
from datahub.ingestion.source.common.subtypes import (
|
|
28
28
|
DatasetContainerSubTypes,
|
|
29
29
|
DatasetSubTypes,
|
|
30
|
+
SourceCapabilityModifier,
|
|
30
31
|
)
|
|
31
32
|
from datahub.ingestion.source.sql.sql_common import (
|
|
32
33
|
SQLAlchemySource,
|
|
@@ -168,6 +169,13 @@ class HiveMetastore(BasicSQLAlchemyConfig):
|
|
|
168
169
|
@capability(
|
|
169
170
|
SourceCapability.LINEAGE_COARSE, "View lineage is not supported", supported=False
|
|
170
171
|
)
|
|
172
|
+
@capability(
|
|
173
|
+
SourceCapability.CONTAINERS,
|
|
174
|
+
"Enabled by default",
|
|
175
|
+
subtype_modifier=[
|
|
176
|
+
SourceCapabilityModifier.CATALOG,
|
|
177
|
+
],
|
|
178
|
+
)
|
|
171
179
|
class HiveMetastoreSource(SQLAlchemySource):
|
|
172
180
|
"""
|
|
173
181
|
This plugin extracts the following:
|
|
@@ -42,6 +42,7 @@ from datahub.ingestion.api.decorators import (
|
|
|
42
42
|
)
|
|
43
43
|
from datahub.ingestion.api.workunit import MetadataWorkUnit
|
|
44
44
|
from datahub.ingestion.graph.client import DataHubGraph
|
|
45
|
+
from datahub.ingestion.source.common.subtypes import SourceCapabilityModifier
|
|
45
46
|
from datahub.ingestion.source.sql.sql_common import register_custom_type
|
|
46
47
|
from datahub.ingestion.source.sql.sql_config import SQLCommonConfig
|
|
47
48
|
from datahub.ingestion.source.sql.sql_report import SQLSourceReport
|
|
@@ -539,7 +540,13 @@ class TeradataConfig(BaseTeradataConfig, BaseTimeWindowConfig):
|
|
|
539
540
|
@config_class(TeradataConfig)
|
|
540
541
|
@support_status(SupportStatus.TESTING)
|
|
541
542
|
@capability(SourceCapability.DOMAINS, "Enabled by default")
|
|
542
|
-
@capability(
|
|
543
|
+
@capability(
|
|
544
|
+
SourceCapability.CONTAINERS,
|
|
545
|
+
"Enabled by default",
|
|
546
|
+
subtype_modifier=[
|
|
547
|
+
SourceCapabilityModifier.DATABASE,
|
|
548
|
+
],
|
|
549
|
+
)
|
|
543
550
|
@capability(SourceCapability.PLATFORM_INSTANCE, "Enabled by default")
|
|
544
551
|
@capability(
|
|
545
552
|
SourceCapability.DELETION_DETECTION,
|