acryl-datahub 1.2.0.3rc2__py3-none-any.whl → 1.2.0.4rc1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of acryl-datahub might be problematic. Click here for more details.

Files changed (32) hide show
  1. {acryl_datahub-1.2.0.3rc2.dist-info → acryl_datahub-1.2.0.4rc1.dist-info}/METADATA +2522 -2522
  2. {acryl_datahub-1.2.0.3rc2.dist-info → acryl_datahub-1.2.0.4rc1.dist-info}/RECORD +32 -32
  3. datahub/_version.py +1 -1
  4. datahub/api/entities/external/external_tag.py +6 -4
  5. datahub/api/entities/external/lake_formation_external_entites.py +50 -49
  6. datahub/api/entities/external/restricted_text.py +107 -182
  7. datahub/api/entities/external/unity_catalog_external_entites.py +51 -52
  8. datahub/ingestion/api/source.py +81 -7
  9. datahub/ingestion/autogenerated/capability_summary.json +47 -19
  10. datahub/ingestion/source/abs/source.py +9 -0
  11. datahub/ingestion/source/aws/glue.py +18 -2
  12. datahub/ingestion/source/aws/tag_entities.py +2 -2
  13. datahub/ingestion/source/datahub/datahub_source.py +8 -1
  14. datahub/ingestion/source/delta_lake/source.py +8 -1
  15. datahub/ingestion/source/dremio/dremio_source.py +19 -2
  16. datahub/ingestion/source/fivetran/fivetran.py +9 -3
  17. datahub/ingestion/source/ge_data_profiler.py +8 -0
  18. datahub/ingestion/source/looker/looker_liquid_tag.py +56 -5
  19. datahub/ingestion/source/mock_data/datahub_mock_data.py +26 -10
  20. datahub/ingestion/source/powerbi/powerbi.py +4 -1
  21. datahub/ingestion/source/redshift/redshift.py +1 -0
  22. datahub/ingestion/source/salesforce.py +8 -0
  23. datahub/ingestion/source/sql/hive_metastore.py +8 -0
  24. datahub/ingestion/source/sql/teradata.py +8 -1
  25. datahub/ingestion/source/sql/trino.py +9 -0
  26. datahub/ingestion/source/unity/tag_entities.py +3 -3
  27. datahub/sdk/entity_client.py +22 -7
  28. datahub/utilities/mapping.py +29 -2
  29. {acryl_datahub-1.2.0.3rc2.dist-info → acryl_datahub-1.2.0.4rc1.dist-info}/WHEEL +0 -0
  30. {acryl_datahub-1.2.0.3rc2.dist-info → acryl_datahub-1.2.0.4rc1.dist-info}/entry_points.txt +0 -0
  31. {acryl_datahub-1.2.0.3rc2.dist-info → acryl_datahub-1.2.0.4rc1.dist-info}/licenses/LICENSE +0 -0
  32. {acryl_datahub-1.2.0.3rc2.dist-info → acryl_datahub-1.2.0.4rc1.dist-info}/top_level.txt +0 -0
@@ -1,9 +1,18 @@
1
1
  {
2
- "generated_at": "2025-07-24T13:24:05.751563+00:00",
2
+ "generated_at": "2025-07-31T12:54:30.557618+00:00",
3
3
  "generated_by": "metadata-ingestion/scripts/capability_summary.py",
4
4
  "plugin_details": {
5
5
  "abs": {
6
6
  "capabilities": [
7
+ {
8
+ "capability": "CONTAINERS",
9
+ "description": "Extract ABS containers and folders",
10
+ "subtype_modifier": [
11
+ "Folder",
12
+ "ABS container"
13
+ ],
14
+ "supported": true
15
+ },
7
16
  {
8
17
  "capability": "DATA_PROFILING",
9
18
  "description": "Optionally enabled via configuration",
@@ -468,7 +477,9 @@
468
477
  {
469
478
  "capability": "CONTAINERS",
470
479
  "description": "Enabled by default",
471
- "subtype_modifier": null,
480
+ "subtype_modifier": [
481
+ "Database"
482
+ ],
472
483
  "supported": true
473
484
  },
474
485
  {
@@ -531,13 +542,6 @@
531
542
  "platform_name": "File Based Lineage",
532
543
  "support_status": "CERTIFIED"
533
544
  },
534
- "datahub-mock-data": {
535
- "capabilities": [],
536
- "classname": "datahub.ingestion.source.mock_data.datahub_mock_data.DataHubMockDataSource",
537
- "platform_id": "datahubmockdata",
538
- "platform_name": "DataHubMockData",
539
- "support_status": "TESTING"
540
- },
541
545
  "dbt": {
542
546
  "capabilities": [
543
547
  {
@@ -607,7 +611,9 @@
607
611
  {
608
612
  "capability": "CONTAINERS",
609
613
  "description": "Enabled by default",
610
- "subtype_modifier": null,
614
+ "subtype_modifier": [
615
+ "Folder"
616
+ ],
611
617
  "supported": true
612
618
  },
613
619
  {
@@ -643,6 +649,14 @@
643
649
  "subtype_modifier": null,
644
650
  "supported": true
645
651
  },
652
+ {
653
+ "capability": "LINEAGE_FINE",
654
+ "description": "Extract column-level lineage",
655
+ "subtype_modifier": [
656
+ "Table"
657
+ ],
658
+ "supported": true
659
+ },
646
660
  {
647
661
  "capability": "DATA_PROFILING",
648
662
  "description": "Optionally enabled via configuration",
@@ -688,7 +702,9 @@
688
702
  {
689
703
  "capability": "LINEAGE_COARSE",
690
704
  "description": "Enabled by default",
691
- "subtype_modifier": null,
705
+ "subtype_modifier": [
706
+ "Table"
707
+ ],
692
708
  "supported": true
693
709
  }
694
710
  ],
@@ -1229,8 +1245,7 @@
1229
1245
  "capability": "CONTAINERS",
1230
1246
  "description": "Enabled by default",
1231
1247
  "subtype_modifier": [
1232
- "Database",
1233
- "Schema"
1248
+ "Catalog"
1234
1249
  ],
1235
1250
  "supported": true
1236
1251
  },
@@ -2387,8 +2402,9 @@
2387
2402
  },
2388
2403
  {
2389
2404
  "capability": "LINEAGE_COARSE",
2390
- "description": "Enabled by default to get lineage for views via `include_view_lineage`",
2405
+ "description": "Extract table-level lineage",
2391
2406
  "subtype_modifier": [
2407
+ "Table",
2392
2408
  "View"
2393
2409
  ],
2394
2410
  "supported": true
@@ -2411,8 +2427,7 @@
2411
2427
  "capability": "CONTAINERS",
2412
2428
  "description": "Enabled by default",
2413
2429
  "subtype_modifier": [
2414
- "Database",
2415
- "Schema"
2430
+ "Catalog"
2416
2431
  ],
2417
2432
  "supported": true
2418
2433
  },
@@ -2598,7 +2613,8 @@
2598
2613
  "capability": "CONTAINERS",
2599
2614
  "description": "Enabled by default",
2600
2615
  "subtype_modifier": [
2601
- "Database"
2616
+ "Database",
2617
+ "Schema"
2602
2618
  ],
2603
2619
  "supported": true
2604
2620
  },
@@ -2812,6 +2828,15 @@
2812
2828
  "description": "Enabled by default",
2813
2829
  "subtype_modifier": null,
2814
2830
  "supported": true
2831
+ },
2832
+ {
2833
+ "capability": "LINEAGE_COARSE",
2834
+ "description": "Extract table-level lineage for Salesforce objects",
2835
+ "subtype_modifier": [
2836
+ "Custom Object",
2837
+ "Object"
2838
+ ],
2839
+ "supported": true
2815
2840
  }
2816
2841
  ],
2817
2842
  "classname": "datahub.ingestion.source.salesforce.SalesforceSource",
@@ -3207,7 +3232,9 @@
3207
3232
  {
3208
3233
  "capability": "CONTAINERS",
3209
3234
  "description": "Enabled by default",
3210
- "subtype_modifier": null,
3235
+ "subtype_modifier": [
3236
+ "Database"
3237
+ ],
3211
3238
  "supported": true
3212
3239
  },
3213
3240
  {
@@ -3339,8 +3366,9 @@
3339
3366
  },
3340
3367
  {
3341
3368
  "capability": "LINEAGE_COARSE",
3342
- "description": "Enabled by default to get lineage for views via `include_view_lineage`",
3369
+ "description": "Extract table-level lineage",
3343
3370
  "subtype_modifier": [
3371
+ "Table",
3344
3372
  "View"
3345
3373
  ],
3346
3374
  "supported": true
@@ -44,6 +44,7 @@ from datahub.ingestion.source.azure.abs_utils import (
44
44
  get_key_prefix,
45
45
  strip_abs_prefix,
46
46
  )
47
+ from datahub.ingestion.source.common.subtypes import SourceCapabilityModifier
47
48
  from datahub.ingestion.source.data_lake_common.data_lake_utils import (
48
49
  ContainerWUCreator,
49
50
  add_partition_columns_to_schema,
@@ -128,6 +129,14 @@ class TableData:
128
129
  @support_status(SupportStatus.INCUBATING)
129
130
  @capability(SourceCapability.DATA_PROFILING, "Optionally enabled via configuration")
130
131
  @capability(SourceCapability.TAGS, "Can extract ABS object/container tags if enabled")
132
+ @capability(
133
+ SourceCapability.CONTAINERS,
134
+ "Extract ABS containers and folders",
135
+ subtype_modifier=[
136
+ SourceCapabilityModifier.FOLDER,
137
+ SourceCapabilityModifier.ABS_CONTAINER,
138
+ ],
139
+ )
131
140
  class ABSSource(StatefulIngestionSourceBase):
132
141
  source_config: DataLakeSourceConfig
133
142
  report: DataLakeSourceReport
@@ -395,7 +395,7 @@ class GlueSource(StatefulIngestionSourceBase):
395
395
  t = LakeFormationTag(
396
396
  key=tag_key,
397
397
  value=tag_value,
398
- catalog_id=catalog_id,
398
+ catalog=catalog_id,
399
399
  )
400
400
  tags.append(t)
401
401
  return tags
@@ -438,7 +438,7 @@ class GlueSource(StatefulIngestionSourceBase):
438
438
  t = LakeFormationTag(
439
439
  key=tag_key,
440
440
  value=tag_value,
441
- catalog_id=catalog_id,
441
+ catalog=catalog_id,
442
442
  )
443
443
  tags.append(t)
444
444
  return tags
@@ -522,6 +522,14 @@ class GlueSource(StatefulIngestionSourceBase):
522
522
  bucket = url.netloc
523
523
  key = url.path[1:]
524
524
 
525
+ # validate that we have a non-empty key
526
+ if not key:
527
+ self.report.num_job_script_location_invalid += 1
528
+ logger.warning(
529
+ f"Error parsing DAG for Glue job. The script {script_path} is not a valid S3 path for flow urn: {flow_urn}."
530
+ )
531
+ return None
532
+
525
533
  # download the script contents
526
534
  # see https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/s3.html#S3.Client.get_object
527
535
  try:
@@ -533,6 +541,14 @@ class GlueSource(StatefulIngestionSourceBase):
533
541
  )
534
542
  self.report.num_job_script_failed_download += 1
535
543
  return None
544
+ except botocore.exceptions.ParamValidationError as e:
545
+ self.report_warning(
546
+ flow_urn,
547
+ f"Invalid S3 path for Glue job script {script_path}: {e}",
548
+ )
549
+ self.report.num_job_script_location_invalid += 1
550
+ return None
551
+
536
552
  script = obj["Body"].read().decode("utf-8")
537
553
 
538
554
  try:
@@ -88,8 +88,8 @@ class LakeFormationTagPlatformResourceId(BaseModel, ExternalEntityId):
88
88
  return existing_platform_resource
89
89
 
90
90
  return LakeFormationTagPlatformResourceId(
91
- tag_key=tag.key,
92
- tag_value=tag.value if tag.value is not None else None,
91
+ tag_key=str(tag.key),
92
+ tag_value=str(tag.value) if tag.value is not None else None,
93
93
  platform_instance=platform_instance,
94
94
  exists_in_lake_formation=exists_in_lake_formation,
95
95
  catalog=catalog,
@@ -19,6 +19,7 @@ from datahub.ingestion.api.source_helpers import (
19
19
  auto_workunit_reporter,
20
20
  )
21
21
  from datahub.ingestion.api.workunit import MetadataWorkUnit
22
+ from datahub.ingestion.source.common.subtypes import SourceCapabilityModifier
22
23
  from datahub.ingestion.source.datahub.config import DataHubSourceConfig
23
24
  from datahub.ingestion.source.datahub.datahub_api_reader import DataHubApiReader
24
25
  from datahub.ingestion.source.datahub.datahub_database_reader import (
@@ -39,7 +40,13 @@ logger = logging.getLogger(__name__)
39
40
  @platform_name("DataHub")
40
41
  @config_class(DataHubSourceConfig)
41
42
  @support_status(SupportStatus.TESTING)
42
- @capability(SourceCapability.CONTAINERS, "Enabled by default")
43
+ @capability(
44
+ SourceCapability.CONTAINERS,
45
+ "Enabled by default",
46
+ subtype_modifier=[
47
+ SourceCapabilityModifier.DATABASE,
48
+ ],
49
+ )
43
50
  class DataHubSource(StatefulIngestionSourceBase):
44
51
  platform: str = "datahub"
45
52
 
@@ -29,6 +29,7 @@ from datahub.ingestion.source.aws.s3_util import (
29
29
  get_key_prefix,
30
30
  strip_s3_prefix,
31
31
  )
32
+ from datahub.ingestion.source.common.subtypes import SourceCapabilityModifier
32
33
  from datahub.ingestion.source.data_lake_common.data_lake_utils import ContainerWUCreator
33
34
  from datahub.ingestion.source.delta_lake.config import DeltaLakeSourceConfig
34
35
  from datahub.ingestion.source.delta_lake.delta_lake_utils import (
@@ -85,7 +86,13 @@ OPERATION_STATEMENT_TYPES = {
85
86
  @config_class(DeltaLakeSourceConfig)
86
87
  @support_status(SupportStatus.INCUBATING)
87
88
  @capability(SourceCapability.TAGS, "Can extract S3 object/bucket tags if enabled")
88
- @capability(SourceCapability.CONTAINERS, "Enabled by default")
89
+ @capability(
90
+ SourceCapability.CONTAINERS,
91
+ "Enabled by default",
92
+ subtype_modifier=[
93
+ SourceCapabilityModifier.FOLDER,
94
+ ],
95
+ )
89
96
  class DeltaLakeSource(StatefulIngestionSourceBase):
90
97
  """
91
98
  This plugin extracts:
@@ -22,6 +22,7 @@ from datahub.ingestion.api.source import (
22
22
  SourceReport,
23
23
  )
24
24
  from datahub.ingestion.api.workunit import MetadataWorkUnit
25
+ from datahub.ingestion.source.common.subtypes import SourceCapabilityModifier
25
26
  from datahub.ingestion.source.dremio.dremio_api import (
26
27
  DremioAPIOperations,
27
28
  DremioEdition,
@@ -86,11 +87,27 @@ class DremioSourceMapEntry:
86
87
  @platform_name("Dremio")
87
88
  @config_class(DremioSourceConfig)
88
89
  @support_status(SupportStatus.CERTIFIED)
89
- @capability(SourceCapability.CONTAINERS, "Enabled by default")
90
+ @capability(
91
+ SourceCapability.CONTAINERS,
92
+ "Enabled by default",
93
+ )
90
94
  @capability(SourceCapability.DATA_PROFILING, "Optionally enabled via configuration")
91
95
  @capability(SourceCapability.DESCRIPTIONS, "Enabled by default")
92
96
  @capability(SourceCapability.DOMAINS, "Supported via the `domain` config field")
93
- @capability(SourceCapability.LINEAGE_COARSE, "Enabled by default")
97
+ @capability(
98
+ SourceCapability.LINEAGE_COARSE,
99
+ "Enabled by default",
100
+ subtype_modifier=[
101
+ SourceCapabilityModifier.TABLE,
102
+ ],
103
+ )
104
+ @capability(
105
+ SourceCapability.LINEAGE_FINE,
106
+ "Extract column-level lineage",
107
+ subtype_modifier=[
108
+ SourceCapabilityModifier.TABLE,
109
+ ],
110
+ )
94
111
  @capability(SourceCapability.OWNERSHIP, "Enabled by default")
95
112
  @capability(SourceCapability.PLATFORM_INSTANCE, "Enabled by default")
96
113
  @capability(SourceCapability.USAGE_STATS, "Enabled by default to get usage stats")
@@ -16,7 +16,11 @@ from datahub.ingestion.api.decorators import (
16
16
  platform_name,
17
17
  support_status,
18
18
  )
19
- from datahub.ingestion.api.source import MetadataWorkUnitProcessor, SourceReport
19
+ from datahub.ingestion.api.source import (
20
+ MetadataWorkUnitProcessor,
21
+ SourceReport,
22
+ StructuredLogCategory,
23
+ )
20
24
  from datahub.ingestion.api.workunit import MetadataWorkUnit
21
25
  from datahub.ingestion.source.fivetran.config import (
22
26
  KNOWN_DATA_PLATFORM_MAPPING,
@@ -96,8 +100,10 @@ class FivetranSource(StatefulIngestionSourceBase):
96
100
  self.report.info(
97
101
  title="Guessing source platform for lineage",
98
102
  message="We encountered a connector type that we don't fully support yet. "
99
- "We will attempt to guess the platform based on the connector type.",
100
- context=f"{connector.connector_name} (connector_id: {connector.connector_id}, connector_type: {connector.connector_type})",
103
+ "We will attempt to guess the platform based on the connector type. "
104
+ "Note that we use connector_id as the key not connector_name which you may see in the UI of Fivetran. ",
105
+ context=f"connector_name: {connector.connector_name} (connector_id: {connector.connector_id}, connector_type: {connector.connector_type})",
106
+ log_category=StructuredLogCategory.LINEAGE,
101
107
  )
102
108
  source_details.platform = connector.connector_type
103
109
 
@@ -216,6 +216,14 @@ def get_column_unique_count_dh_patch(self: SqlAlchemyDataset, column: str) -> in
216
216
  )
217
217
  ).scalar()
218
218
  )
219
+ elif self.engine.dialect.name.lower() == DATABRICKS:
220
+ return convert_to_json_serializable(
221
+ self.engine.execute(
222
+ sa.select(sa.func.approx_count_distinct(sa.column(column))).select_from(
223
+ self._table
224
+ )
225
+ ).scalar()
226
+ )
219
227
  return convert_to_json_serializable(
220
228
  self.engine.execute(
221
229
  sa.select([sa.func.count(sa.func.distinct(sa.column(column)))]).select_from(
@@ -1,5 +1,5 @@
1
1
  from functools import lru_cache
2
- from typing import ClassVar, Optional, TextIO
2
+ from typing import ClassVar, Optional, TextIO, Type
3
3
 
4
4
  from liquid import Environment
5
5
  from liquid.ast import Node
@@ -20,16 +20,27 @@ class CustomTagException(Exception):
20
20
  class ConditionNode(Node):
21
21
  def __init__(self, tok: Token, sql_or_lookml_reference: str, filter_name: str):
22
22
  self.tok = tok
23
-
24
23
  self.sql_or_lookml_reference = sql_or_lookml_reference
25
-
26
24
  self.filter_name = filter_name
27
25
 
28
26
  def render_to_output(self, context: Context, buffer: TextIO) -> Optional[bool]:
29
27
  # This implementation will make sure that sql parse work correctly if looker condition tag
30
28
  # is used in lookml sql field
31
29
  buffer.write(f"{self.sql_or_lookml_reference}='dummy_value'")
30
+ return True
32
31
 
32
+
33
+ class IncrementConditionNode(Node):
34
+ def __init__(self, tok: Token, sql_or_lookml_reference: str):
35
+ self.tok = tok
36
+ self.sql_or_lookml_reference = sql_or_lookml_reference
37
+
38
+ def render_to_output(self, context: Context, buffer: TextIO) -> Optional[bool]:
39
+ # For incrementcondition, we need to generate a condition that would be used
40
+ # in incremental PDT updates. This typically involves date/time comparisons.
41
+ # We'll render it as a date comparison with a placeholder value
42
+ # See details in Looker documentation for incrementcondition tag -> cloud.google.com/looker/docs/reference/param-view-increment-key
43
+ buffer.write(f"{self.sql_or_lookml_reference} > '2023-01-01'")
33
44
  return True
34
45
 
35
46
 
@@ -44,7 +55,6 @@ class ConditionTag(Tag):
44
55
  This class render the below tag as order.region='ap-south-1' if order_region is provided in config.liquid_variables
45
56
  as order_region: 'ap-south-1'
46
57
  {% condition order_region %} order.region {% endcondition %}
47
-
48
58
  """
49
59
 
50
60
  TAG_START: ClassVar[str] = "condition"
@@ -79,7 +89,48 @@ class ConditionTag(Tag):
79
89
  )
80
90
 
81
91
 
82
- custom_tags = [ConditionTag]
92
+ class IncrementConditionTag(Tag):
93
+ """
94
+ IncrementConditionTag is the equivalent implementation of looker's custom liquid tag "incrementcondition".
95
+ Refer doc: https://cloud.google.com/looker/docs/incremental-pdts#using_the_incrementcondition_tag
96
+
97
+ This tag is used for incremental PDTs to determine which records should be updated.
98
+ It typically works with date/time fields to filter data that has changed since the last update.
99
+
100
+ Example usage in Looker:
101
+ {% incrementcondition created_at %} order.created_at {% endincrementcondition %}
102
+
103
+ This would generate SQL like: order.created_at > '2023-01-01 00:00:00'
104
+ """
105
+
106
+ TAG_START: ClassVar[str] = "incrementcondition"
107
+ TAG_END: ClassVar[str] = "endincrementcondition"
108
+ name: str = "incrementcondition"
109
+
110
+ def __init__(self, env: Environment):
111
+ super().__init__(env)
112
+ self.parser = get_parser(self.env)
113
+
114
+ def parse(self, stream: TokenStream) -> Node:
115
+ expect(stream, TOKEN_TAG, value=IncrementConditionTag.TAG_START)
116
+
117
+ start_token = stream.current
118
+
119
+ stream.next_token()
120
+ expect(stream, TOKEN_LITERAL)
121
+ sql_or_lookml_reference: str = stream.current.value.strip()
122
+
123
+ stream.next_token()
124
+ expect(stream, TOKEN_TAG, value=IncrementConditionTag.TAG_END)
125
+
126
+ return IncrementConditionNode(
127
+ tok=start_token,
128
+ sql_or_lookml_reference=sql_or_lookml_reference,
129
+ )
130
+
131
+
132
+ # Updated custom_tags list to include both tags
133
+ custom_tags: list[Type[Tag]] = [ConditionTag, IncrementConditionTag]
83
134
 
84
135
 
85
136
  @string_filter
@@ -13,7 +13,7 @@ from datahub.ingestion.api.decorators import (
13
13
  platform_name,
14
14
  support_status,
15
15
  )
16
- from datahub.ingestion.api.source import Source, SourceReport
16
+ from datahub.ingestion.api.source import Source, SourceReport, StructuredLogCategory
17
17
  from datahub.ingestion.api.workunit import MetadataWorkUnit
18
18
  from datahub.ingestion.source.common.subtypes import DatasetSubTypes
19
19
  from datahub.ingestion.source.mock_data.datahub_mock_data_report import (
@@ -35,6 +35,8 @@ from datahub.utilities.str_enum import StrEnum
35
35
 
36
36
  logger = logging.getLogger(__name__)
37
37
 
38
+ PLATFORM_NAME = "fake"
39
+
38
40
 
39
41
  class SubTypePattern(StrEnum):
40
42
  ALTERNATING = "alternating"
@@ -137,6 +139,10 @@ class DataHubMockDataConfig(ConfigModel):
137
139
  default=0,
138
140
  description="Number of warnings to add in report for testing",
139
141
  )
142
+ num_info: int = Field(
143
+ default=0,
144
+ description="Number of info to add in report for testing",
145
+ )
140
146
 
141
147
  gen_1: LineageConfigGen1 = Field(
142
148
  default_factory=LineageConfigGen1,
@@ -144,7 +150,7 @@ class DataHubMockDataConfig(ConfigModel):
144
150
  )
145
151
 
146
152
 
147
- @platform_name("DataHubMockData")
153
+ @platform_name(PLATFORM_NAME)
148
154
  @config_class(DataHubMockDataConfig)
149
155
  @support_status(SupportStatus.TESTING)
150
156
  class DataHubMockDataSource(Source):
@@ -159,6 +165,9 @@ class DataHubMockDataSource(Source):
159
165
  self.report = DataHubMockDataReport()
160
166
 
161
167
  def get_workunits(self) -> Iterable[MetadataWorkUnit]:
168
+ # We don't want any implicit aspects to be produced
169
+ # so we are not using get_workunits_internal
170
+
162
171
  if self.config.throw_uncaught_exceptions:
163
172
  raise Exception("This is a test exception")
164
173
 
@@ -176,10 +185,17 @@ class DataHubMockDataSource(Source):
176
185
  message="This is test warning",
177
186
  title="Test Warning",
178
187
  context=f"This is test warning {i}",
188
+ log_category=StructuredLogCategory.LINEAGE,
189
+ )
190
+
191
+ if self.config.num_info > 0:
192
+ for i in range(self.config.num_info):
193
+ self.report.info(
194
+ message="This is test info",
195
+ title="Test Info",
196
+ context=f"This is test info {i}",
179
197
  )
180
198
 
181
- # We don't want any implicit aspects to be produced
182
- # so we are not using get_workunits_internal
183
199
  if self.config.gen_1.enabled:
184
200
  for wu in self._data_gen_1():
185
201
  if self.report.first_urn_seen is None:
@@ -309,7 +325,7 @@ class DataHubMockDataSource(Source):
309
325
  table_level, table_index, subtype_pattern, subtype_types, level_subtypes
310
326
  )
311
327
 
312
- urn = make_dataset_urn(platform="fake", name=table_name)
328
+ urn = make_dataset_urn(platform=PLATFORM_NAME, name=table_name)
313
329
  mcp = MetadataChangeProposalWrapper(
314
330
  entityUrn=urn,
315
331
  entityType="dataset",
@@ -433,7 +449,7 @@ class DataHubMockDataSource(Source):
433
449
 
434
450
  def _get_status_aspect(self, table: str) -> MetadataWorkUnit:
435
451
  urn = make_dataset_urn(
436
- platform="fake",
452
+ platform=PLATFORM_NAME,
437
453
  name=table,
438
454
  )
439
455
  mcp = MetadataChangeProposalWrapper(
@@ -448,7 +464,7 @@ class DataHubMockDataSource(Source):
448
464
  ) -> MetadataWorkUnit:
449
465
  mcp = MetadataChangeProposalWrapper(
450
466
  entityUrn=make_dataset_urn(
451
- platform="fake",
467
+ platform=PLATFORM_NAME,
452
468
  name=downstream_table,
453
469
  ),
454
470
  entityType="dataset",
@@ -456,7 +472,7 @@ class DataHubMockDataSource(Source):
456
472
  upstreams=[
457
473
  UpstreamClass(
458
474
  dataset=make_dataset_urn(
459
- platform="fake",
475
+ platform=PLATFORM_NAME,
460
476
  name=upstream_table,
461
477
  ),
462
478
  type=DatasetLineageTypeClass.TRANSFORMED,
@@ -468,7 +484,7 @@ class DataHubMockDataSource(Source):
468
484
 
469
485
  def _get_profile_aspect(self, table: str) -> MetadataWorkUnit:
470
486
  urn = make_dataset_urn(
471
- platform="fake",
487
+ platform=PLATFORM_NAME,
472
488
  name=table,
473
489
  )
474
490
  mcp = MetadataChangeProposalWrapper(
@@ -485,7 +501,7 @@ class DataHubMockDataSource(Source):
485
501
 
486
502
  def _get_usage_aspect(self, table: str) -> MetadataWorkUnit:
487
503
  urn = make_dataset_urn(
488
- platform="fake",
504
+ platform=PLATFORM_NAME,
489
505
  name=table,
490
506
  )
491
507
  mcp = MetadataChangeProposalWrapper(
@@ -1226,7 +1226,10 @@ class Mapper:
1226
1226
  @platform_name("PowerBI")
1227
1227
  @config_class(PowerBiDashboardSourceConfig)
1228
1228
  @support_status(SupportStatus.CERTIFIED)
1229
- @capability(SourceCapability.CONTAINERS, "Enabled by default")
1229
+ @capability(
1230
+ SourceCapability.CONTAINERS,
1231
+ "Enabled by default",
1232
+ )
1230
1233
  @capability(SourceCapability.DESCRIPTIONS, "Enabled by default")
1231
1234
  @capability(SourceCapability.OWNERSHIP, "Enabled by default")
1232
1235
  @capability(SourceCapability.PLATFORM_INSTANCE, "Enabled by default")
@@ -132,6 +132,7 @@ logger: logging.Logger = logging.getLogger(__name__)
132
132
  "Enabled by default",
133
133
  subtype_modifier=[
134
134
  SourceCapabilityModifier.DATABASE,
135
+ SourceCapabilityModifier.SCHEMA,
135
136
  ],
136
137
  )
137
138
  @capability(SourceCapability.DOMAINS, "Supported via the `domain` config field")
@@ -549,6 +549,14 @@ class SalesforceApi:
549
549
  capability_name=SourceCapability.TAGS,
550
550
  description="Enabled by default",
551
551
  )
552
+ @capability(
553
+ capability_name=SourceCapability.LINEAGE_COARSE,
554
+ description="Extract table-level lineage for Salesforce objects",
555
+ subtype_modifier=[
556
+ SourceCapabilityModifier.SALESFORCE_CUSTOM_OBJECT,
557
+ SourceCapabilityModifier.SALESFORCE_STANDARD_OBJECT,
558
+ ],
559
+ )
552
560
  class SalesforceSource(StatefulIngestionSourceBase):
553
561
  def __init__(self, config: SalesforceConfig, ctx: PipelineContext) -> None:
554
562
  super().__init__(config, ctx)
@@ -27,6 +27,7 @@ from datahub.ingestion.api.workunit import MetadataWorkUnit
27
27
  from datahub.ingestion.source.common.subtypes import (
28
28
  DatasetContainerSubTypes,
29
29
  DatasetSubTypes,
30
+ SourceCapabilityModifier,
30
31
  )
31
32
  from datahub.ingestion.source.sql.sql_common import (
32
33
  SQLAlchemySource,
@@ -168,6 +169,13 @@ class HiveMetastore(BasicSQLAlchemyConfig):
168
169
  @capability(
169
170
  SourceCapability.LINEAGE_COARSE, "View lineage is not supported", supported=False
170
171
  )
172
+ @capability(
173
+ SourceCapability.CONTAINERS,
174
+ "Enabled by default",
175
+ subtype_modifier=[
176
+ SourceCapabilityModifier.CATALOG,
177
+ ],
178
+ )
171
179
  class HiveMetastoreSource(SQLAlchemySource):
172
180
  """
173
181
  This plugin extracts the following:
@@ -42,6 +42,7 @@ from datahub.ingestion.api.decorators import (
42
42
  )
43
43
  from datahub.ingestion.api.workunit import MetadataWorkUnit
44
44
  from datahub.ingestion.graph.client import DataHubGraph
45
+ from datahub.ingestion.source.common.subtypes import SourceCapabilityModifier
45
46
  from datahub.ingestion.source.sql.sql_common import register_custom_type
46
47
  from datahub.ingestion.source.sql.sql_config import SQLCommonConfig
47
48
  from datahub.ingestion.source.sql.sql_report import SQLSourceReport
@@ -539,7 +540,13 @@ class TeradataConfig(BaseTeradataConfig, BaseTimeWindowConfig):
539
540
  @config_class(TeradataConfig)
540
541
  @support_status(SupportStatus.TESTING)
541
542
  @capability(SourceCapability.DOMAINS, "Enabled by default")
542
- @capability(SourceCapability.CONTAINERS, "Enabled by default")
543
+ @capability(
544
+ SourceCapability.CONTAINERS,
545
+ "Enabled by default",
546
+ subtype_modifier=[
547
+ SourceCapabilityModifier.DATABASE,
548
+ ],
549
+ )
543
550
  @capability(SourceCapability.PLATFORM_INSTANCE, "Enabled by default")
544
551
  @capability(
545
552
  SourceCapability.DELETION_DETECTION,