acryl-datahub 1.0.0.4rc8__py3-none-any.whl → 1.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of acryl-datahub might be problematic. Click here for more details.
- {acryl_datahub-1.0.0.4rc8.dist-info → acryl_datahub-1.1.0.dist-info}/METADATA +2486 -2489
- {acryl_datahub-1.0.0.4rc8.dist-info → acryl_datahub-1.1.0.dist-info}/RECORD +15 -15
- {acryl_datahub-1.0.0.4rc8.dist-info → acryl_datahub-1.1.0.dist-info}/WHEEL +1 -1
- datahub/_version.py +1 -1
- datahub/api/graphql/base.py +8 -6
- datahub/entrypoints.py +2 -1
- datahub/ingestion/graph/client.py +4 -4
- datahub/ingestion/graph/filters.py +4 -4
- datahub/ingestion/source/looker/looker_common.py +51 -5
- datahub/ingestion/source/mode.py +37 -18
- datahub/sql_parsing/sqlglot_lineage.py +158 -50
- datahub/upgrade/upgrade.py +4 -2
- {acryl_datahub-1.0.0.4rc8.dist-info → acryl_datahub-1.1.0.dist-info}/entry_points.txt +0 -0
- {acryl_datahub-1.0.0.4rc8.dist-info → acryl_datahub-1.1.0.dist-info}/licenses/LICENSE +0 -0
- {acryl_datahub-1.0.0.4rc8.dist-info → acryl_datahub-1.1.0.dist-info}/top_level.txt +0 -0
|
@@ -1,8 +1,8 @@
|
|
|
1
|
-
acryl_datahub-1.
|
|
1
|
+
acryl_datahub-1.1.0.dist-info/licenses/LICENSE,sha256=9xNHpsD0uYF5ONzXsKDCuHHB-xbiCrSbueWXqrTNsxk,11365
|
|
2
2
|
datahub/__init__.py,sha256=aq_i5lVREmoLfYIqcx_pEQicO855YlhD19tWc1eZZNI,59
|
|
3
3
|
datahub/__main__.py,sha256=pegIvQ9hzK7IhqVeUi1MeADSZ2QlP-D3K0OQdEg55RU,106
|
|
4
|
-
datahub/_version.py,sha256=
|
|
5
|
-
datahub/entrypoints.py,sha256=
|
|
4
|
+
datahub/_version.py,sha256=nkKU-PpZCDruOUe__t5Nlc4bAXl7mS88YnUAFaL4qbo,318
|
|
5
|
+
datahub/entrypoints.py,sha256=H-YFTvxTJOgpWsFBVlxyb1opjkq-hjTzNmjy5Fq3RHg,8992
|
|
6
6
|
datahub/errors.py,sha256=p5rFAdAGVCk4Lqolol1YvthceadUSwpaCxLXRcyCCFQ,676
|
|
7
7
|
datahub/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
8
8
|
datahub/_codegen/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
@@ -58,7 +58,7 @@ datahub/api/entities/structuredproperties/__init__.py,sha256=47DEQpj8HBSa-_TImW-
|
|
|
58
58
|
datahub/api/entities/structuredproperties/structuredproperties.py,sha256=FU50bB1bgGDbitdzK9gHxWMj83KUuxK80mhpK1KmGZQ,8535
|
|
59
59
|
datahub/api/graphql/__init__.py,sha256=5yl0dJxO-2d_QuykdJrDIbWq4ja9bo0t2dAEh89JOog,142
|
|
60
60
|
datahub/api/graphql/assertion.py,sha256=o_q6SV7N1rJTVMNKSUBGJnZPk6TcVYoVShgDmPw65dE,2817
|
|
61
|
-
datahub/api/graphql/base.py,sha256=
|
|
61
|
+
datahub/api/graphql/base.py,sha256=zk724_oYSJ0nK7X7Z80MijnA6ry9JqpxnBsJeYuONKA,1737
|
|
62
62
|
datahub/api/graphql/operation.py,sha256=7E80HyE-5JLfLbFkQbgJeNwIaKngjBCrWES8eJO4OYc,5112
|
|
63
63
|
datahub/cli/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
64
64
|
datahub/cli/check_cli.py,sha256=82mO4a1_tEa9ewNzWAL6zxYEdVfUPOj0aH688X8_h28,14392
|
|
@@ -171,11 +171,11 @@ datahub/ingestion/glossary/classifier.py,sha256=daLxnVv_JlfB_jBOxH5LrU_xQRndrsGo
|
|
|
171
171
|
datahub/ingestion/glossary/classifier_registry.py,sha256=yFOYLQhDgCLqXYMG3L1BquXafeLcZDcmp8meyw6k9ts,307
|
|
172
172
|
datahub/ingestion/glossary/datahub_classifier.py,sha256=O7wm6gQT1Jf2QSKdWjJQbS5oSzJwplXzfza26Gdq5Mg,7555
|
|
173
173
|
datahub/ingestion/graph/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
174
|
-
datahub/ingestion/graph/client.py,sha256
|
|
174
|
+
datahub/ingestion/graph/client.py,sha256=-EnFcLnwJ1OWQCCBMhqCavUVtZZLH1DgmcZs6dnJqKo,70623
|
|
175
175
|
datahub/ingestion/graph/config.py,sha256=hNu2CztDhUeWXoQA34mU-54H3aN4i3vMUfOoWYhihp8,958
|
|
176
176
|
datahub/ingestion/graph/connections.py,sha256=9462L0ZWGKURyypAln25eMPhK3pcufBar9tNDoqspXs,741
|
|
177
177
|
datahub/ingestion/graph/entity_versioning.py,sha256=nrcNz0Qm6kpE6oTu_mrYUQDx14KPspBTc6R9SyFUY6c,6901
|
|
178
|
-
datahub/ingestion/graph/filters.py,sha256=
|
|
178
|
+
datahub/ingestion/graph/filters.py,sha256=OfjKhuNRHHLvhHk6Tfwd2IbMLPbbIq4VUyHaSpcDvKk,8664
|
|
179
179
|
datahub/ingestion/graph/links.py,sha256=UwWSdx-j0dPttfJOjfTf4ZmlO7iIsRz5p3nIsqGVHUA,2169
|
|
180
180
|
datahub/ingestion/reporting/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
181
181
|
datahub/ingestion/reporting/datahub_ingestion_run_summary_provider.py,sha256=iEulcZMLBQuUfe9MAYyobMekvMcNm4dqVcS_C_2KfrI,9736
|
|
@@ -207,7 +207,7 @@ datahub/ingestion/source/glue_profiling_config.py,sha256=vpMJH4Lf_qgR32BZy58suab
|
|
|
207
207
|
datahub/ingestion/source/ldap.py,sha256=PKoA5pVjuIxFfW1TcbYNIWSm7-C7shK2FDn7Zo5mrVM,18705
|
|
208
208
|
datahub/ingestion/source/metabase.py,sha256=j8DRV2GvisezidL1JZ5HJLF_hdFdtvaoyDoEdEyh0Ks,32603
|
|
209
209
|
datahub/ingestion/source/mlflow.py,sha256=fh7izN9jlSwbpGIrEyJktlmwFZR5vNG9z9L5VQ31k_4,33141
|
|
210
|
-
datahub/ingestion/source/mode.py,sha256=
|
|
210
|
+
datahub/ingestion/source/mode.py,sha256=g3nhkpW5KS_w3a8JaKWoq3XBNOZKFlmxZq9XI2D5dXY,72161
|
|
211
211
|
datahub/ingestion/source/mongodb.py,sha256=2C2Cxn8DXL53IbNiywIuKt8UT_EMcPg9f8su-OPSNGU,21237
|
|
212
212
|
datahub/ingestion/source/nifi.py,sha256=D1gBXxdpLuUQ0eurwofIR_SGg1rHGhwk3qxsWI1PT9c,56882
|
|
213
213
|
datahub/ingestion/source/openapi.py,sha256=zx976zstg6M2KoTz_iKKgU9VETDeX2rnw6BofiHXbDc,18669
|
|
@@ -353,7 +353,7 @@ datahub/ingestion/source/kafka_connect/sink_connectors.py,sha256=kEnxOXTik5HSDLj
|
|
|
353
353
|
datahub/ingestion/source/kafka_connect/source_connectors.py,sha256=OQ0vjz9xF0T30pRln_gDvelmaOE5jTAxwsCtm1K4SWM,21080
|
|
354
354
|
datahub/ingestion/source/looker/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
355
355
|
datahub/ingestion/source/looker/lkml_patched.py,sha256=XShEU7Wbz0DubDhYMjKf9wjKZrBJa2XPg9MIjp8rPhk,733
|
|
356
|
-
datahub/ingestion/source/looker/looker_common.py,sha256=
|
|
356
|
+
datahub/ingestion/source/looker/looker_common.py,sha256=wwCRHyd0F799CEBboryQ4TZV057krnY2nMJOkUWc268,64765
|
|
357
357
|
datahub/ingestion/source/looker/looker_config.py,sha256=eVKw1nn9D8hUFdRfNyT3MtzL8w-zWhFeokiwSnNKQuc,13607
|
|
358
358
|
datahub/ingestion/source/looker/looker_connection.py,sha256=yDmC6lDsHmL2e_Pw8ULylwOIHPWPp_6gT1iyLvD0fTw,2075
|
|
359
359
|
datahub/ingestion/source/looker/looker_constant.py,sha256=GMKYtNXlpojPxa9azridKfcGLSJwKdUCTesp7U8dIrQ,402
|
|
@@ -948,7 +948,7 @@ datahub/sql_parsing/split_statements.py,sha256=OIQXA9e4k3G9Z1y7rbgdtZhMWt4FPnq41
|
|
|
948
948
|
datahub/sql_parsing/sql_parsing_aggregator.py,sha256=Hg8dyrZtsCjQEDLIFddnvYg8KBAumrj3QHP6xSKfAio,71427
|
|
949
949
|
datahub/sql_parsing/sql_parsing_common.py,sha256=cZ4WvVyHZuXDGjnBvKMX2_fz2EMextB5WQWcK0_saBo,3155
|
|
950
950
|
datahub/sql_parsing/sql_parsing_result_utils.py,sha256=prwWTj1EB2fRPv1eMB4EkpFNafIYAt-X8TIK0NWqank,796
|
|
951
|
-
datahub/sql_parsing/sqlglot_lineage.py,sha256=
|
|
951
|
+
datahub/sql_parsing/sqlglot_lineage.py,sha256=6tuVv64MPO4i2VsmO9pjvP5IBWLEGollT3Ayubj6MU4,58668
|
|
952
952
|
datahub/sql_parsing/sqlglot_utils.py,sha256=TI11oBu1wrGeUuUGBg7hGTr6lTvztahdqiqXNJYRfbQ,14823
|
|
953
953
|
datahub/sql_parsing/tool_meta_extractor.py,sha256=EV_g7sOchTSUm2p6wluNJqND7-rDYokVTqqFCM7hQ6c,7599
|
|
954
954
|
datahub/telemetry/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
@@ -964,7 +964,7 @@ datahub/testing/doctest.py,sha256=1_8WEhHZ2eRQtw8vsXKzr9L5zzvs0Tcr6q4mnkyyvtw,29
|
|
|
964
964
|
datahub/testing/mcp_diff.py,sha256=1BpQ3hST46cOQi1SmKdsto3j6x6Sk6yHm0vG1w9IDL0,10749
|
|
965
965
|
datahub/testing/pytest_hooks.py,sha256=eifmj0M68AIfjTn_-0vtaBkKl75vNKMjsbYX-pJqmGY,1417
|
|
966
966
|
datahub/upgrade/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
967
|
-
datahub/upgrade/upgrade.py,sha256=
|
|
967
|
+
datahub/upgrade/upgrade.py,sha256=K90hxU5bdi3j45s6KR5iLSfywJ_wXyZl4ADND8BKz7k,16732
|
|
968
968
|
datahub/utilities/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
969
969
|
datahub/utilities/_custom_package_loader.py,sha256=9kgPE7Y77E-hNee8l4sKtVby-btUNum3dBfDixMzcVA,2059
|
|
970
970
|
datahub/utilities/_markupsafe_compat.py,sha256=QX7c9KiHs56ASl7bJlgR4FAf3CGiY94zIr0h6Ak15To,444
|
|
@@ -1054,8 +1054,8 @@ datahub_provider/operators/datahub_assertion_operator.py,sha256=uvTQ-jk2F0sbqqxp
|
|
|
1054
1054
|
datahub_provider/operators/datahub_assertion_sensor.py,sha256=lCBj_3x1cf5GMNpHdfkpHuyHfVxsm6ff5x2Z5iizcAo,140
|
|
1055
1055
|
datahub_provider/operators/datahub_operation_operator.py,sha256=aevDp2FzX7FxGlXrR0khoHNbxbhKR2qPEX5e8O2Jyzw,174
|
|
1056
1056
|
datahub_provider/operators/datahub_operation_sensor.py,sha256=8fcdVBCEPgqy1etTXgLoiHoJrRt_nzFZQMdSzHqSG7M,168
|
|
1057
|
-
acryl_datahub-1.
|
|
1058
|
-
acryl_datahub-1.
|
|
1059
|
-
acryl_datahub-1.
|
|
1060
|
-
acryl_datahub-1.
|
|
1061
|
-
acryl_datahub-1.
|
|
1057
|
+
acryl_datahub-1.1.0.dist-info/METADATA,sha256=HTj_32cFGYxZhyAZ0bQcMicc16taaI-zp8M2gxB8Nak,180520
|
|
1058
|
+
acryl_datahub-1.1.0.dist-info/WHEEL,sha256=Nw36Djuh_5VDukK0H78QzOX-_FQEo6V37m3nkm96gtU,91
|
|
1059
|
+
acryl_datahub-1.1.0.dist-info/entry_points.txt,sha256=o3mDeJXSKhsy7XLkuogihraiabBdLn9HaizYXPrxmk0,9710
|
|
1060
|
+
acryl_datahub-1.1.0.dist-info/top_level.txt,sha256=iLjSrLK5ox1YVYcglRUkcvfZPvKlobBWx7CTUXx8_GI,25
|
|
1061
|
+
acryl_datahub-1.1.0.dist-info/RECORD,,
|
datahub/_version.py
CHANGED
datahub/api/graphql/base.py
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
from typing import Dict, List, Optional
|
|
1
|
+
from typing import Dict, List, Optional, Union
|
|
2
2
|
|
|
3
3
|
from gql import Client
|
|
4
4
|
from gql.transport.requests import RequestsHTTPTransport
|
|
@@ -39,16 +39,18 @@ class BaseApi:
|
|
|
39
39
|
|
|
40
40
|
def gen_filter(
|
|
41
41
|
self, filters: Dict[str, Optional[str]]
|
|
42
|
-
) -> Optional[Dict[str, List[Dict[str, str]]]]:
|
|
43
|
-
filter_expression: Optional[
|
|
42
|
+
) -> Optional[Dict[str, List[Dict[str, Union[str, List[str]]]]]]:
|
|
43
|
+
filter_expression: Optional[
|
|
44
|
+
Dict[str, List[Dict[str, Union[str, List[str]]]]]
|
|
45
|
+
] = None
|
|
44
46
|
if not filters:
|
|
45
47
|
return None
|
|
46
48
|
|
|
47
|
-
|
|
49
|
+
filter_list: List[Dict[str, Union[str, List[str]]]] = []
|
|
48
50
|
for key, value in filters.items():
|
|
49
51
|
if value is None:
|
|
50
52
|
continue
|
|
51
|
-
|
|
53
|
+
filter_list.append({"field": key, "values": [value]})
|
|
52
54
|
|
|
53
|
-
filter_expression = {"and":
|
|
55
|
+
filter_expression = {"and": filter_list}
|
|
54
56
|
return filter_expression
|
datahub/entrypoints.py
CHANGED
|
@@ -50,8 +50,9 @@ MAX_CONTENT_WIDTH = 120
|
|
|
50
50
|
|
|
51
51
|
if sys.version_info >= (3, 12):
|
|
52
52
|
click.secho(
|
|
53
|
-
"Python versions above 3.11 are not tested with. Please use Python 3.11.",
|
|
53
|
+
"Python versions above 3.11 are not actively tested with yet. Please use Python 3.11 for now.",
|
|
54
54
|
fg="red",
|
|
55
|
+
err=True,
|
|
55
56
|
)
|
|
56
57
|
|
|
57
58
|
|
|
@@ -474,7 +474,7 @@ class DataHubGraph(DatahubRestEmitter, EntityVersioningAPI):
|
|
|
474
474
|
filter_criteria_map: Dict[str, str],
|
|
475
475
|
) -> Optional[Aspect]:
|
|
476
476
|
filter_criteria = [
|
|
477
|
-
{"field": k, "
|
|
477
|
+
{"field": k, "values": [v], "condition": "EQUAL"}
|
|
478
478
|
for k, v in filter_criteria_map.items()
|
|
479
479
|
]
|
|
480
480
|
filter = {"or": [{"and": filter_criteria}]}
|
|
@@ -669,7 +669,7 @@ class DataHubGraph(DatahubRestEmitter, EntityVersioningAPI):
|
|
|
669
669
|
filter_criteria = [
|
|
670
670
|
{
|
|
671
671
|
"field": "name",
|
|
672
|
-
"
|
|
672
|
+
"values": [domain_name],
|
|
673
673
|
"condition": "EQUAL",
|
|
674
674
|
}
|
|
675
675
|
]
|
|
@@ -789,7 +789,7 @@ class DataHubGraph(DatahubRestEmitter, EntityVersioningAPI):
|
|
|
789
789
|
filter_criteria.append(
|
|
790
790
|
{
|
|
791
791
|
"field": "customProperties",
|
|
792
|
-
"
|
|
792
|
+
"values": [f"instance={env}"],
|
|
793
793
|
"condition": "EQUAL",
|
|
794
794
|
}
|
|
795
795
|
)
|
|
@@ -797,7 +797,7 @@ class DataHubGraph(DatahubRestEmitter, EntityVersioningAPI):
|
|
|
797
797
|
filter_criteria.append(
|
|
798
798
|
{
|
|
799
799
|
"field": "typeNames",
|
|
800
|
-
"
|
|
800
|
+
"values": [container_subtype],
|
|
801
801
|
"condition": "EQUAL",
|
|
802
802
|
}
|
|
803
803
|
)
|
|
@@ -168,7 +168,7 @@ def _get_env_filters(env: str) -> List[RawSearchFilterRule]:
|
|
|
168
168
|
# For most entity types, we look at the origin field.
|
|
169
169
|
{
|
|
170
170
|
"field": "origin",
|
|
171
|
-
"
|
|
171
|
+
"values": [env],
|
|
172
172
|
"condition": "EQUAL",
|
|
173
173
|
},
|
|
174
174
|
# For containers, we look at the customProperties field.
|
|
@@ -176,15 +176,15 @@ def _get_env_filters(env: str) -> List[RawSearchFilterRule]:
|
|
|
176
176
|
# we look for the "env" property. Otherwise, we use the "instance" property.
|
|
177
177
|
{
|
|
178
178
|
"field": "customProperties",
|
|
179
|
-
"
|
|
179
|
+
"values": [f"env={env}"],
|
|
180
180
|
},
|
|
181
181
|
{
|
|
182
182
|
"field": "customProperties",
|
|
183
|
-
"
|
|
183
|
+
"values": [f"instance={env}"],
|
|
184
184
|
},
|
|
185
185
|
{
|
|
186
186
|
"field": "env",
|
|
187
|
-
"
|
|
187
|
+
"values": [env],
|
|
188
188
|
},
|
|
189
189
|
# Note that not all entity types have an env (e.g. dashboards / charts).
|
|
190
190
|
# If the env filter is specified, these will be excluded.
|
|
@@ -299,6 +299,7 @@ class ViewField:
|
|
|
299
299
|
view_name: Optional[str] = None
|
|
300
300
|
is_primary_key: bool = False
|
|
301
301
|
tags: List[str] = dataclasses_field(default_factory=list)
|
|
302
|
+
group_label: Optional[str] = None
|
|
302
303
|
|
|
303
304
|
# It is the list of ColumnRef for derived view defined using SQL otherwise simple column name
|
|
304
305
|
upstream_fields: Union[List[ColumnRef]] = dataclasses_field(default_factory=list)
|
|
@@ -326,6 +327,7 @@ class ViewField:
|
|
|
326
327
|
description = field_dict.get("description", default_description)
|
|
327
328
|
|
|
328
329
|
label = field_dict.get("label", "")
|
|
330
|
+
group_label = field_dict.get("group_label")
|
|
329
331
|
|
|
330
332
|
return ViewField(
|
|
331
333
|
name=name,
|
|
@@ -336,6 +338,7 @@ class ViewField:
|
|
|
336
338
|
field_type=type_cls,
|
|
337
339
|
tags=field_dict.get("tags") or [],
|
|
338
340
|
upstream_fields=upstream_column_ref,
|
|
341
|
+
group_label=group_label,
|
|
339
342
|
)
|
|
340
343
|
|
|
341
344
|
|
|
@@ -704,14 +707,47 @@ class LookerUtil:
|
|
|
704
707
|
),
|
|
705
708
|
}
|
|
706
709
|
|
|
710
|
+
# Add a pattern-based regex for checking if a tag is a group_label tag
|
|
711
|
+
GROUP_LABEL_TAG_PATTERN = re.compile(r"^looker\:group_label\:(.+)$")
|
|
712
|
+
|
|
707
713
|
@staticmethod
|
|
708
714
|
def _get_tag_mce_for_urn(tag_urn: str) -> MetadataChangeEvent:
|
|
709
|
-
|
|
710
|
-
|
|
711
|
-
|
|
712
|
-
|
|
715
|
+
# Check if this is a group_label tag
|
|
716
|
+
tag_name = tag_urn[len("urn:li:tag:") :]
|
|
717
|
+
match = LookerUtil.GROUP_LABEL_TAG_PATTERN.match(tag_name)
|
|
718
|
+
|
|
719
|
+
if match:
|
|
720
|
+
# This is a group_label tag, create tag definition on the fly
|
|
721
|
+
group_label_value = match.group(1)
|
|
722
|
+
tag_properties = TagPropertiesClass(
|
|
723
|
+
name=f"looker:group_label:{group_label_value}",
|
|
724
|
+
description=f"Fields with Looker group label: {group_label_value}",
|
|
725
|
+
)
|
|
726
|
+
|
|
727
|
+
return MetadataChangeEvent(
|
|
728
|
+
proposedSnapshot=TagSnapshotClass(urn=tag_urn, aspects=[tag_properties])
|
|
729
|
+
)
|
|
730
|
+
elif tag_urn in LookerUtil.tag_definitions:
|
|
731
|
+
# This is a predefined tag
|
|
732
|
+
return MetadataChangeEvent(
|
|
733
|
+
proposedSnapshot=TagSnapshotClass(
|
|
734
|
+
urn=tag_urn, aspects=[LookerUtil.tag_definitions[tag_urn]]
|
|
735
|
+
)
|
|
736
|
+
)
|
|
737
|
+
else:
|
|
738
|
+
# Should not happen, but handle gracefully
|
|
739
|
+
logger.warning(f"No tag definition found for tag URN: {tag_urn}")
|
|
740
|
+
return MetadataChangeEvent(
|
|
741
|
+
proposedSnapshot=TagSnapshotClass(
|
|
742
|
+
urn=tag_urn,
|
|
743
|
+
aspects=[
|
|
744
|
+
TagPropertiesClass(
|
|
745
|
+
name=tag_name,
|
|
746
|
+
description=f"Tag: {tag_name}",
|
|
747
|
+
)
|
|
748
|
+
],
|
|
749
|
+
)
|
|
713
750
|
)
|
|
714
|
-
)
|
|
715
751
|
|
|
716
752
|
@staticmethod
|
|
717
753
|
def _get_tags_from_field_type(
|
|
@@ -735,6 +771,14 @@ class LookerUtil:
|
|
|
735
771
|
message=f"Failed to map view field type {field.field_type}. Won't emit tags for measure and dimension",
|
|
736
772
|
)
|
|
737
773
|
|
|
774
|
+
# Add group_label as tags if present
|
|
775
|
+
if field.group_label:
|
|
776
|
+
schema_field_tags.append(
|
|
777
|
+
TagAssociationClass(
|
|
778
|
+
tag=builder.make_tag_urn(f"looker:group_label:{field.group_label}")
|
|
779
|
+
)
|
|
780
|
+
)
|
|
781
|
+
|
|
738
782
|
if schema_field_tags:
|
|
739
783
|
return GlobalTagsClass(tags=schema_field_tags)
|
|
740
784
|
|
|
@@ -1030,6 +1074,7 @@ class LookerExplore:
|
|
|
1030
1074
|
else False
|
|
1031
1075
|
),
|
|
1032
1076
|
upstream_fields=[],
|
|
1077
|
+
group_label=dim_field.field_group_label,
|
|
1033
1078
|
)
|
|
1034
1079
|
)
|
|
1035
1080
|
if explore.fields.measures is not None:
|
|
@@ -1068,6 +1113,7 @@ class LookerExplore:
|
|
|
1068
1113
|
else False
|
|
1069
1114
|
),
|
|
1070
1115
|
upstream_fields=[],
|
|
1116
|
+
group_label=measure_field.field_group_label,
|
|
1071
1117
|
)
|
|
1072
1118
|
)
|
|
1073
1119
|
|
datahub/ingestion/source/mode.py
CHANGED
|
@@ -1467,11 +1467,18 @@ class ModeSource(StatefulIngestionSourceBase):
|
|
|
1467
1467
|
)
|
|
1468
1468
|
yield reports_page
|
|
1469
1469
|
except ModeRequestError as e:
|
|
1470
|
-
|
|
1471
|
-
|
|
1472
|
-
|
|
1473
|
-
|
|
1474
|
-
|
|
1470
|
+
if isinstance(e, HTTPError) and e.response.status_code == 404:
|
|
1471
|
+
self.report.report_warning(
|
|
1472
|
+
title="No Reports Found in Space",
|
|
1473
|
+
message="No reports were found in the space. It may have been recently deleted.",
|
|
1474
|
+
context=f"Space Token: {space_token}, Error: {str(e)}",
|
|
1475
|
+
)
|
|
1476
|
+
else:
|
|
1477
|
+
self.report.report_failure(
|
|
1478
|
+
title="Failed to Retrieve Reports for Space",
|
|
1479
|
+
message="Unable to retrieve reports for space token.",
|
|
1480
|
+
context=f"Space Token: {space_token}, Error: {str(e)}",
|
|
1481
|
+
)
|
|
1475
1482
|
|
|
1476
1483
|
def _get_datasets(self, space_token: str) -> Iterator[List[dict]]:
|
|
1477
1484
|
"""
|
|
@@ -1490,11 +1497,18 @@ class ModeSource(StatefulIngestionSourceBase):
|
|
|
1490
1497
|
)
|
|
1491
1498
|
yield dataset_page
|
|
1492
1499
|
except ModeRequestError as e:
|
|
1493
|
-
|
|
1494
|
-
|
|
1495
|
-
|
|
1496
|
-
|
|
1497
|
-
|
|
1500
|
+
if isinstance(e, HTTPError) and e.response.status_code == 404:
|
|
1501
|
+
self.report.report_warning(
|
|
1502
|
+
title="No Datasets Found in Space",
|
|
1503
|
+
message="No datasets were found in the space. It may have been recently deleted.",
|
|
1504
|
+
context=f"Space Token: {space_token}, Error: {str(e)}",
|
|
1505
|
+
)
|
|
1506
|
+
else:
|
|
1507
|
+
self.report.report_failure(
|
|
1508
|
+
title="Failed to Retrieve Datasets for Space",
|
|
1509
|
+
message=f"Unable to retrieve datasets for space token {space_token}.",
|
|
1510
|
+
context=f"Space Token: {space_token}, Error: {str(e)}",
|
|
1511
|
+
)
|
|
1498
1512
|
|
|
1499
1513
|
def _get_queries(self, report_token: str) -> List[dict]:
|
|
1500
1514
|
try:
|
|
@@ -1555,13 +1569,18 @@ class ModeSource(StatefulIngestionSourceBase):
|
|
|
1555
1569
|
)
|
|
1556
1570
|
return charts.get("_embedded", {}).get("charts", [])
|
|
1557
1571
|
except ModeRequestError as e:
|
|
1558
|
-
|
|
1559
|
-
|
|
1560
|
-
|
|
1561
|
-
|
|
1562
|
-
|
|
1563
|
-
|
|
1564
|
-
|
|
1572
|
+
if isinstance(e, HTTPError) and e.response.status_code == 404:
|
|
1573
|
+
self.report.report_warning(
|
|
1574
|
+
title="No Charts Found for Query",
|
|
1575
|
+
message="No charts were found for the query. The query may have been recently deleted.",
|
|
1576
|
+
context=f"Report Token: {report_token}, Query Token: {query_token}, Error: {str(e)}",
|
|
1577
|
+
)
|
|
1578
|
+
else:
|
|
1579
|
+
self.report.report_failure(
|
|
1580
|
+
title="Failed to Retrieve Charts",
|
|
1581
|
+
message="Unable to retrieve charts from Mode.",
|
|
1582
|
+
context=f"Report Token: {report_token}, Query Token: {query_token}, Error: {str(e)}",
|
|
1583
|
+
)
|
|
1565
1584
|
return []
|
|
1566
1585
|
|
|
1567
1586
|
def _get_paged_request_json(
|
|
@@ -1577,7 +1596,7 @@ class ModeSource(StatefulIngestionSourceBase):
|
|
|
1577
1596
|
yield data
|
|
1578
1597
|
page += 1
|
|
1579
1598
|
|
|
1580
|
-
@lru_cache(maxsize=
|
|
1599
|
+
@lru_cache(maxsize=None)
|
|
1581
1600
|
def _get_request_json(self, url: str) -> Dict:
|
|
1582
1601
|
r = tenacity.Retrying(
|
|
1583
1602
|
wait=wait_exponential(
|