acryl-datahub 1.2.0.10rc4__py3-none-any.whl → 1.2.0.10rc6__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of acryl-datahub might be problematic. Click here for more details.
- {acryl_datahub-1.2.0.10rc4.dist-info → acryl_datahub-1.2.0.10rc6.dist-info}/METADATA +2451 -2425
- {acryl_datahub-1.2.0.10rc4.dist-info → acryl_datahub-1.2.0.10rc6.dist-info}/RECORD +21 -14
- {acryl_datahub-1.2.0.10rc4.dist-info → acryl_datahub-1.2.0.10rc6.dist-info}/entry_points.txt +2 -0
- datahub/_version.py +1 -1
- datahub/ingestion/autogenerated/capability_summary.json +33 -1
- datahub/ingestion/source/looker/looker_source.py +52 -3
- datahub/ingestion/source/redshift/query.py +23 -19
- datahub/ingestion/source/snaplogic/__init__.py +0 -0
- datahub/ingestion/source/snaplogic/snaplogic.py +355 -0
- datahub/ingestion/source/snaplogic/snaplogic_config.py +37 -0
- datahub/ingestion/source/snaplogic/snaplogic_lineage_extractor.py +107 -0
- datahub/ingestion/source/snaplogic/snaplogic_parser.py +168 -0
- datahub/ingestion/source/snaplogic/snaplogic_utils.py +31 -0
- datahub/ingestion/source/tableau/tableau.py +3 -3
- datahub/ingestion/transformer/set_browse_path.py +112 -0
- datahub/sdk/_shared.py +126 -0
- datahub/sdk/chart.py +87 -30
- datahub/sdk/dashboard.py +79 -32
- {acryl_datahub-1.2.0.10rc4.dist-info → acryl_datahub-1.2.0.10rc6.dist-info}/WHEEL +0 -0
- {acryl_datahub-1.2.0.10rc4.dist-info → acryl_datahub-1.2.0.10rc6.dist-info}/licenses/LICENSE +0 -0
- {acryl_datahub-1.2.0.10rc4.dist-info → acryl_datahub-1.2.0.10rc6.dist-info}/top_level.txt +0 -0
|
@@ -1,7 +1,7 @@
|
|
|
1
|
-
acryl_datahub-1.2.0.
|
|
1
|
+
acryl_datahub-1.2.0.10rc6.dist-info/licenses/LICENSE,sha256=9xNHpsD0uYF5ONzXsKDCuHHB-xbiCrSbueWXqrTNsxk,11365
|
|
2
2
|
datahub/__init__.py,sha256=aq_i5lVREmoLfYIqcx_pEQicO855YlhD19tWc1eZZNI,59
|
|
3
3
|
datahub/__main__.py,sha256=pegIvQ9hzK7IhqVeUi1MeADSZ2QlP-D3K0OQdEg55RU,106
|
|
4
|
-
datahub/_version.py,sha256=
|
|
4
|
+
datahub/_version.py,sha256=mSBfn0BVaAuAA6tvKy3C6K9XsxhU7zQ1rYUggP0EN0E,324
|
|
5
5
|
datahub/entrypoints.py,sha256=9Qf-37rNnTzbGlx8S75OCDazIclFp6zWNcCEL1zCZto,9015
|
|
6
6
|
datahub/errors.py,sha256=p5rFAdAGVCk4Lqolol1YvthceadUSwpaCxLXRcyCCFQ,676
|
|
7
7
|
datahub/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
@@ -159,7 +159,7 @@ datahub/ingestion/api/auto_work_units/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCe
|
|
|
159
159
|
datahub/ingestion/api/auto_work_units/auto_dataset_properties_aspect.py,sha256=ID_6N3nWl2qohsSGizUCqo3d2MNyDeVbyWroQpSOSsc,5059
|
|
160
160
|
datahub/ingestion/api/auto_work_units/auto_ensure_aspect_size.py,sha256=0BwkpLhORbsiTHq0g_N_1cVVoZYdLR3qz02mNmsV9-M,4444
|
|
161
161
|
datahub/ingestion/autogenerated/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
162
|
-
datahub/ingestion/autogenerated/capability_summary.json,sha256=
|
|
162
|
+
datahub/ingestion/autogenerated/capability_summary.json,sha256=cSA7jRLLphwkM9C9LK6HdbWRNM0s2febr-4Rh69vQss,111506
|
|
163
163
|
datahub/ingestion/autogenerated/lineage.json,sha256=8BdZF-5V5kJbX4mfFav8Zg-jHjzfkAEGk-pu1atLN4I,10029
|
|
164
164
|
datahub/ingestion/autogenerated/lineage_helper.py,sha256=I_k1pZSCCCjDbUVifPTfy6fkmV8jqdVhbirE8EkpmxI,4748
|
|
165
165
|
datahub/ingestion/extractor/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
@@ -392,7 +392,7 @@ datahub/ingestion/source/looker/looker_file_loader.py,sha256=gb2Z97_w28MsybYe01J
|
|
|
392
392
|
datahub/ingestion/source/looker/looker_lib_wrapper.py,sha256=6smUt_Ya7ZJMHWdGZl3TnhM7XHZVpYQ6gz2i5hHejZ4,11547
|
|
393
393
|
datahub/ingestion/source/looker/looker_liquid_tag.py,sha256=27WnOuTghayaH-HL4lLoq0IcHvNm1UybMqMnoaxN8Cs,5383
|
|
394
394
|
datahub/ingestion/source/looker/looker_query_model.py,sha256=N0jBbFruiCIIGT6sJn6tNeppeQ78KGTkOwTLirhxFNc,2144
|
|
395
|
-
datahub/ingestion/source/looker/looker_source.py,sha256=
|
|
395
|
+
datahub/ingestion/source/looker/looker_source.py,sha256=7mRlIJq2DoM1h2y-heNdNoqok8sNl7Qmpwsx0dQsYP8,67273
|
|
396
396
|
datahub/ingestion/source/looker/looker_template_language.py,sha256=5fZFPKFP3IYbJg3jLifjaji4wWg8wRy-1XDvc8Qucus,17949
|
|
397
397
|
datahub/ingestion/source/looker/looker_usage.py,sha256=qFBX7OHtIcarYIqFe0jQMrDV8MMPV_nN4PZrZRUznTw,23029
|
|
398
398
|
datahub/ingestion/source/looker/looker_view_id_cache.py,sha256=92gDy6NONhJYBp92z_IBzDVZvezmUIkaBCZY1bdk6mE,4392
|
|
@@ -450,7 +450,7 @@ datahub/ingestion/source/redshift/datashares.py,sha256=7G4YoEigZTl-K1aWDw3Jl8s3-
|
|
|
450
450
|
datahub/ingestion/source/redshift/exception.py,sha256=dxzYUIv5B_FAWhOuzG2u5We7FX-ar4jhOXPXAlEIvgM,2055
|
|
451
451
|
datahub/ingestion/source/redshift/lineage.py,sha256=nqrvWJqaI493i1hIZ_7patrdOb16sZrgSSGapdMcEiU,31710
|
|
452
452
|
datahub/ingestion/source/redshift/profile.py,sha256=H1Xtc2rXScUv4w0b2BbM7POjYEwqIql_rpWvlumY_EM,4309
|
|
453
|
-
datahub/ingestion/source/redshift/query.py,sha256=
|
|
453
|
+
datahub/ingestion/source/redshift/query.py,sha256=HKobQ-0crARgT8Mkfe-WBqVR9ZadYCZ9DGaUoEHHHww,48234
|
|
454
454
|
datahub/ingestion/source/redshift/redshift.py,sha256=zalndYg_LK5aJ8cX_ZuXLcTYajtlavmV-dmQIsjGxjg,41260
|
|
455
455
|
datahub/ingestion/source/redshift/redshift_data_reader.py,sha256=zc69jwXHdF-w8J4Hq-ZQ6BjHQ75Ij2iNDMpoRJlcmlU,1724
|
|
456
456
|
datahub/ingestion/source/redshift/redshift_schema.py,sha256=7F-l_omOuKMuGE_rBWXVPG_GWXFKnCMzC4frNxZB9cs,24800
|
|
@@ -482,6 +482,12 @@ datahub/ingestion/source/sigma/sigma.py,sha256=7dbkwk8_wp94XH9mhmtI_8ihR35cqYywt
|
|
|
482
482
|
datahub/ingestion/source/sigma/sigma_api.py,sha256=7PK5AQa838hYeaQ5L0dioi4n4bLrpN-r7COKTTNUYw8,19837
|
|
483
483
|
datahub/ingestion/source/slack/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
484
484
|
datahub/ingestion/source/slack/slack.py,sha256=JWanUfzFGynV_PWcH0YzJIbRcmL880DA6dEI9QW-QiQ,25800
|
|
485
|
+
datahub/ingestion/source/snaplogic/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
486
|
+
datahub/ingestion/source/snaplogic/snaplogic.py,sha256=VoCRSBS0kE4RAsn4hOJgQy3uadFvelLIOv3I0_gne-w,13540
|
|
487
|
+
datahub/ingestion/source/snaplogic/snaplogic_config.py,sha256=oxuNBfAHWMHoOvh52gifOFcBOSN8aaPpFC8QgmgXwWI,1445
|
|
488
|
+
datahub/ingestion/source/snaplogic/snaplogic_lineage_extractor.py,sha256=IzCYwXLN6IfRFqns7XXtZxzQjjHC-XNTGXgEMeSfW8U,3776
|
|
489
|
+
datahub/ingestion/source/snaplogic/snaplogic_parser.py,sha256=q5dRfWtOpSELPZrpyLbszOD49MJBXNbKgnITLMPiyGI,5783
|
|
490
|
+
datahub/ingestion/source/snaplogic/snaplogic_utils.py,sha256=SVrV9ZXVE2cKKPfoVsxjBN2fIcpYbs2PBLiyQIcJMVQ,1068
|
|
485
491
|
datahub/ingestion/source/snowflake/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
486
492
|
datahub/ingestion/source/snowflake/constants.py,sha256=iDTamMozHwLYyglpRfqwTbxPxYPhb-uJGRHIgDRHUkA,2767
|
|
487
493
|
datahub/ingestion/source/snowflake/oauth_config.py,sha256=ol9D3RmruGStJAeL8PYSQguSqcD2HfkjPkMF2AB_eZs,1277
|
|
@@ -554,7 +560,7 @@ datahub/ingestion/source/state_provider/datahub_ingestion_checkpointing_provider
|
|
|
554
560
|
datahub/ingestion/source/state_provider/file_ingestion_checkpointing_provider.py,sha256=DziD57PbHn2Tcy51tYXCG-GQgyTGMUxnkuzVS_xihFY,4079
|
|
555
561
|
datahub/ingestion/source/state_provider/state_provider_registry.py,sha256=SVq4mIyGNmLXE9OZx1taOiNPqDoQp03-Ot9rYnB5F3k,401
|
|
556
562
|
datahub/ingestion/source/tableau/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
557
|
-
datahub/ingestion/source/tableau/tableau.py,sha256=
|
|
563
|
+
datahub/ingestion/source/tableau/tableau.py,sha256=BgR_-IuzUYuaSV3EOvMcefwBZ_h0dYAvNEn5Dv6meA4,158448
|
|
558
564
|
datahub/ingestion/source/tableau/tableau_common.py,sha256=2vE7DIigPvMNcTCWSou0tliaVy9MgFR1qwqnE4pilw8,27086
|
|
559
565
|
datahub/ingestion/source/tableau/tableau_constant.py,sha256=2WPAHN-GAR83_c3eTTNd8cy0-zC8GIXeUdSxX_mNdas,2608
|
|
560
566
|
datahub/ingestion/source/tableau/tableau_server_wrapper.py,sha256=wsVD0SkGUwb-H9_g0aDclKwYkcoxugaWyAcyAMgBCAU,1136
|
|
@@ -612,6 +618,7 @@ datahub/ingestion/transformer/pattern_cleanup_dataset_usage_user.py,sha256=jTURu
|
|
|
612
618
|
datahub/ingestion/transformer/pattern_cleanup_ownership.py,sha256=YJH4lv1ztKAYwsdRz5RiUu6SX08h2GBr1S-9QlcmB18,3755
|
|
613
619
|
datahub/ingestion/transformer/remove_dataset_ownership.py,sha256=kHiIcT19BDKNIuCQqAj827E1ZEvME0eGGrILEACALRc,1195
|
|
614
620
|
datahub/ingestion/transformer/replace_external_url.py,sha256=Nw2V1m86fD1vMGLYGPlaoIJEORV0O9qAqWydL-1n2Ng,4058
|
|
621
|
+
datahub/ingestion/transformer/set_browse_path.py,sha256=K8Y4O9vjeJQCdYGsFuNE0aClT73HKBpmj51Yy6Jm_uQ,4065
|
|
615
622
|
datahub/ingestion/transformer/tags_to_terms.py,sha256=VDcd7cM5hGCxo6QP1x4RNEw5Q9v4WDxjRhQMpAl95-A,5558
|
|
616
623
|
datahub/ingestion/transformer/transform_registry.py,sha256=bartmA1zEaULNy5W1Q7gRF8h5Y57BFC6XNOGfCzh1Zw,251
|
|
617
624
|
datahub/integrations/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
@@ -957,11 +964,11 @@ datahub/pydantic/compat.py,sha256=TUEo4kSEeOWVAhV6LQtst1phrpVgGtK4uif4OI5vQ2M,19
|
|
|
957
964
|
datahub/sdk/__init__.py,sha256=66OOcFi7qlnL6q72c_yUX2mWU2HudbOdRsC5CIoDxow,1922
|
|
958
965
|
datahub/sdk/_all_entities.py,sha256=eQAmD_fcEHlTShe1_nHpdvHxLDN9njk9bdLnuTrYg8M,905
|
|
959
966
|
datahub/sdk/_attribution.py,sha256=0Trh8steVd27GOr9MKCZeawbuDD2_q3GIsZlCtHqEUg,1321
|
|
960
|
-
datahub/sdk/_shared.py,sha256=
|
|
967
|
+
datahub/sdk/_shared.py,sha256=tlyxyxgo7x-8sJhUG9VvUiLpxcOP-GIg4ToqPixagbw,34221
|
|
961
968
|
datahub/sdk/_utils.py,sha256=oXE2BzsXE5zmSkCP3R1tObD4RHnPeH_ps83D_Dw9JaQ,1169
|
|
962
|
-
datahub/sdk/chart.py,sha256=
|
|
969
|
+
datahub/sdk/chart.py,sha256=gsyq_saUInGJKm0s4wwZ9pLMyWLmodrDhrwdZ_Y2h9E,13902
|
|
963
970
|
datahub/sdk/container.py,sha256=IjnFVGDpSFDvgHuuMb7C3VdBxhJuIMq0q6crOs5PupE,7899
|
|
964
|
-
datahub/sdk/dashboard.py,sha256=
|
|
971
|
+
datahub/sdk/dashboard.py,sha256=Vsy_geQzwk3xzqpSuywbgQJ6T__nEvfK5rmrtvY2DPo,16812
|
|
965
972
|
datahub/sdk/dataflow.py,sha256=gdAPVVkyKvsKtsa1AwhN_LpzidG_XzV3nhtd1cjnzDA,11128
|
|
966
973
|
datahub/sdk/datajob.py,sha256=5kU0txTDcn2ce3AhNry83TazPVhoYZ2rAPPNWM1_FP8,13677
|
|
967
974
|
datahub/sdk/dataset.py,sha256=-C4TCJAs1PFkLAgkUZEU1JOg3orm7AAIkqjw7oo_4PQ,31400
|
|
@@ -1114,8 +1121,8 @@ datahub_provider/operators/datahub_assertion_operator.py,sha256=uvTQ-jk2F0sbqqxp
|
|
|
1114
1121
|
datahub_provider/operators/datahub_assertion_sensor.py,sha256=lCBj_3x1cf5GMNpHdfkpHuyHfVxsm6ff5x2Z5iizcAo,140
|
|
1115
1122
|
datahub_provider/operators/datahub_operation_operator.py,sha256=aevDp2FzX7FxGlXrR0khoHNbxbhKR2qPEX5e8O2Jyzw,174
|
|
1116
1123
|
datahub_provider/operators/datahub_operation_sensor.py,sha256=8fcdVBCEPgqy1etTXgLoiHoJrRt_nzFZQMdSzHqSG7M,168
|
|
1117
|
-
acryl_datahub-1.2.0.
|
|
1118
|
-
acryl_datahub-1.2.0.
|
|
1119
|
-
acryl_datahub-1.2.0.
|
|
1120
|
-
acryl_datahub-1.2.0.
|
|
1121
|
-
acryl_datahub-1.2.0.
|
|
1124
|
+
acryl_datahub-1.2.0.10rc6.dist-info/METADATA,sha256=Ruq1g6waFdCzmx2n0vm2h4XcrOfNZpDVjCxZWoLXPZ0,184162
|
|
1125
|
+
acryl_datahub-1.2.0.10rc6.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
|
|
1126
|
+
acryl_datahub-1.2.0.10rc6.dist-info/entry_points.txt,sha256=pzsBoTx-D-iTcmpX8oCGCyzlHP2112EygUMzZWz56M8,10105
|
|
1127
|
+
acryl_datahub-1.2.0.10rc6.dist-info/top_level.txt,sha256=iLjSrLK5ox1YVYcglRUkcvfZPvKlobBWx7CTUXx8_GI,25
|
|
1128
|
+
acryl_datahub-1.2.0.10rc6.dist-info/RECORD,,
|
{acryl_datahub-1.2.0.10rc4.dist-info → acryl_datahub-1.2.0.10rc6.dist-info}/entry_points.txt
RENAMED
|
@@ -94,6 +94,7 @@ sagemaker = datahub.ingestion.source.aws.sagemaker:SagemakerSource
|
|
|
94
94
|
salesforce = datahub.ingestion.source.salesforce:SalesforceSource
|
|
95
95
|
sigma = datahub.ingestion.source.sigma.sigma:SigmaSource
|
|
96
96
|
slack = datahub.ingestion.source.slack.slack:SlackSource
|
|
97
|
+
snaplogic = datahub.ingestion.source.snaplogic.snaplogic:SnaplogicSource
|
|
97
98
|
snowflake = datahub.ingestion.source.snowflake.snowflake_v2:SnowflakeV2Source
|
|
98
99
|
snowflake-queries = datahub.ingestion.source.snowflake.snowflake_queries:SnowflakeQueriesSource
|
|
99
100
|
snowflake-summary = datahub.ingestion.source.snowflake.snowflake_summary:SnowflakeSummarySource
|
|
@@ -130,6 +131,7 @@ pattern_cleanup_dataset_usage_user = datahub.ingestion.transformer.pattern_clean
|
|
|
130
131
|
pattern_cleanup_ownership = datahub.ingestion.transformer.pattern_cleanup_ownership:PatternCleanUpOwnership
|
|
131
132
|
replace_external_url = datahub.ingestion.transformer.replace_external_url:ReplaceExternalUrlDataset
|
|
132
133
|
replace_external_url_container = datahub.ingestion.transformer.replace_external_url:ReplaceExternalUrlContainer
|
|
134
|
+
set_browse_path = datahub.ingestion.transformer.set_browse_path:SetBrowsePathTransformer
|
|
133
135
|
set_dataset_browse_path = datahub.ingestion.transformer.add_dataset_browse_path:AddDatasetBrowsePathTransformer
|
|
134
136
|
simple_add_dataset_dataproduct = datahub.ingestion.transformer.add_dataset_dataproduct:SimpleAddDatasetDataProduct
|
|
135
137
|
simple_add_dataset_domain = datahub.ingestion.transformer.dataset_domain:SimpleAddDatasetDomain
|
datahub/_version.py
CHANGED
|
@@ -2968,6 +2968,38 @@
|
|
|
2968
2968
|
"platform_name": "Slack",
|
|
2969
2969
|
"support_status": "TESTING"
|
|
2970
2970
|
},
|
|
2971
|
+
"snaplogic": {
|
|
2972
|
+
"capabilities": [
|
|
2973
|
+
{
|
|
2974
|
+
"capability": "LINEAGE_FINE",
|
|
2975
|
+
"description": "Enabled by default",
|
|
2976
|
+
"subtype_modifier": null,
|
|
2977
|
+
"supported": true
|
|
2978
|
+
},
|
|
2979
|
+
{
|
|
2980
|
+
"capability": "DELETION_DETECTION",
|
|
2981
|
+
"description": "Not supported yet",
|
|
2982
|
+
"subtype_modifier": null,
|
|
2983
|
+
"supported": false
|
|
2984
|
+
},
|
|
2985
|
+
{
|
|
2986
|
+
"capability": "PLATFORM_INSTANCE",
|
|
2987
|
+
"description": "Snaplogic does not support platform instances",
|
|
2988
|
+
"subtype_modifier": null,
|
|
2989
|
+
"supported": false
|
|
2990
|
+
},
|
|
2991
|
+
{
|
|
2992
|
+
"capability": "LINEAGE_COARSE",
|
|
2993
|
+
"description": "Enabled by default",
|
|
2994
|
+
"subtype_modifier": null,
|
|
2995
|
+
"supported": true
|
|
2996
|
+
}
|
|
2997
|
+
],
|
|
2998
|
+
"classname": "datahub.ingestion.source.snaplogic.snaplogic.SnaplogicSource",
|
|
2999
|
+
"platform_id": "snaplogic",
|
|
3000
|
+
"platform_name": "Snaplogic",
|
|
3001
|
+
"support_status": "TESTING"
|
|
3002
|
+
},
|
|
2971
3003
|
"snowflake": {
|
|
2972
3004
|
"capabilities": [
|
|
2973
3005
|
{
|
|
@@ -3617,4 +3649,4 @@
|
|
|
3617
3649
|
"support_status": "CERTIFIED"
|
|
3618
3650
|
}
|
|
3619
3651
|
}
|
|
3620
|
-
}
|
|
3652
|
+
}
|
|
@@ -736,7 +736,16 @@ class LookerDashboardSource(TestableSource, StatefulIngestionSourceBase):
|
|
|
736
736
|
display_name=dashboard_element.title, # title is (deprecated) using display_name
|
|
737
737
|
extra_aspects=chart_extra_aspects,
|
|
738
738
|
input_datasets=dashboard_element.get_view_urns(self.source_config),
|
|
739
|
-
last_modified=self._get_last_modified_time(
|
|
739
|
+
last_modified=self._get_last_modified_time(
|
|
740
|
+
dashboard
|
|
741
|
+
), # Inherited from Dashboard
|
|
742
|
+
last_modified_by=self._get_last_modified_by(
|
|
743
|
+
dashboard
|
|
744
|
+
), # Inherited from Dashboard
|
|
745
|
+
created_at=self._get_created_at(dashboard), # Inherited from Dashboard
|
|
746
|
+
created_by=self._get_created_by(dashboard), # Inherited from Dashboard
|
|
747
|
+
deleted_on=self._get_deleted_on(dashboard), # Inherited from Dashboard
|
|
748
|
+
deleted_by=self._get_deleted_by(dashboard), # Inherited from Dashboard
|
|
740
749
|
name=dashboard_element.get_urn_element_id(),
|
|
741
750
|
owners=chart_ownership,
|
|
742
751
|
parent_container=chart_parent_container,
|
|
@@ -803,6 +812,11 @@ class LookerDashboardSource(TestableSource, StatefulIngestionSourceBase):
|
|
|
803
812
|
display_name=looker_dashboard.title, # title is (deprecated) using display_name
|
|
804
813
|
extra_aspects=dashboard_extra_aspects,
|
|
805
814
|
last_modified=self._get_last_modified_time(looker_dashboard),
|
|
815
|
+
last_modified_by=self._get_last_modified_by(looker_dashboard),
|
|
816
|
+
created_at=self._get_created_at(looker_dashboard),
|
|
817
|
+
created_by=self._get_created_by(looker_dashboard),
|
|
818
|
+
deleted_on=self._get_deleted_on(looker_dashboard),
|
|
819
|
+
deleted_by=self._get_deleted_by(looker_dashboard),
|
|
806
820
|
name=looker_dashboard.get_urn_dashboard_id(),
|
|
807
821
|
owners=dashboard_ownership,
|
|
808
822
|
parent_container=dashboard_parent_container,
|
|
@@ -988,9 +1002,44 @@ class LookerDashboardSource(TestableSource, StatefulIngestionSourceBase):
|
|
|
988
1002
|
def _get_last_modified_time(
|
|
989
1003
|
self, looker_dashboard: Optional[LookerDashboard]
|
|
990
1004
|
) -> Optional[datetime.datetime]:
|
|
991
|
-
if looker_dashboard
|
|
1005
|
+
return looker_dashboard.last_updated_at if looker_dashboard else None
|
|
1006
|
+
|
|
1007
|
+
def _get_last_modified_by(
|
|
1008
|
+
self, looker_dashboard: Optional[LookerDashboard]
|
|
1009
|
+
) -> Optional[str]:
|
|
1010
|
+
if not looker_dashboard or not looker_dashboard.last_updated_by:
|
|
1011
|
+
return None
|
|
1012
|
+
return looker_dashboard.last_updated_by.get_urn(
|
|
1013
|
+
self.source_config.strip_user_ids_from_email
|
|
1014
|
+
)
|
|
1015
|
+
|
|
1016
|
+
def _get_created_at(
|
|
1017
|
+
self, looker_dashboard: Optional[LookerDashboard]
|
|
1018
|
+
) -> Optional[datetime.datetime]:
|
|
1019
|
+
return looker_dashboard.created_at if looker_dashboard else None
|
|
1020
|
+
|
|
1021
|
+
def _get_created_by(
|
|
1022
|
+
self, looker_dashboard: Optional[LookerDashboard]
|
|
1023
|
+
) -> Optional[str]:
|
|
1024
|
+
if not looker_dashboard or not looker_dashboard.owner:
|
|
1025
|
+
return None
|
|
1026
|
+
return looker_dashboard.owner.get_urn(
|
|
1027
|
+
self.source_config.strip_user_ids_from_email
|
|
1028
|
+
)
|
|
1029
|
+
|
|
1030
|
+
def _get_deleted_on(
|
|
1031
|
+
self, looker_dashboard: Optional[LookerDashboard]
|
|
1032
|
+
) -> Optional[datetime.datetime]:
|
|
1033
|
+
return looker_dashboard.deleted_at if looker_dashboard else None
|
|
1034
|
+
|
|
1035
|
+
def _get_deleted_by(
|
|
1036
|
+
self, looker_dashboard: Optional[LookerDashboard]
|
|
1037
|
+
) -> Optional[str]:
|
|
1038
|
+
if not looker_dashboard or not looker_dashboard.deleted_by:
|
|
992
1039
|
return None
|
|
993
|
-
return looker_dashboard.
|
|
1040
|
+
return looker_dashboard.deleted_by.get_urn(
|
|
1041
|
+
self.source_config.strip_user_ids_from_email
|
|
1042
|
+
)
|
|
994
1043
|
|
|
995
1044
|
def _get_looker_folder(self, folder: Union[Folder, FolderBase]) -> LookerFolder:
|
|
996
1045
|
assert folder.id
|
|
@@ -89,7 +89,7 @@ class RedshiftCommonQuery:
|
|
|
89
89
|
) -> str:
|
|
90
90
|
# NOTE: it looks like description is available only in pg_description
|
|
91
91
|
# So this remains preferrred way
|
|
92
|
-
tables_query = """
|
|
92
|
+
tables_query = f"""
|
|
93
93
|
SELECT CASE c.relkind
|
|
94
94
|
WHEN 'r' THEN 'TABLE'
|
|
95
95
|
WHEN 'v' THEN 'VIEW'
|
|
@@ -120,6 +120,7 @@ class RedshiftCommonQuery:
|
|
|
120
120
|
LEFT JOIN pg_catalog.pg_namespace n ON n.oid = c.relnamespace
|
|
121
121
|
LEFT JOIN pg_class_info as ci on c.oid = ci.reloid
|
|
122
122
|
LEFT JOIN pg_catalog.pg_description pgd ON pgd.objsubid = 0 AND pgd.objoid = c.oid
|
|
123
|
+
JOIN svv_redshift_schemas rs ON rs.schema_name = n.nspname AND rs.database_name = '{database}'
|
|
123
124
|
WHERE c.relkind IN ('r','v','m','S','f')
|
|
124
125
|
AND n.nspname !~ '^pg_'
|
|
125
126
|
AND n.nspname != 'information_schema'
|
|
@@ -128,23 +129,24 @@ class RedshiftCommonQuery:
|
|
|
128
129
|
external_tables_query = f"""
|
|
129
130
|
SELECT 'EXTERNAL_TABLE' as tabletype,
|
|
130
131
|
NULL AS "schema_oid",
|
|
131
|
-
schemaname AS "schema",
|
|
132
|
+
t.schemaname AS "schema",
|
|
132
133
|
NULL AS "rel_oid",
|
|
133
|
-
tablename AS "relname",
|
|
134
|
+
t.tablename AS "relname",
|
|
134
135
|
NULL as "creation_time",
|
|
135
136
|
NULL AS "diststyle",
|
|
136
137
|
NULL AS "owner_id",
|
|
137
138
|
NULL AS "owner_name",
|
|
138
139
|
NULL AS "view_definition",
|
|
139
140
|
NULL AS "privileges",
|
|
140
|
-
"location",
|
|
141
|
-
parameters,
|
|
142
|
-
input_format,
|
|
143
|
-
output_format,
|
|
144
|
-
serde_parameters,
|
|
141
|
+
t."location",
|
|
142
|
+
t.parameters,
|
|
143
|
+
t.input_format,
|
|
144
|
+
t.output_format,
|
|
145
|
+
t.serde_parameters,
|
|
145
146
|
NULL as table_description
|
|
146
|
-
FROM pg_catalog.svv_external_tables
|
|
147
|
-
|
|
147
|
+
FROM pg_catalog.svv_external_tables t
|
|
148
|
+
JOIN SVV_EXTERNAL_SCHEMAS s ON t.schemaname = s.schemaname
|
|
149
|
+
WHERE t.redshift_database_name='{database}'
|
|
148
150
|
ORDER BY "schema",
|
|
149
151
|
"relname"
|
|
150
152
|
"""
|
|
@@ -232,11 +234,12 @@ class RedshiftCommonQuery:
|
|
|
232
234
|
ON att.attrelid = c.oid
|
|
233
235
|
LEFT JOIN pg_catalog.pg_attrdef ad
|
|
234
236
|
ON (att.attrelid, att.attnum) = (ad.adrelid, ad.adnum)
|
|
237
|
+
JOIN svv_redshift_schemas rs ON rs.schema_name = n.nspname AND rs.database_name = '{database_name}'
|
|
235
238
|
WHERE n.nspname !~ '^pg_'
|
|
236
239
|
AND n.nspname != 'information_schema'
|
|
237
240
|
AND att.attnum > 0
|
|
238
241
|
AND NOT att.attisdropped
|
|
239
|
-
and
|
|
242
|
+
and n.nspname = '{schema_name}'
|
|
240
243
|
UNION
|
|
241
244
|
SELECT
|
|
242
245
|
view_schema as "schema",
|
|
@@ -263,26 +266,27 @@ class RedshiftCommonQuery:
|
|
|
263
266
|
WHERE 1 and schema = '{schema_name}'
|
|
264
267
|
UNION
|
|
265
268
|
SELECT
|
|
266
|
-
schemaname as "schema",
|
|
267
|
-
tablename as "table_name",
|
|
268
|
-
columnname as "name",
|
|
269
|
+
c.schemaname as "schema",
|
|
270
|
+
c.tablename as "table_name",
|
|
271
|
+
c.columnname as "name",
|
|
269
272
|
null as "encode",
|
|
270
273
|
-- Spectrum represents data types differently.
|
|
271
274
|
-- Standardize, so we can infer types.
|
|
272
|
-
external_type AS "type",
|
|
275
|
+
c.external_type AS "type",
|
|
273
276
|
null as "distkey",
|
|
274
277
|
0 as "sortkey",
|
|
275
278
|
null as "notnull",
|
|
276
279
|
null as "comment",
|
|
277
280
|
null as "adsrc",
|
|
278
281
|
null as "attnum",
|
|
279
|
-
external_type AS "format_type",
|
|
282
|
+
c.external_type AS "format_type",
|
|
280
283
|
null as "default",
|
|
281
284
|
null as "schema_oid",
|
|
282
285
|
null as "table_oid"
|
|
283
|
-
FROM SVV_EXTERNAL_COLUMNS
|
|
284
|
-
|
|
285
|
-
|
|
286
|
+
FROM SVV_EXTERNAL_COLUMNS c
|
|
287
|
+
JOIN SVV_EXTERNAL_SCHEMAS s ON c.schemaname = s.schemaname
|
|
288
|
+
WHERE c.schemaname = '{schema_name}'
|
|
289
|
+
AND c.redshift_database_name = '{database_name}'
|
|
286
290
|
ORDER BY "schema", "table_name", "attnum"
|
|
287
291
|
"""
|
|
288
292
|
|
|
File without changes
|