acryl-datahub 0.15.0.1rc10__py3-none-any.whl → 0.15.0.1rc12__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of acryl-datahub might be problematic. Click here for more details.

Files changed (33) hide show
  1. {acryl_datahub-0.15.0.1rc10.dist-info → acryl_datahub-0.15.0.1rc12.dist-info}/METADATA +2376 -2380
  2. {acryl_datahub-0.15.0.1rc10.dist-info → acryl_datahub-0.15.0.1rc12.dist-info}/RECORD +33 -33
  3. datahub/__init__.py +1 -1
  4. datahub/api/circuit_breaker/assertion_circuit_breaker.py +5 -4
  5. datahub/configuration/common.py +2 -5
  6. datahub/emitter/mce_builder.py +17 -1
  7. datahub/emitter/mcp_builder.py +2 -7
  8. datahub/emitter/mcp_patch_builder.py +2 -2
  9. datahub/emitter/rest_emitter.py +2 -2
  10. datahub/ingestion/api/closeable.py +3 -3
  11. datahub/ingestion/api/ingestion_job_checkpointing_provider_base.py +4 -7
  12. datahub/ingestion/api/report.py +4 -1
  13. datahub/ingestion/api/sink.py +4 -3
  14. datahub/ingestion/api/source_helpers.py +2 -6
  15. datahub/ingestion/source/bigquery_v2/bigquery_schema.py +5 -20
  16. datahub/ingestion/source/datahub/datahub_kafka_reader.py +2 -1
  17. datahub/ingestion/source/gc/dataprocess_cleanup.py +19 -6
  18. datahub/ingestion/source/s3/source.py +1 -1
  19. datahub/ingestion/source/sql/hive.py +15 -0
  20. datahub/ingestion/source/sql/hive_metastore.py +7 -0
  21. datahub/ingestion/source/sql/mssql/source.py +1 -1
  22. datahub/ingestion/source/sql/sql_common.py +41 -102
  23. datahub/ingestion/source/sql/sql_generic_profiler.py +5 -6
  24. datahub/ingestion/source/sql/sql_report.py +2 -0
  25. datahub/ingestion/source/state/checkpoint.py +2 -1
  26. datahub/ingestion/source/tableau/tableau.py +14 -6
  27. datahub/ingestion/source/unity/proxy.py +8 -27
  28. datahub/metadata/_urns/urn_defs.py +168 -168
  29. datahub/utilities/time.py +8 -3
  30. datahub/utilities/urns/_urn_base.py +5 -7
  31. {acryl_datahub-0.15.0.1rc10.dist-info → acryl_datahub-0.15.0.1rc12.dist-info}/WHEEL +0 -0
  32. {acryl_datahub-0.15.0.1rc10.dist-info → acryl_datahub-0.15.0.1rc12.dist-info}/entry_points.txt +0 -0
  33. {acryl_datahub-0.15.0.1rc10.dist-info → acryl_datahub-0.15.0.1rc12.dist-info}/top_level.txt +0 -0
@@ -1,4 +1,4 @@
1
- datahub/__init__.py,sha256=_-iwjV9mhNtK3Q_48sB1x7crxfllh3ay-QVv4WQ8458,577
1
+ datahub/__init__.py,sha256=LdmQMvZSZxJJKW3u06itY2EkTfgOjWTGkJHo9YvmkV0,577
2
2
  datahub/__main__.py,sha256=pegIvQ9hzK7IhqVeUi1MeADSZ2QlP-D3K0OQdEg55RU,106
3
3
  datahub/entrypoints.py,sha256=3-qSfXAx3Z0FEkBV5tlO8fQr4xk4ySeDRMVTpS5Xd6A,7793
4
4
  datahub/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -6,7 +6,7 @@ datahub/_codegen/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,
6
6
  datahub/_codegen/aspect.py,sha256=PJRa-Z4ouXHq3OkulfyWhwZn-fFUBDK_UPvmqaWdbWk,1063
7
7
  datahub/api/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
8
8
  datahub/api/circuit_breaker/__init__.py,sha256=-DEDzzFNdX9OCYYj5QYinyv3KORT5TZ_H5h4K8eWG3c,537
9
- datahub/api/circuit_breaker/assertion_circuit_breaker.py,sha256=UnKrqj33RRQCdSQ7fDWsBQM5pUImfvM5bLc08drLGNE,5325
9
+ datahub/api/circuit_breaker/assertion_circuit_breaker.py,sha256=TfebJJHwi8oeoCXLzfBCgUBLXXGlyUnRWzth5CLhyn4,5378
10
10
  datahub/api/circuit_breaker/circuit_breaker.py,sha256=diowuNvlpG7rBypiqz1ZFoL0MGpbsLO_quaxDdrZkzw,1518
11
11
  datahub/api/circuit_breaker/operation_circuit_breaker.py,sha256=6HctXeYCfxmzZnQDV1kxWkdjZYReKoS3qGtVe54ZgNE,2908
12
12
  datahub/api/entities/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -89,7 +89,7 @@ datahub/cli/specific/structuredproperties_cli.py,sha256=qP7kHpN7y3cOR0IGZkD4PGlR
89
89
  datahub/cli/specific/user_cli.py,sha256=jGAokb1NRu8obs6P2g4OL2NQdFgpUBa9De55TBBtun0,1897
90
90
  datahub/configuration/__init__.py,sha256=5TN3a7CWNsLRHpdj-sv2bxKWF2IslvJwE6EpNMFrIS4,123
91
91
  datahub/configuration/_config_enum.py,sha256=ul2hr5gMmdLvBINicFkMNMi1ApmnmZSwNdUYYted5nk,1447
92
- datahub/configuration/common.py,sha256=Ngj2-HKPEhCMbcx3phUqyoOHayhqWNt1t0e2hO3GQNY,10508
92
+ datahub/configuration/common.py,sha256=QN256dCSCZSxbl6JtS_rjcs1i-T3veTDxlZhMhYhkys,10416
93
93
  datahub/configuration/config_loader.py,sha256=4V8rrbKvCbfEys2Tlw2uZXb3yC9Hpoubn2O8GXhGe3A,5785
94
94
  datahub/configuration/connection_resolver.py,sha256=n4-6MwMiOEDgTouxO0SMjTILKVhJPo6-naE6FuR5qMs,1516
95
95
  datahub/configuration/datetimes.py,sha256=nayNc0mmlVKH6oVv9ud6C1dDUiZPGabW-YZxvrkosPg,2870
@@ -114,32 +114,32 @@ datahub/emitter/aspect.py,sha256=ef0DVycqg-tRPurkYjc-5zknmLP2p2Y2RxP55WkvAEc,480
114
114
  datahub/emitter/enum_helpers.py,sha256=ZeALUAPi10Q4Z6VM0_WiU9Y60_d0ugZHcUoVmuOCEec,321
115
115
  datahub/emitter/generic_emitter.py,sha256=i37ZFm9VR_tmiZm9kIypEkQEB_cLKbzj_tJvViN-fm8,828
116
116
  datahub/emitter/kafka_emitter.py,sha256=Uix1W1WaXF8VqUTUfzdRZKca2XrR1w50Anx2LVkROlc,5822
117
- datahub/emitter/mce_builder.py,sha256=CMES7gTj4w_kZhOsWuxbLwdz5e2tdiilibnZXstRGhw,16117
117
+ datahub/emitter/mce_builder.py,sha256=5oZHXs85GGwfL8tY72IPnicyYrRXraN4LgtVQQcZyq8,16417
118
118
  datahub/emitter/mcp.py,sha256=hAAYziDdkwjazQU0DtWMbQWY8wS09ACrKJbqxoWXdgc,9637
119
- datahub/emitter/mcp_builder.py,sha256=ju-1dZMKs5dlWcTi4zcNRVmhkfhmfX3JFULZSbgxSFs,9968
120
- datahub/emitter/mcp_patch_builder.py,sha256=W85q1maVUMpOIo5lwLRn82rLXRVoZ_gurl_a-pvVCpE,4291
119
+ datahub/emitter/mcp_builder.py,sha256=eOcuz41c4a3oTkNk39yYl9bTxpksxqATPHLcqyhPGT0,9856
120
+ datahub/emitter/mcp_patch_builder.py,sha256=ykQFJshFrVF6DjkjcHQ8ZhDEws3ki0gmNjkHNfQtHwQ,4277
121
121
  datahub/emitter/request_helper.py,sha256=33ORG3S3OVy97_jlWBRn7yUM5XCIkRN6WSdJvN7Ofcg,670
122
- datahub/emitter/rest_emitter.py,sha256=d5Zjo3GXDu9rUqlSsK9aOx-yEbHjDFZHelfq_ZFeb5M,16393
122
+ datahub/emitter/rest_emitter.py,sha256=oqyRuXG1o1dYjiEIH5TFMb1q0xhRbpxPIA5qkyz0iQ8,16407
123
123
  datahub/emitter/serialization_helper.py,sha256=q12Avmf70Vy4ttQGMJoTKlE5EsybMKNg2w3MQeZiHvk,3652
124
124
  datahub/emitter/sql_parsing_builder.py,sha256=Cr5imZrm3dYDSCACt5MFscgHCtVbHTD6IjUmsvsKoEs,11991
125
125
  datahub/emitter/synchronized_file_emitter.py,sha256=s4ATuxalI4GDAkrZTaGSegxBdvvNPZ9jRSdtElU0kNs,1805
126
126
  datahub/ingestion/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
127
127
  datahub/ingestion/api/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
128
- datahub/ingestion/api/closeable.py,sha256=c86iu0Av3i9qMhlwN_kSVIiYBnBa3TN1I0jsKArFP8M,625
128
+ datahub/ingestion/api/closeable.py,sha256=k12AT--s4GDtZ-po_rVm5QKgvGIDteeRPByZPIOfecA,599
129
129
  datahub/ingestion/api/committable.py,sha256=4S6GuBzvX2vb1A8P506NbspOKfZ1621sBG8t0lvRb8o,886
130
130
  datahub/ingestion/api/common.py,sha256=nJVL8YdvokYFajOjmVpSNlLbZJ5iVOFS4KJDlGtJ_jc,2735
131
131
  datahub/ingestion/api/decorators.py,sha256=KTNdf2B20L-wlEPF8UsL89a8zwvRSOfA7gOOZnnYalY,3933
132
132
  datahub/ingestion/api/global_context.py,sha256=OdSJg4a_RKE52nu8MSiEkK2UqRRDhDTyOleHEAzPKho,575
133
133
  datahub/ingestion/api/incremental_lineage_helper.py,sha256=Qke8T4Yba0c-ZzNfSzzXnFP0WjuqUQ1fuN4V3KK4gv4,5913
134
134
  datahub/ingestion/api/incremental_properties_helper.py,sha256=KzdxdrQtaMV2XMHfPsCtRa7ffDGPA1w1hgPUjeenZBU,2514
135
- datahub/ingestion/api/ingestion_job_checkpointing_provider_base.py,sha256=g5jfu3rEdwFRdWx0K5VoqYhXu1cVtwADCZHP7SdBLf0,1839
135
+ datahub/ingestion/api/ingestion_job_checkpointing_provider_base.py,sha256=3lLdkkxVqE9MVc26cdXImPeWy16az5BwgcorWxeBV50,1759
136
136
  datahub/ingestion/api/pipeline_run_listener.py,sha256=5uBP__LbMQxJ2utlf07cIzQINqPbUOKiZyOJta6a0og,713
137
137
  datahub/ingestion/api/registry.py,sha256=LGElUdzhNQoEr-k2SN23mJaIYnA1PYfF97LQxBmWmD8,7262
138
- datahub/ingestion/api/report.py,sha256=CpQHqLAoYGV4bxNIpYQugLY0EUoxROlp2NUM9ONHj_I,4364
138
+ datahub/ingestion/api/report.py,sha256=zb5Y_9ogmWm00KqX7_64sIMT24Wfpk7txRwEfKacw5I,4652
139
139
  datahub/ingestion/api/report_helpers.py,sha256=WbUC1kQeaKqIagGV3XzfPmPs7slAT1mfNY4og2BH2A8,994
140
- datahub/ingestion/api/sink.py,sha256=6g01wou8pv79s0leDWyK12cgl7eLtpiwSUHqOw08vx4,4503
140
+ datahub/ingestion/api/sink.py,sha256=3jw7-x9gXGreOPwn49wG5fT3C8pYhaNMQITdMN6kbag,4478
141
141
  datahub/ingestion/api/source.py,sha256=pHfFIBZa57ySpZWnt03mmayWLdbbBAGOhWqWZnf1KUA,18815
142
- datahub/ingestion/api/source_helpers.py,sha256=k40ofHHPsfbYFJgZXWaD6ORvEa0SBmsaOacF1ttolcQ,19743
142
+ datahub/ingestion/api/source_helpers.py,sha256=AVO0ogiCKgYmX1ubJaSs6L30TCCgOIalp6awXPF5XM0,19643
143
143
  datahub/ingestion/api/transform.py,sha256=X0GpjMJzYkLuZx8MTWxH50cWGm9rGsnn3k188mmC8J8,582
144
144
  datahub/ingestion/api/workunit.py,sha256=e8n8RfSjHZZm2R4ShNH0UuMtUkMjyqqM2j2t7oL74lo,6327
145
145
  datahub/ingestion/api/auto_work_units/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -242,7 +242,7 @@ datahub/ingestion/source/bigquery_v2/bigquery_helper.py,sha256=QER3gY8e_k1_eNVj7
242
242
  datahub/ingestion/source/bigquery_v2/bigquery_platform_resource_helper.py,sha256=8nuQ8hMuJEswWDZtV2RjbK8RvDJUzT_S74dnyPpGFdQ,4857
243
243
  datahub/ingestion/source/bigquery_v2/bigquery_queries.py,sha256=EoHo9twb0_QdX7Nvd1HJC1Yn0rqtrfR52EVk7Hu3XOQ,3296
244
244
  datahub/ingestion/source/bigquery_v2/bigquery_report.py,sha256=WxiLPFc7LwZXNDYfV9oySUD43kc2GcOf_pUokp3vFNM,8098
245
- datahub/ingestion/source/bigquery_v2/bigquery_schema.py,sha256=tyKPfxg88ERX6z7Q5lCmJS-H7cgsWRUwM2epqe62bI0,32277
245
+ datahub/ingestion/source/bigquery_v2/bigquery_schema.py,sha256=E5GOx4NWjyZM0xzdpBlNXbvDdKNfW9UtS64XtCYFpzI,31809
246
246
  datahub/ingestion/source/bigquery_v2/bigquery_schema_gen.py,sha256=Sv6BrK62nu3xpgjYGE-x1xdSTouvvnKDJtazPobhiKQ,50813
247
247
  datahub/ingestion/source/bigquery_v2/bigquery_test_connection.py,sha256=cATxwi5IPzj3BldRRAVcLqzSFmmYEPvqa7U0RFJbaAc,7645
248
248
  datahub/ingestion/source/bigquery_v2/common.py,sha256=Cxjf1a8ibkL_YRQeS0BqsjlyMgFJpaZ3iq_d7e8T8MQ,4030
@@ -268,7 +268,7 @@ datahub/ingestion/source/datahub/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRk
268
268
  datahub/ingestion/source/datahub/config.py,sha256=rqZFvEmjxjBcW2cTEPYDVTAk3OLzuGIjEFghXPNeZNY,3955
269
269
  datahub/ingestion/source/datahub/datahub_api_reader.py,sha256=hlKADVEPoTFiRGKqRsMF5mL4fSu_IrIW8Nx7LpEzvkM,2134
270
270
  datahub/ingestion/source/datahub/datahub_database_reader.py,sha256=F8JrOjSrmJ2B6m1MWh83A1EYFDcGMla749HUeQWMnL0,9464
271
- datahub/ingestion/source/datahub/datahub_kafka_reader.py,sha256=8x9_u5kRjgSmu7c295ZIZjxP6bgoZZbWsKRicuLStRQ,4145
271
+ datahub/ingestion/source/datahub/datahub_kafka_reader.py,sha256=gnxhhlK-jrfnHqD_4eVmfcdtBNW6pi1N_qkDZ7uSb3o,4187
272
272
  datahub/ingestion/source/datahub/datahub_source.py,sha256=2jDnsHEzpGhr00qQI9unSUJYD6Cb1McYFKOVbA-Zcm4,8487
273
273
  datahub/ingestion/source/datahub/report.py,sha256=VHBfCbwFRzdLdB7hQG9ST4EiZxl_vBCU0XxGcZR6Xxs,940
274
274
  datahub/ingestion/source/datahub/state.py,sha256=PZoT7sSK1wadVf5vN6phrgr7I6LL7ePP-EJjP1OO0bQ,3507
@@ -303,7 +303,7 @@ datahub/ingestion/source/fivetran/fivetran_log_api.py,sha256=EAak3hJpe75WZSgz6wP
303
303
  datahub/ingestion/source/fivetran/fivetran_query.py,sha256=vLrTj7e-0NxZ2U4bWTB57pih42WirqPlUvwtIRfStlQ,5275
304
304
  datahub/ingestion/source/gc/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
305
305
  datahub/ingestion/source/gc/datahub_gc.py,sha256=WOg3yIaNmwdbSTwytKeSfIUihsM7FMYBip9u2Dnwk3c,12849
306
- datahub/ingestion/source/gc/dataprocess_cleanup.py,sha256=u90XEmW1vRFbvp4CQ8ujPxTGJUyJqO2U6ApcI6mFrjE,16588
306
+ datahub/ingestion/source/gc/dataprocess_cleanup.py,sha256=9brJW_HVrxJk1kAP20M7flmgYOMemOmaEl2zheWFW3c,17105
307
307
  datahub/ingestion/source/gc/execution_request_cleanup.py,sha256=sZbdkg3MuPVGf8eeeRg_2khGMZ01QoH4dgJiTxf7Srg,9813
308
308
  datahub/ingestion/source/gc/soft_deleted_entity_cleanup.py,sha256=wRnRaIVUG483tY4nyDkEn6Xi2RL5MjrVvoCoZimqwSg,7514
309
309
  datahub/ingestion/source/gcs/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -403,7 +403,7 @@ datahub/ingestion/source/s3/config.py,sha256=Zs1nrBZKLImteZreIcSMMRLj8vBGgxakNDs
403
403
  datahub/ingestion/source/s3/datalake_profiler_config.py,sha256=k7S9Xcmgr3-CvWrd5NEX-V8JSrcAwkm7vbHPTVZicow,3620
404
404
  datahub/ingestion/source/s3/profiling.py,sha256=yKNCKpr6w7qpCH-baeSkNE9VjkN6eBot_weD-2_Jxzk,17579
405
405
  datahub/ingestion/source/s3/report.py,sha256=fzkTdTewYlWrTk4f2Cyl-e8RV4qw9wEVtm0cdKD-Xgw,542
406
- datahub/ingestion/source/s3/source.py,sha256=8O_vu1J91h7owQlYyK27AZAQHxKsDpNC_jsLNpMed98,47336
406
+ datahub/ingestion/source/s3/source.py,sha256=USjq86vUU7hKYKi8bhplBhHOjvoQTgguy91uFE24rUI,47336
407
407
  datahub/ingestion/source/sac/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
408
408
  datahub/ingestion/source/sac/sac.py,sha256=zPSO9ukuyhvNaaVzeAYpA-_sFma_XMcCQMPaGvDWuTk,30226
409
409
  datahub/ingestion/source/sac/sac_common.py,sha256=-xQTDBtgH56AnpRXWGDnlmQqUuLRx-7wF1U1kQFWtX8,998
@@ -451,18 +451,18 @@ datahub/ingestion/source/sql/clickhouse.py,sha256=jzvaXP5Wr0SMhj2rtuvVE821xnfpKi
451
451
  datahub/ingestion/source/sql/cockroachdb.py,sha256=XaD7eae34plU9ISRC6PzYX9q6RdT2qkzjH6CpTOgkx4,1443
452
452
  datahub/ingestion/source/sql/druid.py,sha256=lhO9CCOlHV-6LjBuAxAxtB9I1pvPtsGSdr63bz6_ilA,2837
453
453
  datahub/ingestion/source/sql/hana.py,sha256=0PIvcX0Rz59NyR7Ag5Bv1MBV_UbJwxl9UAopo_xe_CA,1342
454
- datahub/ingestion/source/sql/hive.py,sha256=Ueicc3EhKQSC77IDzl_xcTDDalbH1pvB8mox1oFI4Hw,29738
455
- datahub/ingestion/source/sql/hive_metastore.py,sha256=PisLrswev583xW0xDJ5yfKWCWm_ZTl1OeuaMcv_SvXc,35865
454
+ datahub/ingestion/source/sql/hive.py,sha256=NRUrEWnR1JN5U0q4CHlRacdKzxJhS4unFXnXYZT7vZE,30306
455
+ datahub/ingestion/source/sql/hive_metastore.py,sha256=n9WvJzGBYVwjSUKuAWQcYuRJttH81k2S2zjHuw8gvME,36074
456
456
  datahub/ingestion/source/sql/mariadb.py,sha256=Hm102kmfs_1rd4lsTYhzVMZq5S3B6cyfvpHSzJjqvMw,737
457
457
  datahub/ingestion/source/sql/mysql.py,sha256=nDWK4YbqomcJgnit9b8geUGrp_3eix4bt0_k94o7g-0,3350
458
458
  datahub/ingestion/source/sql/oracle.py,sha256=ibBtjaneCFto-Rw3k2OxsbT3YHgux1aCtPtv5oA8St4,24533
459
459
  datahub/ingestion/source/sql/postgres.py,sha256=uC1kYEI8VdxiZ1Y9IxMWzwmg11wtMqYN0e2fkok1rxo,11972
460
460
  datahub/ingestion/source/sql/presto.py,sha256=PB-CS5MX2dSRFRHjlxfkLHGXLZXFNCsVAAyRBtY6HMg,3611
461
- datahub/ingestion/source/sql/sql_common.py,sha256=FMLbRQVSxLWjLNTiBBQVb9SG4F7h9lwsfNzAoKhOyAU,51356
461
+ datahub/ingestion/source/sql/sql_common.py,sha256=E1QmJ35ZuDLiZj-s1niHvIdNMyEsZrwvq_Wuy2EoYMQ,48586
462
462
  datahub/ingestion/source/sql/sql_config.py,sha256=M-l_uXau0ODolLZHBzAXhy-Rq5yYxvJ6cLbCIea7Mww,9449
463
463
  datahub/ingestion/source/sql/sql_generic.py,sha256=9AERvkK8kdJUeDOzCYJDb93xdv6Z4DGho0NfeHj5Uyg,2740
464
- datahub/ingestion/source/sql/sql_generic_profiler.py,sha256=6QbhkQH_F13GV1HsavVTq3BE9F7Pr_vfGOjCX2o2c60,11675
465
- datahub/ingestion/source/sql/sql_report.py,sha256=Knh6KhmuO2jIgBI5ChEBoBrWYPJYOl7wNze03gCuI2Q,2539
464
+ datahub/ingestion/source/sql/sql_generic_profiler.py,sha256=oLjqgsxVKGerj5dZnCCRMremrxjp-kr5_P45gFOM4Pg,11602
465
+ datahub/ingestion/source/sql/sql_report.py,sha256=gw-OPHSExp_b6DRjvwqE1U6BpkwekxGrsvNMGYSGDio,2671
466
466
  datahub/ingestion/source/sql/sql_types.py,sha256=lrJpavRTE7aDVAKOrKZcrp4CsKydiiaza1wt2ieqWzs,15041
467
467
  datahub/ingestion/source/sql/sql_utils.py,sha256=w9YFNm_qJNjOcWAWBI_lUoFMbd0wT8q0LoT7Ia71tIE,8100
468
468
  datahub/ingestion/source/sql/sqlalchemy_data_reader.py,sha256=FvHZ4JEK3aR2DYOBZiT_ZsAy12RjTu4t_KIR_92B11k,2644
@@ -473,10 +473,10 @@ datahub/ingestion/source/sql/two_tier_sql_source.py,sha256=YDrGBb5WKVls6qv17QU5f
473
473
  datahub/ingestion/source/sql/vertica.py,sha256=_9OgSgIgqBml0av063rb8nACiT3SAmzpw0ouyF91wv8,33382
474
474
  datahub/ingestion/source/sql/mssql/__init__.py,sha256=1agpl8S_uDW40olkhCX_W19dbr5GO9qgjS3R7pLRZSk,87
475
475
  datahub/ingestion/source/sql/mssql/job_models.py,sha256=ztXDrD4anhzwWvACIm9fucE2WhMDMKkJ4alMYOQOqWA,7083
476
- datahub/ingestion/source/sql/mssql/source.py,sha256=ODdsOIbDA3X0E7En6GT15mD49W6RW9sXLwRoUgw2a8I,30925
476
+ datahub/ingestion/source/sql/mssql/source.py,sha256=WV2rU_sN5pqd4MEu6p4kwQRpADFjG0qh27tx7qP5AOw,30931
477
477
  datahub/ingestion/source/sql/mssql/stored_procedure_lineage.py,sha256=RpnvKPalAAaOD_eUg8bZ4VkGTSeLFWuy0mefwc4s3x8,2837
478
478
  datahub/ingestion/source/state/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
479
- datahub/ingestion/source/state/checkpoint.py,sha256=x9Xww-MIFXSKjeg1tOZXE72LehCm5OfKy3HfucgIRWM,8833
479
+ datahub/ingestion/source/state/checkpoint.py,sha256=-fTUZKkY4nHTFqSWZ0jJkkdIu_tWlOjRNhm4FTr4ul4,8860
480
480
  datahub/ingestion/source/state/entity_removal_state.py,sha256=zvIsmYg7oiIu2FhecU0VfLBNToUqvKoKyDeiFfkOcyc,6611
481
481
  datahub/ingestion/source/state/profiling_state.py,sha256=lsWu7oZhB9nSlqoklvjs-LjS4XF0p6BxSAcLY-xKRzM,512
482
482
  datahub/ingestion/source/state/profiling_state_handler.py,sha256=IgvmTszxjR53JX-uTNjFx7ZMWK34DlAWjA3QKSLa644,4293
@@ -491,7 +491,7 @@ datahub/ingestion/source/state_provider/datahub_ingestion_checkpointing_provider
491
491
  datahub/ingestion/source/state_provider/file_ingestion_checkpointing_provider.py,sha256=xsH7Ao_05VTjqpkzLkhdf5B1ULMzFoD8vkJJIJU9w-U,4077
492
492
  datahub/ingestion/source/state_provider/state_provider_registry.py,sha256=SVq4mIyGNmLXE9OZx1taOiNPqDoQp03-Ot9rYnB5F3k,401
493
493
  datahub/ingestion/source/tableau/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
494
- datahub/ingestion/source/tableau/tableau.py,sha256=nu71B56vB6h5io5GcMXQPlYVCbE-UNAtdxHCm8nXr9o,139751
494
+ datahub/ingestion/source/tableau/tableau.py,sha256=SWEJi0LoIhb8rVVmmhVxngENo53QtXFvJE02aOIzG6Q,140034
495
495
  datahub/ingestion/source/tableau/tableau_common.py,sha256=9gQLq_3BlAsKll83uVlnWJRWaIDtFtREUyuimXF13Z0,26219
496
496
  datahub/ingestion/source/tableau/tableau_constant.py,sha256=ZcAeHsQUXVVL26ORly0ByZk_GJAFbxaKuJAlX_sYMac,2686
497
497
  datahub/ingestion/source/tableau/tableau_server_wrapper.py,sha256=nSyx9RzC6TCQDm-cTVJ657qT8iDwzk_8JMKpohhmOc4,1046
@@ -502,7 +502,7 @@ datahub/ingestion/source/unity/config.py,sha256=m4-n7mYz4Ct4L1QdfJFklwHyj8boKCbV
502
502
  datahub/ingestion/source/unity/connection_test.py,sha256=B143Wb28fS0V4GhygU9hzKqiArWBjsQO54IUCPf23dc,2586
503
503
  datahub/ingestion/source/unity/ge_profiler.py,sha256=DFQKOqryMWFg-NqwfFGPklNH2hHSmZGKs8ij8QmXd7w,6402
504
504
  datahub/ingestion/source/unity/hive_metastore_proxy.py,sha256=IAWWJjaW0si_UF52Se2D7wmdYRY_afUG4QlVmQu6xaw,15351
505
- datahub/ingestion/source/unity/proxy.py,sha256=2-pYQ-3B9UVUwO1yB9iTdi3DqgqZ2JrpQknLodI7UjM,18976
505
+ datahub/ingestion/source/unity/proxy.py,sha256=_6kCI7M4-26pZ9ZMGJUh6LwYmbGAZlnvc8GY4yd6QAs,18403
506
506
  datahub/ingestion/source/unity/proxy_profiling.py,sha256=WLqvYP6MziaisA4LYL4T_GA-kPt6Xdde7bfaYsjYw40,9663
507
507
  datahub/ingestion/source/unity/proxy_types.py,sha256=qrvHiwPzl5cPX-KRvcIGGeJVdr0I8XUQmoAI6ErZ-v8,9371
508
508
  datahub/ingestion/source/unity/report.py,sha256=0Y-ciHVTI6ZKNCJ5zWoQh3Ze1c_GMqmTMKFwzXDuuOg,2788
@@ -571,7 +571,7 @@ datahub/metadata/schema.avsc,sha256=Ulqzumt0EK7nD_OATi0hbCgw42ngoenja9SXWWsobIk,
571
571
  datahub/metadata/schema_classes.py,sha256=X5Jl5EaSxyHdXOQv14pJ5WkQALun4MRpJ4q12wVFE18,1299
572
572
  datahub/metadata/urns.py,sha256=nfrCTExR-k2P9w272WVtWSN3xW1VUJngPwP3xnvULjU,1217
573
573
  datahub/metadata/_urns/__init__.py,sha256=cOF3GHMDgPhmbLKbN02NPpuLGHSu0qNgQyBRv08eqF0,243
574
- datahub/metadata/_urns/urn_defs.py,sha256=LFHZGzHlDA0KJes1Xg7-lWetXusi7bubA7Q5hu4ER88,107119
574
+ datahub/metadata/_urns/urn_defs.py,sha256=gcUHCVwelD5aSuPgE1vmao242tQQiHe2m9kH8Fs1y1E,107287
575
575
  datahub/metadata/com/__init__.py,sha256=gsAIuTxzfJdI7a9ybZlgMIHMAYksM1SxGxXjtySgKSc,202
576
576
  datahub/metadata/com/linkedin/__init__.py,sha256=gsAIuTxzfJdI7a9ybZlgMIHMAYksM1SxGxXjtySgKSc,202
577
577
  datahub/metadata/com/linkedin/events/__init__.py,sha256=s_dR0plZF-rOxxIbE8ojekJqwiHzl2WYR-Z3kW6kKS0,298
@@ -937,7 +937,7 @@ datahub/utilities/str_enum.py,sha256=EsqCLPbrqyQ2YU_wt7QP-a6P5fnpIshXJ3AI8gLBlVA
937
937
  datahub/utilities/tee_io.py,sha256=jBrsUfTPTk9IICntfGOG0HR-Fjp8BQMde-FPQ4r3kuI,601
938
938
  datahub/utilities/threaded_iterator_executor.py,sha256=WC4tvJ4TQRkH0VO_FD91GbedcKUqx0lc4tHDNOiF6ps,1770
939
939
  datahub/utilities/threading_timeout.py,sha256=hOzDI55E3onXblHNwGsePJUWMXo5zqaWCnoYdL2-KPM,1316
940
- datahub/utilities/time.py,sha256=RkjtFDEwYh_Koxf-g9hAC_z9Z1Hgth7gMVZp9oQr3Vc,1635
940
+ datahub/utilities/time.py,sha256=Q7S_Zyom8C2zcl2xFbjNw6K8nZsCub5XGAB4OEmIS34,1847
941
941
  datahub/utilities/topological_sort.py,sha256=kcK5zPSR393fgItr-KSLV3bDqfJfBRS8E5kkCpPBgUY,1358
942
942
  datahub/utilities/type_annotations.py,sha256=FvcB__a6X0CLoz-sBXwqpdceqSqTHgkLXGQ6wSmiV8w,970
943
943
  datahub/utilities/unified_diff.py,sha256=8uRvM_kN-sdAzR4Ym6CgmpjrmO4CrcKtzZ4P-Cn6aEA,8422
@@ -947,7 +947,7 @@ datahub/utilities/yaml_sync_utils.py,sha256=65IEe8quW3_zHCR8CyoDkZyopeZJazU-IyMr
947
947
  datahub/utilities/registries/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
948
948
  datahub/utilities/registries/domain_registry.py,sha256=0SfcZNop-PXBbl-AWw92vAyb28i0YXTr-TKdBwixmOw,2452
949
949
  datahub/utilities/urns/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
950
- datahub/utilities/urns/_urn_base.py,sha256=IOYjC3NPn0DlB26cPdyihFAah3mUc4CHpS2dVM65ctI,9295
950
+ datahub/utilities/urns/_urn_base.py,sha256=oly7C6EWmEA0kAfy9TxidH8U97ouRFo2CyJuXZgfLi8,9211
951
951
  datahub/utilities/urns/corp_group_urn.py,sha256=6H5Q6nZvAXu80IZBDCeM8xo_9ap9pgwtyi60QXx3hzY,75
952
952
  datahub/utilities/urns/corpuser_urn.py,sha256=h-Yh-9QRbtQOhxxzxEBc7skoavpGaKDKVNrsxSXZ1yQ,88
953
953
  datahub/utilities/urns/data_flow_urn.py,sha256=w1Z7ET1L1OtYD1w-xiUYtyCczsxZZ1l3LRyTRv5NdpE,73
@@ -982,8 +982,8 @@ datahub_provider/operators/datahub_assertion_operator.py,sha256=uvTQ-jk2F0sbqqxp
982
982
  datahub_provider/operators/datahub_assertion_sensor.py,sha256=lCBj_3x1cf5GMNpHdfkpHuyHfVxsm6ff5x2Z5iizcAo,140
983
983
  datahub_provider/operators/datahub_operation_operator.py,sha256=aevDp2FzX7FxGlXrR0khoHNbxbhKR2qPEX5e8O2Jyzw,174
984
984
  datahub_provider/operators/datahub_operation_sensor.py,sha256=8fcdVBCEPgqy1etTXgLoiHoJrRt_nzFZQMdSzHqSG7M,168
985
- acryl_datahub-0.15.0.1rc10.dist-info/METADATA,sha256=GCgEH25gXF0roGuAivBGRw1IyiBv_Xv4wbWj9jGlpIo,173645
986
- acryl_datahub-0.15.0.1rc10.dist-info/WHEEL,sha256=PZUExdf71Ui_so67QXpySuHtCi3-J3wvF4ORK6k_S8U,91
987
- acryl_datahub-0.15.0.1rc10.dist-info/entry_points.txt,sha256=xnPSPLK3bJGADxe4TDS4wL4u0FT_PGlahDa-ENYdYCQ,9512
988
- acryl_datahub-0.15.0.1rc10.dist-info/top_level.txt,sha256=iLjSrLK5ox1YVYcglRUkcvfZPvKlobBWx7CTUXx8_GI,25
989
- acryl_datahub-0.15.0.1rc10.dist-info/RECORD,,
985
+ acryl_datahub-0.15.0.1rc12.dist-info/METADATA,sha256=w8H0vrzaAbeZnX_mqNopX7I929V_AYXeWGKidlrUrE8,173444
986
+ acryl_datahub-0.15.0.1rc12.dist-info/WHEEL,sha256=PZUExdf71Ui_so67QXpySuHtCi3-J3wvF4ORK6k_S8U,91
987
+ acryl_datahub-0.15.0.1rc12.dist-info/entry_points.txt,sha256=xnPSPLK3bJGADxe4TDS4wL4u0FT_PGlahDa-ENYdYCQ,9512
988
+ acryl_datahub-0.15.0.1rc12.dist-info/top_level.txt,sha256=iLjSrLK5ox1YVYcglRUkcvfZPvKlobBWx7CTUXx8_GI,25
989
+ acryl_datahub-0.15.0.1rc12.dist-info/RECORD,,
datahub/__init__.py CHANGED
@@ -3,7 +3,7 @@ import warnings
3
3
 
4
4
  # Published at https://pypi.org/project/acryl-datahub/.
5
5
  __package_name__ = "acryl-datahub"
6
- __version__ = "0.15.0.1rc10"
6
+ __version__ = "0.15.0.1rc12"
7
7
 
8
8
 
9
9
  def is_dev_mode() -> bool:
@@ -1,6 +1,6 @@
1
1
  import logging
2
2
  from dataclasses import dataclass
3
- from datetime import datetime, timedelta
3
+ from datetime import datetime, timedelta, timezone
4
4
  from typing import Any, Dict, List, Optional
5
5
 
6
6
  from pydantic import Field
@@ -10,6 +10,7 @@ from datahub.api.circuit_breaker.circuit_breaker import (
10
10
  CircuitBreakerConfig,
11
11
  )
12
12
  from datahub.api.graphql import Assertion, Operation
13
+ from datahub.emitter.mce_builder import parse_ts_millis
13
14
 
14
15
  logger: logging.Logger = logging.getLogger(__name__)
15
16
 
@@ -49,7 +50,7 @@ class AssertionCircuitBreaker(AbstractCircuitBreaker):
49
50
  if not operations:
50
51
  return None
51
52
  else:
52
- return datetime.fromtimestamp(operations[0]["lastUpdatedTimestamp"] / 1000)
53
+ return parse_ts_millis(operations[0]["lastUpdatedTimestamp"])
53
54
 
54
55
  def _check_if_assertion_failed(
55
56
  self, assertions: List[Dict[str, Any]], last_updated: Optional[datetime] = None
@@ -93,7 +94,7 @@ class AssertionCircuitBreaker(AbstractCircuitBreaker):
93
94
  logger.info(f"Found successful assertion: {assertion_urn}")
94
95
  result = False
95
96
  if last_updated is not None:
96
- last_run = datetime.fromtimestamp(last_assertion.time / 1000)
97
+ last_run = parse_ts_millis(last_assertion.time)
97
98
  if last_updated > last_run:
98
99
  logger.error(
99
100
  f"Missing assertion run for {assertion_urn}. The dataset was updated on {last_updated} but the last assertion run was at {last_run}"
@@ -117,7 +118,7 @@ class AssertionCircuitBreaker(AbstractCircuitBreaker):
117
118
  )
118
119
 
119
120
  if not last_updated:
120
- last_updated = datetime.now() - self.config.time_delta
121
+ last_updated = datetime.now(tz=timezone.utc) - self.config.time_delta
121
122
  logger.info(
122
123
  f"Dataset {urn} doesn't have last updated or check_last_assertion_time is false, using calculated min assertion date {last_updated}"
123
124
  )
@@ -10,7 +10,6 @@ from typing import (
10
10
  List,
11
11
  Optional,
12
12
  Type,
13
- TypeVar,
14
13
  Union,
15
14
  runtime_checkable,
16
15
  )
@@ -19,14 +18,12 @@ import pydantic
19
18
  from cached_property import cached_property
20
19
  from pydantic import BaseModel, Extra, ValidationError
21
20
  from pydantic.fields import Field
22
- from typing_extensions import Protocol
21
+ from typing_extensions import Protocol, Self
23
22
 
24
23
  from datahub.configuration._config_enum import ConfigEnum as ConfigEnum # noqa: I250
25
24
  from datahub.configuration.pydantic_migration_helpers import PYDANTIC_VERSION_2
26
25
  from datahub.utilities.dedup_list import deduplicate_list
27
26
 
28
- _ConfigSelf = TypeVar("_ConfigSelf", bound="ConfigModel")
29
-
30
27
  REDACT_KEYS = {
31
28
  "password",
32
29
  "token",
@@ -109,7 +106,7 @@ class ConfigModel(BaseModel):
109
106
  schema_extra = _schema_extra
110
107
 
111
108
  @classmethod
112
- def parse_obj_allow_extras(cls: Type[_ConfigSelf], obj: Any) -> _ConfigSelf:
109
+ def parse_obj_allow_extras(cls, obj: Any) -> Self:
113
110
  if PYDANTIC_VERSION_2:
114
111
  try:
115
112
  with unittest.mock.patch.dict(
@@ -6,7 +6,7 @@ import logging
6
6
  import os
7
7
  import re
8
8
  import time
9
- from datetime import datetime
9
+ from datetime import datetime, timezone
10
10
  from enum import Enum
11
11
  from typing import (
12
12
  TYPE_CHECKING,
@@ -103,6 +103,22 @@ def make_ts_millis(ts: Optional[datetime]) -> Optional[int]:
103
103
  return int(ts.timestamp() * 1000)
104
104
 
105
105
 
106
+ @overload
107
+ def parse_ts_millis(ts: float) -> datetime:
108
+ ...
109
+
110
+
111
+ @overload
112
+ def parse_ts_millis(ts: None) -> None:
113
+ ...
114
+
115
+
116
+ def parse_ts_millis(ts: Optional[float]) -> Optional[datetime]:
117
+ if ts is None:
118
+ return None
119
+ return datetime.fromtimestamp(ts / 1000, tz=timezone.utc)
120
+
121
+
106
122
  def make_data_platform_urn(platform: str) -> str:
107
123
  if platform.startswith("urn:li:dataPlatform:"):
108
124
  return platform
@@ -4,8 +4,8 @@ from pydantic.fields import Field
4
4
  from pydantic.main import BaseModel
5
5
 
6
6
  from datahub.cli.env_utils import get_boolean_env_variable
7
- from datahub.emitter.enum_helpers import get_enum_options
8
7
  from datahub.emitter.mce_builder import (
8
+ ALL_ENV_TYPES,
9
9
  Aspect,
10
10
  datahub_guid,
11
11
  make_container_urn,
@@ -25,7 +25,6 @@ from datahub.metadata.schema_classes import (
25
25
  ContainerClass,
26
26
  DomainsClass,
27
27
  EmbedClass,
28
- FabricTypeClass,
29
28
  GlobalTagsClass,
30
29
  MetadataChangeEventClass,
31
30
  OwnerClass,
@@ -206,11 +205,7 @@ def gen_containers(
206
205
  # Extra validation on the env field.
207
206
  # In certain cases (mainly for backwards compatibility), the env field will actually
208
207
  # have a platform instance name.
209
- env = (
210
- container_key.env
211
- if container_key.env in get_enum_options(FabricTypeClass)
212
- else None
213
- )
208
+ env = container_key.env if container_key.env in ALL_ENV_TYPES else None
214
209
 
215
210
  container_urn = container_key.as_urn()
216
211
 
@@ -2,7 +2,7 @@ import json
2
2
  import time
3
3
  from collections import defaultdict
4
4
  from dataclasses import dataclass
5
- from typing import Any, Dict, Iterable, List, Optional, Sequence, Union
5
+ from typing import Any, Dict, List, Optional, Sequence, Union
6
6
 
7
7
  from datahub.emitter.aspect import JSON_PATCH_CONTENT_TYPE
8
8
  from datahub.emitter.serialization_helper import pre_json_transform
@@ -75,7 +75,7 @@ class MetadataPatchProposal:
75
75
  # TODO: Validate that aspectName is a valid aspect for this entityType
76
76
  self.patches[aspect_name].append(_Patch(op, path, value))
77
77
 
78
- def build(self) -> Iterable[MetadataChangeProposalClass]:
78
+ def build(self) -> List[MetadataChangeProposalClass]:
79
79
  return [
80
80
  MetadataChangeProposalClass(
81
81
  entityUrn=self.urn,
@@ -3,7 +3,7 @@ import json
3
3
  import logging
4
4
  import os
5
5
  from json.decoder import JSONDecodeError
6
- from typing import TYPE_CHECKING, Any, Callable, Dict, List, Optional, Union
6
+ from typing import TYPE_CHECKING, Any, Callable, Dict, List, Optional, Sequence, Union
7
7
 
8
8
  import requests
9
9
  from deprecated import deprecated
@@ -288,7 +288,7 @@ class DataHubRestEmitter(Closeable, Emitter):
288
288
 
289
289
  def emit_mcps(
290
290
  self,
291
- mcps: List[Union[MetadataChangeProposal, MetadataChangeProposalWrapper]],
291
+ mcps: Sequence[Union[MetadataChangeProposal, MetadataChangeProposalWrapper]],
292
292
  async_flag: Optional[bool] = None,
293
293
  ) -> int:
294
294
  logger.debug("Attempting to emit batch mcps")
@@ -1,9 +1,9 @@
1
1
  from abc import abstractmethod
2
2
  from contextlib import AbstractContextManager
3
3
  from types import TracebackType
4
- from typing import Optional, Type, TypeVar
4
+ from typing import Optional, Type
5
5
 
6
- _Self = TypeVar("_Self", bound="Closeable")
6
+ from typing_extensions import Self
7
7
 
8
8
 
9
9
  class Closeable(AbstractContextManager):
@@ -11,7 +11,7 @@ class Closeable(AbstractContextManager):
11
11
  def close(self) -> None:
12
12
  pass
13
13
 
14
- def __enter__(self: _Self) -> _Self:
14
+ def __enter__(self) -> Self:
15
15
  # This method is mainly required for type checking.
16
16
  return self
17
17
 
@@ -1,6 +1,8 @@
1
1
  from abc import abstractmethod
2
2
  from dataclasses import dataclass
3
- from typing import Any, Dict, NewType, Optional, Type, TypeVar
3
+ from typing import Any, Dict, NewType, Optional
4
+
5
+ from typing_extensions import Self
4
6
 
5
7
  import datahub.emitter.mce_builder as builder
6
8
  from datahub.configuration.common import ConfigModel
@@ -17,9 +19,6 @@ class IngestionCheckpointingProviderConfig(ConfigModel):
17
19
  pass
18
20
 
19
21
 
20
- _Self = TypeVar("_Self", bound="IngestionCheckpointingProviderBase")
21
-
22
-
23
22
  @dataclass()
24
23
  class IngestionCheckpointingProviderBase(StatefulCommittable[CheckpointJobStatesMap]):
25
24
  """
@@ -32,9 +31,7 @@ class IngestionCheckpointingProviderBase(StatefulCommittable[CheckpointJobStates
32
31
 
33
32
  @classmethod
34
33
  @abstractmethod
35
- def create(
36
- cls: Type[_Self], config_dict: Dict[str, Any], ctx: PipelineContext
37
- ) -> "_Self":
34
+ def create(cls, config_dict: Dict[str, Any], ctx: PipelineContext) -> Self:
38
35
  pass
39
36
 
40
37
  @abstractmethod
@@ -42,7 +42,10 @@ class Report(SupportsAsObj):
42
42
  return some_val.as_obj()
43
43
  elif isinstance(some_val, pydantic.BaseModel):
44
44
  return Report.to_pure_python_obj(some_val.dict())
45
- elif dataclasses.is_dataclass(some_val):
45
+ elif dataclasses.is_dataclass(some_val) and not isinstance(some_val, type):
46
+ # The `is_dataclass` function returns `True` for both instances and classes.
47
+ # We need an extra check to ensure an instance was passed in.
48
+ # https://docs.python.org/3/library/dataclasses.html#dataclasses.is_dataclass
46
49
  return dataclasses.asdict(some_val)
47
50
  elif isinstance(some_val, list):
48
51
  return [Report.to_pure_python_obj(v) for v in some_val if v is not None]
@@ -3,6 +3,8 @@ from abc import ABCMeta, abstractmethod
3
3
  from dataclasses import dataclass, field
4
4
  from typing import Any, Generic, Optional, Type, TypeVar, cast
5
5
 
6
+ from typing_extensions import Self
7
+
6
8
  from datahub.configuration.common import ConfigModel
7
9
  from datahub.ingestion.api.closeable import Closeable
8
10
  from datahub.ingestion.api.common import PipelineContext, RecordEnvelope, WorkUnit
@@ -79,7 +81,6 @@ class NoopWriteCallback(WriteCallback):
79
81
 
80
82
  SinkReportType = TypeVar("SinkReportType", bound=SinkReport, covariant=True)
81
83
  SinkConfig = TypeVar("SinkConfig", bound=ConfigModel, covariant=True)
82
- Self = TypeVar("Self", bound="Sink")
83
84
 
84
85
 
85
86
  class Sink(Generic[SinkConfig, SinkReportType], Closeable, metaclass=ABCMeta):
@@ -90,7 +91,7 @@ class Sink(Generic[SinkConfig, SinkReportType], Closeable, metaclass=ABCMeta):
90
91
  report: SinkReportType
91
92
 
92
93
  @classmethod
93
- def get_config_class(cls: Type[Self]) -> Type[SinkConfig]:
94
+ def get_config_class(cls) -> Type[SinkConfig]:
94
95
  config_class = get_class_from_annotation(cls, Sink, ConfigModel)
95
96
  assert config_class, "Sink subclasses must define a config class"
96
97
  return cast(Type[SinkConfig], config_class)
@@ -112,7 +113,7 @@ class Sink(Generic[SinkConfig, SinkReportType], Closeable, metaclass=ABCMeta):
112
113
  pass
113
114
 
114
115
  @classmethod
115
- def create(cls: Type[Self], config_dict: dict, ctx: PipelineContext) -> "Self":
116
+ def create(cls, config_dict: dict, ctx: PipelineContext) -> "Self":
116
117
  return cls(ctx, cls.get_config_class().parse_obj(config_dict))
117
118
 
118
119
  def handle_work_unit_start(self, workunit: WorkUnit) -> None:
@@ -1,5 +1,4 @@
1
1
  import logging
2
- from datetime import datetime, timezone
3
2
  from typing import (
4
3
  TYPE_CHECKING,
5
4
  Dict,
@@ -14,7 +13,7 @@ from typing import (
14
13
  )
15
14
 
16
15
  from datahub.configuration.time_window_config import BaseTimeWindowConfig
17
- from datahub.emitter.mce_builder import make_dataplatform_instance_urn
16
+ from datahub.emitter.mce_builder import make_dataplatform_instance_urn, parse_ts_millis
18
17
  from datahub.emitter.mcp import MetadataChangeProposalWrapper
19
18
  from datahub.emitter.mcp_builder import entity_supports_aspect
20
19
  from datahub.ingestion.api.workunit import MetadataWorkUnit
@@ -479,10 +478,7 @@ def auto_empty_dataset_usage_statistics(
479
478
  if invalid_timestamps:
480
479
  logger.warning(
481
480
  f"Usage statistics with unexpected timestamps, bucket_duration={config.bucket_duration}:\n"
482
- ", ".join(
483
- str(datetime.fromtimestamp(ts / 1000, tz=timezone.utc))
484
- for ts in invalid_timestamps
485
- )
481
+ ", ".join(str(parse_ts_millis(ts)) for ts in invalid_timestamps)
486
482
  )
487
483
 
488
484
  for bucket in bucket_timestamps:
@@ -1,7 +1,7 @@
1
1
  import logging
2
2
  from collections import defaultdict
3
3
  from dataclasses import dataclass, field
4
- from datetime import datetime, timezone
4
+ from datetime import datetime
5
5
  from functools import lru_cache
6
6
  from typing import Any, Dict, FrozenSet, Iterable, Iterator, List, Optional
7
7
 
@@ -15,6 +15,7 @@ from google.cloud.bigquery.table import (
15
15
  TimePartitioningType,
16
16
  )
17
17
 
18
+ from datahub.emitter.mce_builder import parse_ts_millis
18
19
  from datahub.ingestion.api.source import SourceReport
19
20
  from datahub.ingestion.source.bigquery_v2.bigquery_audit import BigqueryTableIdentifier
20
21
  from datahub.ingestion.source.bigquery_v2.bigquery_helper import parse_labels
@@ -393,13 +394,7 @@ class BigQuerySchemaApi:
393
394
  name=table.table_name,
394
395
  created=table.created,
395
396
  table_type=table.table_type,
396
- last_altered=(
397
- datetime.fromtimestamp(
398
- table.get("last_altered") / 1000, tz=timezone.utc
399
- )
400
- if table.get("last_altered") is not None
401
- else None
402
- ),
397
+ last_altered=parse_ts_millis(table.get("last_altered")),
403
398
  size_in_bytes=table.get("bytes"),
404
399
  rows_count=table.get("row_count"),
405
400
  comment=table.comment,
@@ -460,11 +455,7 @@ class BigQuerySchemaApi:
460
455
  return BigqueryView(
461
456
  name=view.table_name,
462
457
  created=view.created,
463
- last_altered=(
464
- datetime.fromtimestamp(view.get("last_altered") / 1000, tz=timezone.utc)
465
- if view.get("last_altered") is not None
466
- else None
467
- ),
458
+ last_altered=(parse_ts_millis(view.get("last_altered"))),
468
459
  comment=view.comment,
469
460
  view_definition=view.view_definition,
470
461
  materialized=view.table_type == BigqueryTableType.MATERIALIZED_VIEW,
@@ -705,13 +696,7 @@ class BigQuerySchemaApi:
705
696
  return BigqueryTableSnapshot(
706
697
  name=snapshot.table_name,
707
698
  created=snapshot.created,
708
- last_altered=(
709
- datetime.fromtimestamp(
710
- snapshot.get("last_altered") / 1000, tz=timezone.utc
711
- )
712
- if snapshot.get("last_altered") is not None
713
- else None
714
- ),
699
+ last_altered=parse_ts_millis(snapshot.get("last_altered")),
715
700
  comment=snapshot.comment,
716
701
  ddl=snapshot.ddl,
717
702
  snapshot_time=snapshot.snapshot_time,
@@ -12,6 +12,7 @@ from confluent_kafka.schema_registry import SchemaRegistryClient
12
12
  from confluent_kafka.schema_registry.avro import AvroDeserializer
13
13
 
14
14
  from datahub.configuration.kafka import KafkaConsumerConnectionConfig
15
+ from datahub.emitter.mce_builder import parse_ts_millis
15
16
  from datahub.ingestion.api.closeable import Closeable
16
17
  from datahub.ingestion.api.common import PipelineContext
17
18
  from datahub.ingestion.source.datahub.config import DataHubSourceConfig
@@ -92,7 +93,7 @@ class DataHubKafkaReader(Closeable):
92
93
  if mcl.created and mcl.created.time > stop_time.timestamp() * 1000:
93
94
  logger.info(
94
95
  f"Stopped reading from kafka, reached MCL "
95
- f"with audit stamp {datetime.fromtimestamp(mcl.created.time / 1000)}"
96
+ f"with audit stamp {parse_ts_millis(mcl.created.time)}"
96
97
  )
97
98
  break
98
99