acryl-datahub 1.0.0.2rc4__py3-none-any.whl → 1.0.0.2rc5__py3-none-any.whl

{acryl_datahub-1.0.0.2rc4.dist-info → acryl_datahub-1.0.0.2rc5.dist-info}/RECORD

@@ -1,7 +1,7 @@
-acryl_datahub-1.0.0.2rc4.dist-info/licenses/LICENSE,sha256=9xNHpsD0uYF5ONzXsKDCuHHB-xbiCrSbueWXqrTNsxk,11365
+acryl_datahub-1.0.0.2rc5.dist-info/licenses/LICENSE,sha256=9xNHpsD0uYF5ONzXsKDCuHHB-xbiCrSbueWXqrTNsxk,11365
 datahub/__init__.py,sha256=aq_i5lVREmoLfYIqcx_pEQicO855YlhD19tWc1eZZNI,59
 datahub/__main__.py,sha256=pegIvQ9hzK7IhqVeUi1MeADSZ2QlP-D3K0OQdEg55RU,106
-datahub/_version.py,sha256=tio-Iic7Okp5tRZa3VWguBhJfeAqAu3sfuYo-W1E7Wc,323
+datahub/_version.py,sha256=TEFaI0SngUMeKPXQwQz9bnZDzmSywu7Y6e6m6k--k00,323
 datahub/entrypoints.py,sha256=2TYgHhs3sCxJlojIHjqfxzt3_ImPwPzq4vBtsUuMqu4,8885
 datahub/errors.py,sha256=BzKdcmYseHOt36zfjJXc17WNutFhp9Y23cU_L6cIkxc,612
 datahub/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -332,7 +332,7 @@ datahub/ingestion/source/hex/constants.py,sha256=8hUTMWyG5keTNfXoLu_Dh413Hw_mGGJ
 datahub/ingestion/source/hex/hex.py,sha256=PIRl8fPkKtlHV7cqR4H8RKVYdTLgEFXHFzc3QAqJLhE,12733
 datahub/ingestion/source/hex/mapper.py,sha256=N3mTlEcrOmhv9ia1dnHGFgFJD2ddyTtU3H5IUbb-UxU,13344
 datahub/ingestion/source/hex/model.py,sha256=S9bUhfFcjzuio2dBS6HzSyRVPiSJvRvMQ0qyVrjV5-E,1766
-datahub/ingestion/source/hex/query_fetcher.py,sha256=5r065vL7XohcgZ_fj-1h6o8cxrPin37IeYsC99GU6LA,12287
+datahub/ingestion/source/hex/query_fetcher.py,sha256=ZaRrja05mK0VlIvpsFi-8-EBoJr0GSLbUxBUjycibIU,12505
 datahub/ingestion/source/iceberg/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 datahub/ingestion/source/iceberg/iceberg.py,sha256=s69XzCGD5oV_hqTyvzCt5eLKZVEzVIJo_DiAEDk3p6A,34759
 datahub/ingestion/source/iceberg/iceberg_common.py,sha256=VGosqYPmn_j6GETSnDHZ8Ay1BVOedmx2x5LHxw16I3A,12278
@@ -496,7 +496,7 @@ datahub/ingestion/source/sql/mssql/__init__.py,sha256=1agpl8S_uDW40olkhCX_W19dbr
 datahub/ingestion/source/sql/mssql/job_models.py,sha256=nAo3rciu-w2-dXCz6_ekDEbGMEjCMEfh8WvSfXoF2l0,9359
 datahub/ingestion/source/sql/mssql/source.py,sha256=Er0uTAsEOzLg2KVddf22pOdzEAwkOjNdqYCPxeCiHdA,32755
 datahub/ingestion/source/sql/stored_procedures/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-datahub/ingestion/source/sql/stored_procedures/base.py,sha256=X4RXYknK8ZPsMgXQFzo2_CsMcXYhDWLm-fLFmZNkZfc,7888
+datahub/ingestion/source/sql/stored_procedures/base.py,sha256=n0l5OaTuW-m3TRvkxs3TqvgMeWF6BagzW3tjyWUcC1A,8631
 datahub/ingestion/source/sql/stored_procedures/lineage.py,sha256=9kBoq4JLa4-I3TMAYwChXp76Kag8YBUEIAkaQMplIkY,1844
 datahub/ingestion/source/state/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 datahub/ingestion/source/state/checkpoint.py,sha256=-fTUZKkY4nHTFqSWZ0jJkkdIu_tWlOjRNhm4FTr4ul4,8860
@@ -592,12 +592,12 @@ datahub/lite/lite_registry.py,sha256=bpH0kasP-LtwwUFNA2QsOIehfekAYfJtN-AkQLmSWnw
 datahub/lite/lite_server.py,sha256=p9Oa2nNs65mqcssSIVOr7VOzWqfVstz6ZQEdT4f82S0,1949
 datahub/lite/lite_util.py,sha256=G0LQHKkyEb1pc_q183g6hflShclGx7kikgMaOxtVVcs,4545
 datahub/metadata/__init__.py,sha256=AjhXPjI6cnpdcrBRrE5gOWo15vv2TTl2ctU4UAnUN7A,238
-datahub/metadata/_schema_classes.py,sha256=HvQKAu3eTDt3lkQcoLpBqYlPo0jfBONWCd6T9a1Meck,1011170
-datahub/metadata/schema.avsc,sha256=rX4HzpW5NPL28A2vfu6ESo-hSyQDnK7LUnvasa7dQCE,752628
+datahub/metadata/_schema_classes.py,sha256=n8NwTeBKsnQdtNCMG85LDc6mPgDT5JGUbsgg9KcNj18,1012635
+datahub/metadata/schema.avsc,sha256=ckv1TFKtvz0eGTSqL2qijL4eqfTuQYMIj91pO0vbc2M,753114
 datahub/metadata/schema_classes.py,sha256=X5Jl5EaSxyHdXOQv14pJ5WkQALun4MRpJ4q12wVFE18,1299
 datahub/metadata/urns.py,sha256=nfrCTExR-k2P9w272WVtWSN3xW1VUJngPwP3xnvULjU,1217
 datahub/metadata/_urns/__init__.py,sha256=cOF3GHMDgPhmbLKbN02NPpuLGHSu0qNgQyBRv08eqF0,243
-datahub/metadata/_urns/urn_defs.py,sha256=mQ52ozRUt19MyBLNZh1f1ETlafCzCYmEbcKxAjR_8o4,133983
+datahub/metadata/_urns/urn_defs.py,sha256=SCNCcJBVZjQd2LE98oMyAA8Lg30Umc6mExPdzAKBhgs,136344
 datahub/metadata/com/__init__.py,sha256=gsAIuTxzfJdI7a9ybZlgMIHMAYksM1SxGxXjtySgKSc,202
 datahub/metadata/com/linkedin/__init__.py,sha256=gsAIuTxzfJdI7a9ybZlgMIHMAYksM1SxGxXjtySgKSc,202
 datahub/metadata/com/linkedin/events/__init__.py,sha256=s_dR0plZF-rOxxIbE8ojekJqwiHzl2WYR-Z3kW6kKS0,298
@@ -639,7 +639,7 @@ datahub/metadata/com/linkedin/pegasus2avro/identity/__init__.py,sha256=1U583fdMT
 datahub/metadata/com/linkedin/pegasus2avro/incident/__init__.py,sha256=LfB7ytT1uuGl5Y8oeU6ILCpSGsGmUKgCjsHphn7AThg,916
 datahub/metadata/com/linkedin/pegasus2avro/ingestion/__init__.py,sha256=1bfG2naq4iS_pwU4J-BVer_gfL0hDbJbnH0gh1MPNgA,871
 datahub/metadata/com/linkedin/pegasus2avro/metadata/__init__.py,sha256=gsAIuTxzfJdI7a9ybZlgMIHMAYksM1SxGxXjtySgKSc,202
-datahub/metadata/com/linkedin/pegasus2avro/metadata/key/__init__.py,sha256=kooj_lICFtq7GjWmYFtQOGuXYy9u4QomKrhTTVKFnDg,4812
+datahub/metadata/com/linkedin/pegasus2avro/metadata/key/__init__.py,sha256=bvCD3zLtdgAtLT6ZpjeBQ8-QglBL9fGBPW89whWGNwM,4929
 datahub/metadata/com/linkedin/pegasus2avro/metadata/query/__init__.py,sha256=gsAIuTxzfJdI7a9ybZlgMIHMAYksM1SxGxXjtySgKSc,202
 datahub/metadata/com/linkedin/pegasus2avro/metadata/query/filter/__init__.py,sha256=DBP_QtxkFmC5q_kuk4dGjb4uOKbB4xKgqTWXGxmNbBQ,532
 datahub/metadata/com/linkedin/pegasus2avro/metadata/snapshot/__init__.py,sha256=OPboF8SV11wGnjvWQB-rxtB0otMdCsE7Tcy7xkOUgz8,2358
@@ -717,6 +717,7 @@ datahub/metadata/schemas/DataHubConnectionDetails.avsc,sha256=IvZj6OA7HRvy-ZIIn0
 datahub/metadata/schemas/DataHubConnectionKey.avsc,sha256=VwbamVFoEdp6epz1lJm_UShBl6ksBxoA7jAYuPI5u3M,522
 datahub/metadata/schemas/DataHubIngestionSourceInfo.avsc,sha256=4wac7sluRIq-0ZjODE5SmuVKuQeW8ajLJNRpqEBRyio,4601
 datahub/metadata/schemas/DataHubIngestionSourceKey.avsc,sha256=CSmoOx_Eqa1-he5dRaVOUQWIv1l2e2lraEPIixKK-lo,526
+datahub/metadata/schemas/DataHubOpenAPISchemaKey.avsc,sha256=q6ZyMoxInwmrkrXkUgMe-i-WZzAxbjcvJ-EI99SnEp8,599
 datahub/metadata/schemas/DataHubPersonaInfo.avsc,sha256=OUvbTgPQsBtzkDDb9pxHXpQ6A7dkL77ZnCXZ-MLEG14,227
 datahub/metadata/schemas/DataHubPersonaKey.avsc,sha256=ddj-DhXa0_YMdLaGkKLLSklfIeDRvSwPXu8o__YEXUE,448
 datahub/metadata/schemas/DataHubPolicyInfo.avsc,sha256=mK8tvgX4hDYuck2R6EihKSYzo6s8hNvvA_hgTOgMrFc,9397
@@ -752,7 +753,7 @@ datahub/metadata/schemas/DataProcessInstanceRunEvent.avsc,sha256=zwTYULEnpMbqwkL
 datahub/metadata/schemas/DataProcessKey.avsc,sha256=sQzsBZAAIJwwZcDrEU3d4G50qlGwrISC1Tqgb9BLq2Y,2258
 datahub/metadata/schemas/DataProductKey.avsc,sha256=5az_f7g0F0oMwWTI4OyEJn47PKQWNVC6OnMZPs9P4Oo,639
 datahub/metadata/schemas/DataProductProperties.avsc,sha256=nYEK6JgpTprU0iZaqWLZsBGYJLkh6HCi1qCu-wbYhvM,6925
-datahub/metadata/schemas/DataTransformLogic.avsc,sha256=wDng1GK9znVoK0INHGiSCSa-AH5MrDkVdMzz4wOWmrY,2011
+datahub/metadata/schemas/DataTransformLogic.avsc,sha256=nHTH6UzJ2Zz88N2aWa96hawLUR20HP7eSynfPtI1kzg,2111
 datahub/metadata/schemas/DataTypeInfo.avsc,sha256=MCjzal71P8uIXZg161LrU8rZTJocZeizK-YxYA0Det0,704
 datahub/metadata/schemas/DataTypeKey.avsc,sha256=Gs5uc_azwg10e36ZbwDTFQMevr0IfiFvJoEGHRzEilw,546
 datahub/metadata/schemas/DatahubIngestionCheckpoint.avsc,sha256=m2Zyrx3ZWDc5gHuwbmBSRJ3JN4NFkpUhDEKM2Yeuqrw,5681
@@ -797,7 +798,7 @@ datahub/metadata/schemas/ExecutionRequestKey.avsc,sha256=SvjnlTAGYsSnvVE0rZ9-7UP
 datahub/metadata/schemas/ExecutionRequestResult.avsc,sha256=kg3xMNr9kYLPnFsV-iqcGm1sh1muQVGJvxUt15L1yKo,2333
 datahub/metadata/schemas/ExecutionRequestSignal.avsc,sha256=dsIUa6tfVSXqYOgh4cW6_Hzi8RjHuJJoO-mBAuZukpA,2515
 datahub/metadata/schemas/Filter.avsc,sha256=PU-aGkc2-sI3ZXY7ci-Y0A7zp1jux3VW_6c8MJRAokg,5933
-datahub/metadata/schemas/FormInfo.avsc,sha256=FbN34htiCgm3LqKDL3sVsJhMUHIyc5jYpGJtYm7Ysd4,6270
+datahub/metadata/schemas/FormInfo.avsc,sha256=tlNI-m6uWJ46yF6Ls6Q_DOEZM2bQgXTd4bSeH30N9MA,6438
 datahub/metadata/schemas/FormKey.avsc,sha256=1-wE28B8T3WJ3JtexreNtFvP3To3n7U-jvYudCuSM9o,437
 datahub/metadata/schemas/Forms.avsc,sha256=shmkhRoHN2gTaTsqGrGDRoNwe_z-nrFbbLjH9MtVDCs,10955
 datahub/metadata/schemas/GlobalSettingsInfo.avsc,sha256=OVMM6FwhHhufHkezYcVePK0zI2llzFYLVFJhmAiHoiI,10102
@@ -828,7 +829,7 @@ datahub/metadata/schemas/MLFeatureTableProperties.avsc,sha256=BtrqcsxoQXObPZXSGR
 datahub/metadata/schemas/MLHyperParam.avsc,sha256=dE6i5r6LTYMNrQe9yy-jKoP09GOJUf__1bO69ldpydc,833
 datahub/metadata/schemas/MLMetric.avsc,sha256=y8WPVVwjhu3YGtqpFFJYNYK8w778RRL_d2sHG1Dc7uM,804
 datahub/metadata/schemas/MLModelDeploymentKey.avsc,sha256=vt04jFF_ZHSvWhqLoxC8C_KspiRLkvNNIXJI0aKPF1Q,2425
-datahub/metadata/schemas/MLModelDeploymentProperties.avsc,sha256=I3v-uNOeYxO4hooPHOjafWWHuVyeGvG90oma0tzpNFg,5409
+datahub/metadata/schemas/MLModelDeploymentProperties.avsc,sha256=7IlGrMmX8nfgezvaZyrXskCTCRlwvRzGOYUOpFV3r6Y,5480
 datahub/metadata/schemas/MLModelFactorPrompts.avsc,sha256=8kX-P4F4mVLFT980z3MwIautt1_6uA-c_Z87nYNDK-k,2712
 datahub/metadata/schemas/MLModelGroupKey.avsc,sha256=3LoMWejMfCwdoqz3PFinRbY1_Yy4Kypw7pwg3tL42Jg,2497
 datahub/metadata/schemas/MLModelGroupProperties.avsc,sha256=AZ5Pohk3_pCctQ4hcE1UOURQFYHQne0dw_lRUpOu5WY,6924
@@ -837,9 +838,9 @@ datahub/metadata/schemas/MLModelProperties.avsc,sha256=hDCBHxGe-cmCBeU1k0ANuQlKj
 datahub/metadata/schemas/MLPrimaryKeyKey.avsc,sha256=Kq2Q9WxZ6nQ8wR4P6wpPCI-J7FwXQyoa10s6BvXtkm8,1110
 datahub/metadata/schemas/MLPrimaryKeyProperties.avsc,sha256=URIuOpS93RVk8MZVcbZ-dmTwu_cN3KSOKxSR8fm-eTo,6744
 datahub/metadata/schemas/MLTrainingRunProperties.avsc,sha256=WGgj0MuQrGD4UgvyHCJHzTnHja2LlJTOr1gLu8SySj0,4269
-datahub/metadata/schemas/MetadataChangeEvent.avsc,sha256=rM6klxSttK-cS083tR0BaVgWVJskjMmhwFbHuV8VKI8,375579
-datahub/metadata/schemas/MetadataChangeLog.avsc,sha256=mpdodpx25E6M1Gq_7slEcPAm-1Es5xPsoqV60HgO7zg,12167
-datahub/metadata/schemas/MetadataChangeProposal.avsc,sha256=EMfQrYsuHf1p6UvBjoLtfdTHGe-vGNJaCFEHz8hdKU0,9698
+datahub/metadata/schemas/MetadataChangeEvent.avsc,sha256=OiEGLnlNxypkw2pyupuRj88k5Bkl0hLoiBQ3bODXQjc,375776
+datahub/metadata/schemas/MetadataChangeLog.avsc,sha256=Cf5eECeShCA_XHFr2MRhRQpPE61F6Xv-z1jjoBLJLgc,12239
+datahub/metadata/schemas/MetadataChangeProposal.avsc,sha256=tvO5cGIqZAIvUbMon1RAKgSY4E0jvBqT5VmLWAuNGkY,9770
 datahub/metadata/schemas/Metrics.avsc,sha256=O7DJGjOwmHbb1x_Zj7AuM_HaHKjBvkfJKfUsX8icXD4,690
 datahub/metadata/schemas/NativeGroupMembership.avsc,sha256=9mh9tzyj3ErmTIhX7ERRUm78j1QtGwXUl9UuIXPndBg,588
 datahub/metadata/schemas/NotebookContent.avsc,sha256=ck3yDt0qK5Hn3-mWTNLlYnohXCs6kMUUWPXm7o1JEWE,12496
@@ -858,7 +859,7 @@ datahub/metadata/schemas/PostInfo.avsc,sha256=CxL1Z-6jWO-xQ7mei5dwAFe_U7sfkQrkSB
 datahub/metadata/schemas/PostKey.avsc,sha256=MEA-S-RonGrsrpwYEejR6CXRka2DcqP7S7qvO5JgEFU,475
 datahub/metadata/schemas/QuantitativeAnalyses.avsc,sha256=XrGYVoGUh7ZRjAbMORSiqF5Zl4a0IbYDPiQuSJhABZM,960
 datahub/metadata/schemas/QueryKey.avsc,sha256=VI4oIHvAO7f0lN_7V3QVuBfHcPz31c57XtW6IrlDfxc,518
-datahub/metadata/schemas/QueryProperties.avsc,sha256=fiLYT539EM0jVEAngWXC7P9QnnIMoEsAaUp7nClwu0s,5509
+datahub/metadata/schemas/QueryProperties.avsc,sha256=26Q3zzuzJbUCUG7IJ3q_OEdcNbyzloZzDJWPbv3GQAk,5589
 datahub/metadata/schemas/QuerySubjects.avsc,sha256=WbnUGiOBdc1Ypn4P_R5gCfPH-wNHJUIk7YeyJ2NiOZQ,1747
 datahub/metadata/schemas/QueryUsageStatistics.avsc,sha256=z1gfAnXdBoPEeERi5RESjrdBuS6AcIdqdN5JqWOSuNo,6192
 datahub/metadata/schemas/RoleKey.avsc,sha256=Uas5jFViSHXhFqq8D4P6-UXqywOth3coztjQ5wA7wL0,449
@@ -877,6 +878,7 @@ datahub/metadata/schemas/StructuredPropertyDefinition.avsc,sha256=a-6TaOQ4A7LDFL
 datahub/metadata/schemas/StructuredPropertyKey.avsc,sha256=lp7tQBgeriEU1YMQ6a4-6aUGSWDqNl00lLDym97j1yI,618
 datahub/metadata/schemas/StructuredPropertySettings.avsc,sha256=EDNlXfT1TqogfulCanIc-nuYO9ZxRFOGzD9tl3ZJdB8,3732
 datahub/metadata/schemas/SubTypes.avsc,sha256=bhXbzK020zDyQno97Xp05vmoMeZ82IGu2jz7pWDo3RQ,655
+datahub/metadata/schemas/SystemMetadata.avsc,sha256=wDVdpa9LSAlMzHIiWw-fMLHTCrxcJdnDOY_n5CDNTN8,2068
 datahub/metadata/schemas/TagKey.avsc,sha256=BfckMlx-wg_LV1_PFVgItfNBPtCQ8_erGeQM4LzOXmY,640
 datahub/metadata/schemas/TagProperties.avsc,sha256=Qzttxd7BB38JUwwl7tZzIV1Warnh-uQO-Ahw9Sd-vH4,883
 datahub/metadata/schemas/TelemetryClientId.avsc,sha256=GScej0kXFZxoBUcRVrVynzArFSYQpO_dnhY5Po5dlx0,408
@@ -1043,8 +1045,8 @@ datahub_provider/operators/datahub_assertion_operator.py,sha256=uvTQ-jk2F0sbqqxp
 datahub_provider/operators/datahub_assertion_sensor.py,sha256=lCBj_3x1cf5GMNpHdfkpHuyHfVxsm6ff5x2Z5iizcAo,140
 datahub_provider/operators/datahub_operation_operator.py,sha256=aevDp2FzX7FxGlXrR0khoHNbxbhKR2qPEX5e8O2Jyzw,174
 datahub_provider/operators/datahub_operation_sensor.py,sha256=8fcdVBCEPgqy1etTXgLoiHoJrRt_nzFZQMdSzHqSG7M,168
-acryl_datahub-1.0.0.2rc4.dist-info/METADATA,sha256=Alsmazr6nq0qPL29zOJJo5M92W2webjcRMxOD7j2BIY,176853
-acryl_datahub-1.0.0.2rc4.dist-info/WHEEL,sha256=CmyFI0kx5cdEMTLiONQRbGQwjIoR1aIYB7eCAQ4KPJ0,91
-acryl_datahub-1.0.0.2rc4.dist-info/entry_points.txt,sha256=o3mDeJXSKhsy7XLkuogihraiabBdLn9HaizYXPrxmk0,9710
-acryl_datahub-1.0.0.2rc4.dist-info/top_level.txt,sha256=iLjSrLK5ox1YVYcglRUkcvfZPvKlobBWx7CTUXx8_GI,25
-acryl_datahub-1.0.0.2rc4.dist-info/RECORD,,
+acryl_datahub-1.0.0.2rc5.dist-info/METADATA,sha256=urp7GO85YeQHY_-wuzs6YWZ6xzfGkunfiD-r-e7CvfY,176853
+acryl_datahub-1.0.0.2rc5.dist-info/WHEEL,sha256=CmyFI0kx5cdEMTLiONQRbGQwjIoR1aIYB7eCAQ4KPJ0,91
+acryl_datahub-1.0.0.2rc5.dist-info/entry_points.txt,sha256=o3mDeJXSKhsy7XLkuogihraiabBdLn9HaizYXPrxmk0,9710
+acryl_datahub-1.0.0.2rc5.dist-info/top_level.txt,sha256=iLjSrLK5ox1YVYcglRUkcvfZPvKlobBWx7CTUXx8_GI,25
+acryl_datahub-1.0.0.2rc5.dist-info/RECORD,,

datahub/_version.py

@@ -1,6 +1,6 @@
 # Published at https://pypi.org/project/acryl-datahub/.
 __package_name__ = "acryl-datahub"
-__version__ = "1.0.0.2rc4"
+__version__ = "1.0.0.2rc5"
 
 
 def is_dev_mode() -> bool:

datahub/ingestion/source/hex/query_fetcher.py

@@ -18,7 +18,8 @@ from datahub.utilities.time import datetime_to_ts_millis
 logger = logging.getLogger(__name__)
 
 # Pattern to extract both project_id and workspace_name from Hex metadata in SQL comments
-HEX_METADATA_PATTERN = r'-- Hex query metadata: \{.*?"project_id": "([^"]+)".*?"project_url": "https?://[^/]+/([^/]+)/hex/.*?\}'
+# Only match metadata with "context": "SCHEDULED_RUN" to filter out non-scheduled runs
+HEX_METADATA_PATTERN = r'-- Hex query metadata: \{.*?"context": "SCHEDULED_RUN".*?"project_id": "([^"]+)".*?"project_url": "https?://[^/]+/([^/]+)/hex/.*?\}'
 
 
 @dataclass
@@ -39,6 +40,7 @@ class HexQueryFetcherReport(SourceReport):
     fetched_query_objects: int = 0
     filtered_out_queries_missing_metadata: int = 0
     filtered_out_queries_different_workspace: int = 0
+    filtered_out_queries_no_match: int = 0
     filtered_out_queries_no_subjects: int = 0
     total_queries: int = 0
     total_dataset_subjects: int = 0
@@ -210,6 +212,7 @@ class HexQueryFetcher:
         match = re.search(HEX_METADATA_PATTERN, sql_statement)
 
         if not match:
+            self.report.filtered_out_queries_no_match += 1
             return None
 
         try:

datahub/ingestion/source/sql/stored_procedures/base.py

@@ -26,6 +26,7 @@ from datahub.metadata.schema_classes import (
     DataPlatformInstanceClass,
     DataTransformClass,
     DataTransformLogicClass,
+    QueryLanguageClass,
     QueryStatementClass,
     SubTypesClass,
 )
@@ -176,7 +177,17 @@ def _generate_job_workunits(
                     DataTransformClass(
                         queryStatement=QueryStatementClass(
                             value=procedure.procedure_definition,
-                            language=procedure.language,
+                            language=(
+                                QueryLanguageClass.SQL
+                                if procedure.language == "SQL"
+                                # The language field uses a pretty limited enum.
+                                # The "UNKNOWN" enum value is pretty new, so we don't want to
+                                # emit it until it has broader server-side support. As a
+                                # short-term solution, we map all languages to "SQL".
+                                # TODO: Once we've released server 1.1.0, we should change
+                                # this to be "UNKNOWN" for all languages except "SQL".
+                                else QueryLanguageClass.SQL
+                            ),
                         ),
                     )
                 ]

datahub/metadata/_schema_classes.py

@@ -15442,6 +15442,35 @@ class DataHubIngestionSourceKeyClass(_Aspect):
         self._inner_dict['id'] = value
     
     
+class DataHubOpenAPISchemaKeyClass(_Aspect):
+    """Key for a Query"""
+
+
+    ASPECT_NAME = 'dataHubOpenAPISchemaKey'
+    ASPECT_INFO = {'keyForEntity': 'dataHubOpenAPISchema', 'entityCategory': 'internal', 'entityAspects': ['systemMetadata'], 'entityDoc': 'Contains aspects which are used in OpenAPI requests/responses which are not otherwise present in the data model.'}
+    RECORD_SCHEMA = get_schema_type("com.linkedin.pegasus2avro.metadata.key.DataHubOpenAPISchemaKey")
+
+    def __init__(self,
+        id: str,
+    ):
+        super().__init__()
+        
+        self.id = id
+    
+    def _restore_defaults(self) -> None:
+        self.id = str()
+    
+    
+    @property
+    def id(self) -> str:
+        """A unique id for the DataHub OpenAPI schema."""
+        return self._inner_dict.get('id')  # type: ignore
+    
+    @id.setter
+    def id(self, value: str) -> None:
+        self._inner_dict['id'] = value
+    
+    
 class DataHubPersonaKeyClass(_Aspect):
     """Key for a persona type"""
 
@@ -20128,10 +20157,14 @@ class PlatformEventHeaderClass(DictWrapper):
         self._inner_dict['timestampMillis'] = value
     
     
-class SystemMetadataClass(DictWrapper):
+class SystemMetadataClass(_Aspect):
     """Metadata associated with each metadata change that is processed by the system"""
-    
+
+
+    ASPECT_NAME = 'systemMetadata'
+    ASPECT_INFO = {}
     RECORD_SCHEMA = get_schema_type("com.linkedin.pegasus2avro.mxe.SystemMetadata")
+
     def __init__(self,
         lastObserved: Optional[Union[int, None]]=None,
         runId: Optional[Union[str, None]]=None,
@@ -21738,6 +21771,9 @@ class QueryLanguageClass(object):
     SQL = "SQL"
     """A SQL Query"""
     
+    UNKNOWN = "UNKNOWN"
+    """Unknown query language"""
+    
     
     
 class QueryPropertiesClass(_Aspect):
@@ -26135,6 +26171,7 @@ __SCHEMA_TYPES = {
     'com.linkedin.pegasus2avro.metadata.key.DataHubActionKey': DataHubActionKeyClass,
     'com.linkedin.pegasus2avro.metadata.key.DataHubConnectionKey': DataHubConnectionKeyClass,
     'com.linkedin.pegasus2avro.metadata.key.DataHubIngestionSourceKey': DataHubIngestionSourceKeyClass,
+    'com.linkedin.pegasus2avro.metadata.key.DataHubOpenAPISchemaKey': DataHubOpenAPISchemaKeyClass,
     'com.linkedin.pegasus2avro.metadata.key.DataHubPersonaKey': DataHubPersonaKeyClass,
     'com.linkedin.pegasus2avro.metadata.key.DataHubPolicyKey': DataHubPolicyKeyClass,
     'com.linkedin.pegasus2avro.metadata.key.DataHubRetentionKey': DataHubRetentionKeyClass,
@@ -26620,6 +26657,7 @@ __SCHEMA_TYPES = {
     'DataHubActionKey': DataHubActionKeyClass,
     'DataHubConnectionKey': DataHubConnectionKeyClass,
     'DataHubIngestionSourceKey': DataHubIngestionSourceKeyClass,
+    'DataHubOpenAPISchemaKey': DataHubOpenAPISchemaKeyClass,
     'DataHubPersonaKey': DataHubPersonaKeyClass,
     'DataHubPolicyKey': DataHubPolicyKeyClass,
     'DataHubRetentionKey': DataHubRetentionKeyClass,
@@ -26879,6 +26917,7 @@ ASPECT_CLASSES: List[Type[_Aspect]] = [
     ContainerClass,
     ContainerPropertiesClass,
     EditableContainerPropertiesClass,
+    SystemMetadataClass,
     DataHubSecretValueClass,
     DataHubUpgradeRequestClass,
     DataHubUpgradeResultClass,
@@ -26935,6 +26974,7 @@ ASPECT_CLASSES: List[Type[_Aspect]] = [
     MLModelKeyClass,
     NotebookKeyClass,
     RoleKeyClass,
+    DataHubOpenAPISchemaKeyClass,
     GlobalSettingsKeyClass,
     DatasetKeyClass,
     ChartKeyClass,
@@ -27102,6 +27142,7 @@ class AspectBag(TypedDict, total=False):
     container: ContainerClass
     containerProperties: ContainerPropertiesClass
     editableContainerProperties: EditableContainerPropertiesClass
+    systemMetadata: SystemMetadataClass
     dataHubSecretValue: DataHubSecretValueClass
     dataHubUpgradeRequest: DataHubUpgradeRequestClass
     dataHubUpgradeResult: DataHubUpgradeResultClass
@@ -27158,6 +27199,7 @@ class AspectBag(TypedDict, total=False):
     mlModelKey: MLModelKeyClass
     notebookKey: NotebookKeyClass
     roleKey: RoleKeyClass
+    dataHubOpenAPISchemaKey: DataHubOpenAPISchemaKeyClass
     globalSettingsKey: GlobalSettingsKeyClass
     datasetKey: DatasetKeyClass
     chartKey: ChartKeyClass
@@ -27292,6 +27334,7 @@ KEY_ASPECTS: Dict[str, Type[_Aspect]] = {
     'mlModel': MLModelKeyClass,
     'notebook': NotebookKeyClass,
     'role': RoleKeyClass,
+    'dataHubOpenAPISchema': DataHubOpenAPISchemaKeyClass,
     'globalSettings': GlobalSettingsKeyClass,
     'dataset': DatasetKeyClass,
     'chart': ChartKeyClass,
@@ -27352,6 +27395,7 @@ ENTITY_TYPE_NAMES: List[str] = [
     'mlModel',
     'notebook',
     'role',
+    'dataHubOpenAPISchema',
     'globalSettings',
     'dataset',
     'chart',
@@ -27411,6 +27455,7 @@ EntityTypeName = Literal[
     'mlModel',
     'notebook',
     'role',
+    'dataHubOpenAPISchema',
     'globalSettings',
     'dataset',
     'chart',

datahub/metadata/_urns/urn_defs.py

@@ -594,6 +594,62 @@ class RoleUrn(_SpecificUrn):
     def id(self) -> str:
         return self._entity_ids[0]
 
+if TYPE_CHECKING:
+    from datahub.metadata.schema_classes import DataHubOpenAPISchemaKeyClass
+
+class DataHubOpenAPISchemaUrn(_SpecificUrn):
+    ENTITY_TYPE: ClassVar[Literal["dataHubOpenAPISchema"]] = "dataHubOpenAPISchema"
+    _URN_PARTS: ClassVar[int] = 1
+
+    def __init__(self, id: Union["DataHubOpenAPISchemaUrn", str], *, _allow_coercion: bool = True) -> None:
+        if _allow_coercion:
+            # Field coercion logic (if any is required).
+            if isinstance(id, str):
+                if id.startswith('urn:li:'):
+                    try:
+                        id = DataHubOpenAPISchemaUrn.from_string(id)
+                    except InvalidUrnError:
+                        raise InvalidUrnError(f'Expecting a DataHubOpenAPISchemaUrn but got {id}')
+                else:
+                    id = UrnEncoder.encode_string(id)
+
+        # Validation logic.
+        if not id:
+            raise InvalidUrnError("DataHubOpenAPISchemaUrn id cannot be empty")
+        if isinstance(id, DataHubOpenAPISchemaUrn):
+            id = id.id
+        elif isinstance(id, Urn):
+            raise InvalidUrnError(f'Expecting a DataHubOpenAPISchemaUrn but got {id}')
+        if UrnEncoder.contains_reserved_char(id):
+            raise InvalidUrnError(f'DataHubOpenAPISchemaUrn id contains reserved characters')
+
+        super().__init__(self.ENTITY_TYPE, [id])
+
+    @classmethod
+    def _parse_ids(cls, entity_ids: List[str]) -> "DataHubOpenAPISchemaUrn":
+        if len(entity_ids) != cls._URN_PARTS:
+            raise InvalidUrnError(f"DataHubOpenAPISchemaUrn should have {cls._URN_PARTS} parts, got {len(entity_ids)}: {entity_ids}")
+        return cls(id=entity_ids[0], _allow_coercion=False)
+
+    @classmethod
+    def underlying_key_aspect_type(cls) -> Type["DataHubOpenAPISchemaKeyClass"]:
+        from datahub.metadata.schema_classes import DataHubOpenAPISchemaKeyClass
+
+        return DataHubOpenAPISchemaKeyClass
+
+    def to_key_aspect(self) -> "DataHubOpenAPISchemaKeyClass":
+        from datahub.metadata.schema_classes import DataHubOpenAPISchemaKeyClass
+
+        return DataHubOpenAPISchemaKeyClass(id=self.id)
+
+    @classmethod
+    def from_key_aspect(cls, key_aspect: "DataHubOpenAPISchemaKeyClass") -> "DataHubOpenAPISchemaUrn":
+        return cls(id=key_aspect.id)
+
+    @property
+    def id(self) -> str:
+        return self._entity_ids[0]
+
 if TYPE_CHECKING:
     from datahub.metadata.schema_classes import GlobalSettingsKeyClass
 

datahub/metadata/com/linkedin/pegasus2avro/metadata/key/__init__.py

@@ -19,6 +19,7 @@ from ......schema_classes import DataHubAccessTokenKeyClass
 from ......schema_classes import DataHubActionKeyClass
 from ......schema_classes import DataHubConnectionKeyClass
 from ......schema_classes import DataHubIngestionSourceKeyClass
+from ......schema_classes import DataHubOpenAPISchemaKeyClass
 from ......schema_classes import DataHubPersonaKeyClass
 from ......schema_classes import DataHubPolicyKeyClass
 from ......schema_classes import DataHubRetentionKeyClass
@@ -72,6 +73,7 @@ DataHubAccessTokenKey = DataHubAccessTokenKeyClass
 DataHubActionKey = DataHubActionKeyClass
 DataHubConnectionKey = DataHubConnectionKeyClass
 DataHubIngestionSourceKey = DataHubIngestionSourceKeyClass
+DataHubOpenAPISchemaKey = DataHubOpenAPISchemaKeyClass
 DataHubPersonaKey = DataHubPersonaKeyClass
 DataHubPolicyKey = DataHubPolicyKeyClass
 DataHubRetentionKey = DataHubRetentionKeyClass

datahub/metadata/schema.avsc

@@ -2343,12 +2343,14 @@
               "type": {
                 "type": "enum",
                 "symbolDocs": {
-                  "SQL": "A SQL Query"
+                  "SQL": "A SQL Query",
+                  "UNKNOWN": "Unknown query language"
                 },
                 "name": "QueryLanguage",
                 "namespace": "com.linkedin.pegasus2avro.query",
                 "symbols": [
-                  "SQL"
+                  "SQL",
+                  "UNKNOWN"
                 ]
               },
               "name": "language",
@@ -2887,6 +2889,11 @@
             "namespace": "com.linkedin.pegasus2avro.form",
             "fields": [
               {
+                "Searchable": {
+                  "fieldName": "promptId",
+                  "fieldType": "KEYWORD",
+                  "queryByDefault": false
+                },
                 "type": "string",
                 "name": "id",
                 "doc": "The unique id for this prompt. This must be GLOBALLY unique."
@@ -4130,6 +4137,92 @@
     ],
     "doc": "Editable information about an Asset Container as defined on the DataHub Platform"
   },
+  {
+    "type": "record",
+    "Aspect": {
+      "name": "systemMetadata"
+    },
+    "name": "SystemMetadata",
+    "namespace": "com.linkedin.pegasus2avro.mxe",
+    "fields": [
+      {
+        "type": [
+          "long",
+          "null"
+        ],
+        "name": "lastObserved",
+        "default": 0,
+        "doc": "The timestamp the metadata was observed at"
+      },
+      {
+        "type": [
+          "string",
+          "null"
+        ],
+        "name": "runId",
+        "default": "no-run-id-provided",
+        "doc": "The original run id that produced the metadata. Populated in case of batch-ingestion."
+      },
+      {
+        "type": [
+          "string",
+          "null"
+        ],
+        "name": "lastRunId",
+        "default": "no-run-id-provided",
+        "doc": "The last run id that produced the metadata. Populated in case of batch-ingestion."
+      },
+      {
+        "type": [
+          "null",
+          "string"
+        ],
+        "name": "pipelineName",
+        "default": null,
+        "doc": "The ingestion pipeline id that produced the metadata. Populated in case of batch ingestion."
+      },
+      {
+        "type": [
+          "null",
+          "string"
+        ],
+        "name": "registryName",
+        "default": null,
+        "doc": "The model registry name that was used to process this event"
+      },
+      {
+        "type": [
+          "null",
+          "string"
+        ],
+        "name": "registryVersion",
+        "default": null,
+        "doc": "The model registry version that was used to process this event"
+      },
+      {
+        "type": [
+          "null",
+          {
+            "type": "map",
+            "values": "string"
+          }
+        ],
+        "name": "properties",
+        "default": null,
+        "doc": "Additional properties"
+      },
+      {
+        "type": [
+          "null",
+          "string"
+        ],
+        "name": "version",
+        "default": null,
+        "doc": "Aspect version\n   Initial implementation will use the aspect version's number, however stored as\n   a string in the case where a different aspect versioning scheme is later adopted."
+      }
+    ],
+    "doc": "Metadata associated with each metadata change that is processed by the system"
+  },
   {
     "type": "record",
     "name": "PlatformEvent",
@@ -4369,89 +4462,7 @@
       {
         "type": [
           "null",
-          {
-            "type": "record",
-            "name": "SystemMetadata",
-            "namespace": "com.linkedin.pegasus2avro.mxe",
-            "fields": [
-              {
-                "type": [
-                  "long",
-                  "null"
-                ],
-                "name": "lastObserved",
-                "default": 0,
-                "doc": "The timestamp the metadata was observed at"
-              },
-              {
-                "type": [
-                  "string",
-                  "null"
-                ],
-                "name": "runId",
-                "default": "no-run-id-provided",
-                "doc": "The original run id that produced the metadata. Populated in case of batch-ingestion."
-              },
-              {
-                "type": [
-                  "string",
-                  "null"
-                ],
-                "name": "lastRunId",
-                "default": "no-run-id-provided",
-                "doc": "The last run id that produced the metadata. Populated in case of batch-ingestion."
-              },
-              {
-                "type": [
-                  "null",
-                  "string"
-                ],
-                "name": "pipelineName",
-                "default": null,
-                "doc": "The ingestion pipeline id that produced the metadata. Populated in case of batch ingestion."
-              },
-              {
-                "type": [
-                  "null",
-                  "string"
-                ],
-                "name": "registryName",
-                "default": null,
-                "doc": "The model registry name that was used to process this event"
-              },
-              {
-                "type": [
-                  "null",
-                  "string"
-                ],
-                "name": "registryVersion",
-                "default": null,
-                "doc": "The model registry version that was used to process this event"
-              },
-              {
-                "type": [
-                  "null",
-                  {
-                    "type": "map",
-                    "values": "string"
-                  }
-                ],
-                "name": "properties",
-                "default": null,
-                "doc": "Additional properties"
-              },
-              {
-                "type": [
-                  "null",
-                  "string"
-                ],
-                "name": "version",
-                "default": null,
-                "doc": "Aspect version\n   Initial implementation will use the aspect version's number, however stored as\n   a string in the case where a different aspect versioning scheme is later adopted."
-              }
-            ],
-            "doc": "Metadata associated with each metadata change that is processed by the system"
-          }
+          "com.linkedin.pegasus2avro.mxe.SystemMetadata"
         ],
         "name": "systemMetadata",
         "default": null,
@@ -10156,6 +10167,9 @@
                           "doc": "Version of the MLModelDeployment"
                         },
                         {
+                          "Searchable": {
+                            "fieldName": "deploymentStatus"
+                          },
                           "type": [
                             "null",
                             {
@@ -15705,6 +15719,28 @@
     ],
     "doc": "Key for a External AccessManagement"
   },
+  {
+    "type": "record",
+    "Aspect": {
+      "name": "dataHubOpenAPISchemaKey",
+      "keyForEntity": "dataHubOpenAPISchema",
+      "entityCategory": "internal",
+      "entityAspects": [
+        "systemMetadata"
+      ],
+      "entityDoc": "Contains aspects which are used in OpenAPI requests/responses which are not otherwise present in the data model."
+    },
+    "name": "DataHubOpenAPISchemaKey",
+    "namespace": "com.linkedin.pegasus2avro.metadata.key",
+    "fields": [
+      {
+        "type": "string",
+        "name": "id",
+        "doc": "A unique id for the DataHub OpenAPI schema."
+      }
+    ],
+    "doc": "Key for a Query"
+  },
   {
     "type": "record",
     "Aspect": {