acryl-datahub-cloud 0.3.7.9rc1__py3-none-any.whl → 0.3.8__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.

This version of acryl-datahub-cloud might be problematic. Click here for more details.

Files changed (66) hide show
  1. acryl_datahub_cloud/_codegen_config.json +1 -1
  2. acryl_datahub_cloud/acryl_cs_issues/source.py +0 -1
  3. acryl_datahub_cloud/api/__init__.py +1 -0
  4. acryl_datahub_cloud/api/client.py +6 -0
  5. acryl_datahub_cloud/api/entity_versioning.py +167 -0
  6. acryl_datahub_cloud/datahub_metadata_sharing/__init__.py +0 -0
  7. acryl_datahub_cloud/datahub_metadata_sharing/metadata_sharing_source.py +267 -0
  8. acryl_datahub_cloud/datahub_metadata_sharing/query.py +7 -0
  9. acryl_datahub_cloud/datahub_metadata_sharing/scroll_shared_entities.gql +204 -0
  10. acryl_datahub_cloud/datahub_metadata_sharing/share_entity.gql +9 -0
  11. acryl_datahub_cloud/datahub_reporting/datahub_dataset.py +0 -2
  12. acryl_datahub_cloud/datahub_reporting/datahub_form_reporting.py +0 -1
  13. acryl_datahub_cloud/datahub_reporting/extract_graph.py +0 -1
  14. acryl_datahub_cloud/datahub_reporting/extract_sql.py +0 -1
  15. acryl_datahub_cloud/datahub_usage_reporting/query_builder.py +79 -57
  16. acryl_datahub_cloud/datahub_usage_reporting/usage_feature_reporter.py +284 -258
  17. acryl_datahub_cloud/lineage_features/source.py +22 -5
  18. acryl_datahub_cloud/metadata/_urns/urn_defs.py +1564 -1465
  19. acryl_datahub_cloud/metadata/com/linkedin/pegasus2avro/common/__init__.py +6 -0
  20. acryl_datahub_cloud/metadata/com/linkedin/pegasus2avro/executor/__init__.py +15 -0
  21. acryl_datahub_cloud/metadata/com/linkedin/pegasus2avro/metadata/key/__init__.py +4 -0
  22. acryl_datahub_cloud/metadata/com/linkedin/pegasus2avro/ml/metadata/__init__.py +2 -0
  23. acryl_datahub_cloud/metadata/com/linkedin/pegasus2avro/versionset/__init__.py +17 -0
  24. acryl_datahub_cloud/metadata/schema.avsc +23777 -22729
  25. acryl_datahub_cloud/metadata/schema_classes.py +1322 -519
  26. acryl_datahub_cloud/metadata/schemas/AssertionAnalyticsRunEvent.avsc +1 -1
  27. acryl_datahub_cloud/metadata/schemas/AssertionInferenceDetails.avsc +1 -1
  28. acryl_datahub_cloud/metadata/schemas/AssertionInfo.avsc +1 -1
  29. acryl_datahub_cloud/metadata/schemas/AssertionRunEvent.avsc +1 -1
  30. acryl_datahub_cloud/metadata/schemas/ContainerKey.avsc +1 -0
  31. acryl_datahub_cloud/metadata/schemas/DataFlowKey.avsc +1 -0
  32. acryl_datahub_cloud/metadata/schemas/DataHubIngestionSourceInfo.avsc +6 -0
  33. acryl_datahub_cloud/metadata/schemas/DataHubViewInfo.avsc +2 -0
  34. acryl_datahub_cloud/metadata/schemas/DataJobKey.avsc +3 -1
  35. acryl_datahub_cloud/metadata/schemas/DataProcessInstanceKey.avsc +4 -0
  36. acryl_datahub_cloud/metadata/schemas/DataProcessInstanceOutput.avsc +2 -1
  37. acryl_datahub_cloud/metadata/schemas/DataTransformLogic.avsc +63 -0
  38. acryl_datahub_cloud/metadata/schemas/DatasetKey.avsc +2 -1
  39. acryl_datahub_cloud/metadata/schemas/Deprecation.avsc +12 -0
  40. acryl_datahub_cloud/metadata/schemas/DynamicFormAssignment.avsc +2 -0
  41. acryl_datahub_cloud/metadata/schemas/EntityTypeKey.avsc +1 -0
  42. acryl_datahub_cloud/metadata/schemas/ExecutionRequestInput.avsc +9 -0
  43. acryl_datahub_cloud/metadata/schemas/ExecutionRequestResult.avsc +14 -0
  44. acryl_datahub_cloud/metadata/schemas/Filter.avsc +2 -0
  45. acryl_datahub_cloud/metadata/schemas/MLFeatureProperties.avsc +51 -0
  46. acryl_datahub_cloud/metadata/schemas/MLModelDeploymentProperties.avsc +51 -0
  47. acryl_datahub_cloud/metadata/schemas/MLModelGroupProperties.avsc +155 -0
  48. acryl_datahub_cloud/metadata/schemas/MLModelKey.avsc +2 -1
  49. acryl_datahub_cloud/metadata/schemas/MLModelProperties.avsc +155 -47
  50. acryl_datahub_cloud/metadata/schemas/MLPrimaryKeyProperties.avsc +51 -0
  51. acryl_datahub_cloud/metadata/schemas/MLTrainingRunProperties.avsc +171 -0
  52. acryl_datahub_cloud/metadata/schemas/MetadataChangeEvent.avsc +178 -47
  53. acryl_datahub_cloud/metadata/schemas/MonitorInfo.avsc +10 -1
  54. acryl_datahub_cloud/metadata/schemas/PostInfo.avsc +23 -0
  55. acryl_datahub_cloud/metadata/schemas/RecommendationModule.avsc +2 -0
  56. acryl_datahub_cloud/metadata/schemas/RemoteExecutorKey.avsc +21 -0
  57. acryl_datahub_cloud/metadata/schemas/RemoteExecutorStatus.avsc +80 -0
  58. acryl_datahub_cloud/metadata/schemas/SchemaFieldKey.avsc +2 -1
  59. acryl_datahub_cloud/metadata/schemas/VersionProperties.avsc +216 -0
  60. acryl_datahub_cloud/metadata/schemas/VersionSetKey.avsc +26 -0
  61. acryl_datahub_cloud/metadata/schemas/VersionSetProperties.avsc +49 -0
  62. {acryl_datahub_cloud-0.3.7.9rc1.dist-info → acryl_datahub_cloud-0.3.8.dist-info}/METADATA +57 -47
  63. {acryl_datahub_cloud-0.3.7.9rc1.dist-info → acryl_datahub_cloud-0.3.8.dist-info}/RECORD +66 -49
  64. {acryl_datahub_cloud-0.3.7.9rc1.dist-info → acryl_datahub_cloud-0.3.8.dist-info}/WHEEL +1 -1
  65. {acryl_datahub_cloud-0.3.7.9rc1.dist-info → acryl_datahub_cloud-0.3.8.dist-info}/entry_points.txt +1 -0
  66. {acryl_datahub_cloud-0.3.7.9rc1.dist-info → acryl_datahub_cloud-0.3.8.dist-info}/top_level.txt +0 -0
acryl_datahub_cloud/lineage_features/source.py CHANGED
@@ -83,16 +83,31 @@ class DataHubLineageFeaturesSource(Source):
83
83
  query = {
84
84
  "query": {
85
85
  "bool": {
86
- "filter": [
87
- {"term": {"source.entityType": "schemaField"}},
88
- {"term": {"destination.entityType": "schemaField"}},
86
+ "should": [
87
+ {"term": {"relationshipType": "Consumes"}},
89
88
  {"term": {"relationshipType": "DownstreamOf"}},
89
+ {"term": {"relationshipType": "TrainedBy"}},
90
+ {"term": {"relationshipType": "UsedBy"}},
91
+ {"term": {"relationshipType": "MemberOf"}},
92
+ {"term": {"relationshipType": "DerivedFrom"}},
93
+ {"term": {"relationshipType": "Produces"}},
94
+ {"term": {"relationshipType": "DashboardContainsDashboard"}},
95
+ {
96
+ "bool": {
97
+ "must": [
98
+ {"term": {"relationshipType": "Contains"}},
99
+ {"term": {"source.entityType": "dashboard"}},
100
+ {"term": {"destination.entityType": "chart"}},
101
+ ]
102
+ }
103
+ },
90
104
  ],
91
- }
105
+ },
92
106
  },
93
107
  "sort": [
94
108
  {"source.urn": {"order": "desc"}},
95
109
  {"destination.urn": {"order": "desc"}},
110
+ {"relationshipType": {"order": "desc"}},
96
111
  {"lifecycleOwner": {"order": "desc"}},
97
112
  ],
98
113
  }
@@ -124,7 +139,9 @@ class DataHubLineageFeaturesSource(Source):
124
139
  for urn in set(self.upstream_counts.keys()).union(
125
140
  self.downstream_counts.keys()
126
141
  ):
127
- print(urn, self.upstream_counts[urn], self.downstream_counts[urn])
142
+ logger.debug(
143
+ f"{urn}: {self.upstream_counts[urn]}, {self.downstream_counts[urn]}"
144
+ )
128
145
  yield MetadataChangeProposalWrapper(
129
146
  entityUrn=urn,
130
147
  aspect=LineageFeaturesClass(