acryl-datahub 1.3.0.1rc9__py3-none-any.whl → 1.3.1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.

This version of acryl-datahub might be problematic. Click here for more details.

Files changed (74) hide show
  1. {acryl_datahub-1.3.0.1rc9.dist-info → acryl_datahub-1.3.1.dist-info}/METADATA +2468 -2461
  2. {acryl_datahub-1.3.0.1rc9.dist-info → acryl_datahub-1.3.1.dist-info}/RECORD +74 -74
  3. datahub/_version.py +1 -1
  4. datahub/ingestion/graph/client.py +5 -1
  5. datahub/ingestion/run/pipeline.py +1 -0
  6. datahub/ingestion/source/dremio/dremio_api.py +212 -78
  7. datahub/ingestion/source/dremio/dremio_entities.py +55 -39
  8. datahub/ingestion/source/dremio/dremio_profiling.py +14 -3
  9. datahub/ingestion/source/dremio/dremio_source.py +9 -11
  10. datahub/ingestion/source/elastic_search.py +106 -29
  11. datahub/ingestion/source/snowflake/snowflake_queries.py +27 -3
  12. datahub/ingestion/source/sql_queries.py +164 -15
  13. datahub/metadata/_internal_schema_classes.py +62 -2
  14. datahub/metadata/com/linkedin/pegasus2avro/assertion/__init__.py +2 -0
  15. datahub/metadata/schema.avsc +264 -89
  16. datahub/metadata/schemas/ApplicationProperties.avsc +5 -2
  17. datahub/metadata/schemas/AssertionInfo.avsc +48 -5
  18. datahub/metadata/schemas/BusinessAttributeInfo.avsc +8 -4
  19. datahub/metadata/schemas/ChartInfo.avsc +12 -5
  20. datahub/metadata/schemas/ContainerProperties.avsc +12 -5
  21. datahub/metadata/schemas/CorpGroupEditableInfo.avsc +2 -1
  22. datahub/metadata/schemas/CorpGroupInfo.avsc +7 -3
  23. datahub/metadata/schemas/CorpUserInfo.avsc +5 -2
  24. datahub/metadata/schemas/CorpUserSettings.avsc +4 -2
  25. datahub/metadata/schemas/DashboardInfo.avsc +16 -4
  26. datahub/metadata/schemas/DataFlowInfo.avsc +11 -5
  27. datahub/metadata/schemas/DataHubPageModuleProperties.avsc +4 -2
  28. datahub/metadata/schemas/DataJobInfo.avsc +9 -4
  29. datahub/metadata/schemas/DataPlatformInfo.avsc +3 -1
  30. datahub/metadata/schemas/DataPlatformInstanceProperties.avsc +5 -2
  31. datahub/metadata/schemas/DataProductProperties.avsc +5 -2
  32. datahub/metadata/schemas/DataTypeInfo.avsc +5 -0
  33. datahub/metadata/schemas/DatasetKey.avsc +2 -1
  34. datahub/metadata/schemas/DatasetProperties.avsc +12 -5
  35. datahub/metadata/schemas/DomainProperties.avsc +7 -3
  36. datahub/metadata/schemas/EditableContainerProperties.avsc +2 -1
  37. datahub/metadata/schemas/EditableDashboardProperties.avsc +2 -1
  38. datahub/metadata/schemas/EditableDataFlowProperties.avsc +2 -1
  39. datahub/metadata/schemas/EditableDataJobProperties.avsc +2 -1
  40. datahub/metadata/schemas/EditableDatasetProperties.avsc +2 -1
  41. datahub/metadata/schemas/EditableERModelRelationshipProperties.avsc +2 -1
  42. datahub/metadata/schemas/EditableMLFeatureProperties.avsc +2 -1
  43. datahub/metadata/schemas/EditableMLFeatureTableProperties.avsc +2 -1
  44. datahub/metadata/schemas/EditableMLModelGroupProperties.avsc +2 -1
  45. datahub/metadata/schemas/EditableMLModelProperties.avsc +2 -1
  46. datahub/metadata/schemas/EditableNotebookProperties.avsc +2 -1
  47. datahub/metadata/schemas/EditableSchemaMetadata.avsc +5 -3
  48. datahub/metadata/schemas/EntityTypeInfo.avsc +5 -0
  49. datahub/metadata/schemas/GlobalTags.avsc +3 -2
  50. datahub/metadata/schemas/GlossaryNodeInfo.avsc +3 -1
  51. datahub/metadata/schemas/GlossaryTermInfo.avsc +3 -1
  52. datahub/metadata/schemas/InputFields.avsc +3 -2
  53. datahub/metadata/schemas/MLFeatureKey.avsc +3 -1
  54. datahub/metadata/schemas/MLFeatureTableKey.avsc +3 -1
  55. datahub/metadata/schemas/MLModelDeploymentKey.avsc +3 -1
  56. datahub/metadata/schemas/MLModelGroupKey.avsc +3 -1
  57. datahub/metadata/schemas/MLModelKey.avsc +3 -1
  58. datahub/metadata/schemas/MLModelProperties.avsc +4 -2
  59. datahub/metadata/schemas/MLPrimaryKeyKey.avsc +3 -1
  60. datahub/metadata/schemas/MetadataChangeEvent.avsc +124 -50
  61. datahub/metadata/schemas/NotebookInfo.avsc +5 -2
  62. datahub/metadata/schemas/Ownership.avsc +3 -2
  63. datahub/metadata/schemas/RoleProperties.avsc +3 -1
  64. datahub/metadata/schemas/SchemaFieldInfo.avsc +3 -1
  65. datahub/metadata/schemas/SchemaMetadata.avsc +3 -2
  66. datahub/metadata/schemas/StructuredPropertyDefinition.avsc +9 -3
  67. datahub/metadata/schemas/TagProperties.avsc +3 -1
  68. datahub/metadata/schemas/TestInfo.avsc +2 -1
  69. datahub/sql_parsing/schema_resolver.py +29 -0
  70. datahub/sql_parsing/sql_parsing_aggregator.py +15 -0
  71. {acryl_datahub-1.3.0.1rc9.dist-info → acryl_datahub-1.3.1.dist-info}/WHEEL +0 -0
  72. {acryl_datahub-1.3.0.1rc9.dist-info → acryl_datahub-1.3.1.dist-info}/entry_points.txt +0 -0
  73. {acryl_datahub-1.3.0.1rc9.dist-info → acryl_datahub-1.3.1.dist-info}/licenses/LICENSE +0 -0
  74. {acryl_datahub-1.3.0.1rc9.dist-info → acryl_datahub-1.3.1.dist-info}/top_level.txt +0 -0
datahub/metadata/schemas/EditableDashboardProperties.avsc CHANGED
@@ -83,7 +83,8 @@
83
83
  {
84
84
  "Searchable": {
85
85
  "fieldName": "editedDescription",
86
- "fieldType": "TEXT"
86
+ "fieldType": "TEXT",
87
+ "searchTier": 2
87
88
  },
88
89
  "type": [
89
90
  "null",
datahub/metadata/schemas/EditableDataFlowProperties.avsc CHANGED
@@ -83,7 +83,8 @@
83
83
  {
84
84
  "Searchable": {
85
85
  "fieldName": "editedDescription",
86
- "fieldType": "TEXT"
86
+ "fieldType": "TEXT",
87
+ "searchTier": 2
87
88
  },
88
89
  "type": [
89
90
  "null",
datahub/metadata/schemas/EditableDataJobProperties.avsc CHANGED
@@ -83,7 +83,8 @@
83
83
  {
84
84
  "Searchable": {
85
85
  "fieldName": "editedDescription",
86
- "fieldType": "TEXT"
86
+ "fieldType": "TEXT",
87
+ "searchTier": 2
87
88
  },
88
89
  "type": [
89
90
  "null",
datahub/metadata/schemas/EditableDatasetProperties.avsc CHANGED
@@ -83,7 +83,8 @@
83
83
  {
84
84
  "Searchable": {
85
85
  "fieldName": "editedDescription",
86
- "fieldType": "TEXT"
86
+ "fieldType": "TEXT",
87
+ "searchTier": 2
87
88
  },
88
89
  "type": [
89
90
  "null",
datahub/metadata/schemas/EditableERModelRelationshipProperties.avsc CHANGED
@@ -83,7 +83,8 @@
83
83
  {
84
84
  "Searchable": {
85
85
  "fieldName": "editedDescription",
86
- "fieldType": "TEXT"
86
+ "fieldType": "TEXT",
87
+ "searchTier": 2
87
88
  },
88
89
  "type": [
89
90
  "null",
datahub/metadata/schemas/EditableMLFeatureProperties.avsc CHANGED
@@ -9,7 +9,8 @@
9
9
  {
10
10
  "Searchable": {
11
11
  "fieldName": "editedDescription",
12
- "fieldType": "TEXT"
12
+ "fieldType": "TEXT",
13
+ "searchTier": 2
13
14
  },
14
15
  "type": [
15
16
  "null",
datahub/metadata/schemas/EditableMLFeatureTableProperties.avsc CHANGED
@@ -9,7 +9,8 @@
9
9
  {
10
10
  "Searchable": {
11
11
  "fieldName": "editedDescription",
12
- "fieldType": "TEXT"
12
+ "fieldType": "TEXT",
13
+ "searchTier": 2
13
14
  },
14
15
  "type": [
15
16
  "null",
datahub/metadata/schemas/EditableMLModelGroupProperties.avsc CHANGED
@@ -9,7 +9,8 @@
9
9
  {
10
10
  "Searchable": {
11
11
  "fieldName": "editedDescription",
12
- "fieldType": "TEXT"
12
+ "fieldType": "TEXT",
13
+ "searchTier": 2
13
14
  },
14
15
  "type": [
15
16
  "null",
datahub/metadata/schemas/EditableMLModelProperties.avsc CHANGED
@@ -9,7 +9,8 @@
9
9
  {
10
10
  "Searchable": {
11
11
  "fieldName": "editedDescription",
12
- "fieldType": "TEXT"
12
+ "fieldType": "TEXT",
13
+ "searchTier": 2
13
14
  },
14
15
  "type": [
15
16
  "null",
datahub/metadata/schemas/EditableNotebookProperties.avsc CHANGED
@@ -83,7 +83,8 @@
83
83
  {
84
84
  "Searchable": {
85
85
  "fieldName": "editedDescription",
86
- "fieldType": "TEXT"
86
+ "fieldType": "TEXT",
87
+ "searchTier": 2
87
88
  },
88
89
  "type": [
89
90
  "null",
datahub/metadata/schemas/EditableSchemaMetadata.avsc CHANGED
@@ -97,7 +97,8 @@
97
97
  "Searchable": {
98
98
  "boostScore": 0.1,
99
99
  "fieldName": "editedFieldDescriptions",
100
- "fieldType": "TEXT"
100
+ "fieldType": "TEXT",
101
+ "searchTier": 2
101
102
  },
102
103
  "type": [
103
104
  "null",
@@ -162,9 +163,10 @@
162
163
  "boostScore": 0.5,
163
164
  "fieldName": "tags",
164
165
  "fieldType": "URN",
165
- "filterNameOverride": "Tag",
166
+ "filterNameOverride": "Tagged With",
166
167
  "hasValuesFieldName": "hasTags",
167
- "queryByDefault": true
168
+ "queryByDefault": true,
169
+ "searchTier": 2
168
170
  }
169
171
  },
170
172
  "type": {
datahub/metadata/schemas/EntityTypeInfo.avsc CHANGED
@@ -7,6 +7,11 @@
7
7
  "namespace": "com.linkedin.pegasus2avro.entitytype",
8
8
  "fields": [
9
9
  {
10
+ "Searchable": {
11
+ "fieldType": "KEYWORD",
12
+ "searchLabel": "qualifiedName",
13
+ "searchTier": 1
14
+ },
10
15
  "type": "string",
11
16
  "name": "qualifiedName",
12
17
  "doc": "The fully qualified name for the entity type, which usually consists of a namespace\nplus an identifier or name, e.g. datahub.dataset"
datahub/metadata/schemas/GlobalTags.avsc CHANGED
@@ -21,9 +21,10 @@
21
21
  "boostScore": 0.5,
22
22
  "fieldName": "tags",
23
23
  "fieldType": "URN",
24
- "filterNameOverride": "Tag",
24
+ "filterNameOverride": "Tagged With",
25
25
  "hasValuesFieldName": "hasTags",
26
- "queryByDefault": true
26
+ "queryByDefault": true,
27
+ "searchTier": 2
27
28
  }
28
29
  },
29
30
  "type": {
datahub/metadata/schemas/GlossaryNodeInfo.avsc CHANGED
@@ -62,7 +62,9 @@
62
62
  "fieldNameAliases": [
63
63
  "_entityName"
64
64
  ],
65
- "fieldType": "WORD_GRAM"
65
+ "fieldType": "WORD_GRAM",
66
+ "searchLabel": "entityName",
67
+ "searchTier": 1
66
68
  },
67
69
  "type": [
68
70
  "null",
datahub/metadata/schemas/GlossaryTermInfo.avsc CHANGED
@@ -40,7 +40,9 @@
40
40
  "fieldNameAliases": [
41
41
  "_entityName"
42
42
  ],
43
- "fieldType": "WORD_GRAM"
43
+ "fieldType": "WORD_GRAM",
44
+ "searchLabel": "entityName",
45
+ "searchTier": 1
44
46
  },
45
47
  "type": [
46
48
  "null",
datahub/metadata/schemas/InputFields.avsc CHANGED
@@ -378,9 +378,10 @@
378
378
  "boostScore": 0.5,
379
379
  "fieldName": "tags",
380
380
  "fieldType": "URN",
381
- "filterNameOverride": "Tag",
381
+ "filterNameOverride": "Tagged With",
382
382
  "hasValuesFieldName": "hasTags",
383
- "queryByDefault": true
383
+ "queryByDefault": true,
384
+ "searchTier": 2
384
385
  }
385
386
  },
386
387
  "type": {
datahub/metadata/schemas/MLFeatureKey.avsc CHANGED
@@ -42,7 +42,9 @@
42
42
  "fieldNameAliases": [
43
43
  "_entityName"
44
44
  ],
45
- "fieldType": "WORD_GRAM"
45
+ "fieldType": "WORD_GRAM",
46
+ "searchLabel": "entityName",
47
+ "searchTier": 1
46
48
  },
47
49
  "type": "string",
48
50
  "name": "name",
datahub/metadata/schemas/MLFeatureTableKey.avsc CHANGED
@@ -52,7 +52,9 @@
52
52
  "fieldNameAliases": [
53
53
  "_entityName"
54
54
  ],
55
- "fieldType": "WORD_GRAM"
55
+ "fieldType": "WORD_GRAM",
56
+ "searchLabel": "entityName",
57
+ "searchTier": 1
56
58
  },
57
59
  "type": "string",
58
60
  "name": "name",
datahub/metadata/schemas/MLModelDeploymentKey.avsc CHANGED
@@ -34,7 +34,9 @@
34
34
  "fieldNameAliases": [
35
35
  "_entityName"
36
36
  ],
37
- "fieldType": "WORD_GRAM"
37
+ "fieldType": "WORD_GRAM",
38
+ "searchLabel": "entityName",
39
+ "searchTier": 1
38
40
  },
39
41
  "type": "string",
40
42
  "name": "name",
datahub/metadata/schemas/MLModelGroupKey.avsc CHANGED
@@ -44,7 +44,9 @@
44
44
  "fieldNameAliases": [
45
45
  "_entityName"
46
46
  ],
47
- "fieldType": "WORD_GRAM"
47
+ "fieldType": "WORD_GRAM",
48
+ "searchLabel": "entityName",
49
+ "searchTier": 1
48
50
  },
49
51
  "type": "string",
50
52
  "name": "name",
datahub/metadata/schemas/MLModelKey.avsc CHANGED
@@ -55,7 +55,9 @@
55
55
  "fieldNameAliases": [
56
56
  "_entityName"
57
57
  ],
58
- "fieldType": "WORD_GRAM"
58
+ "fieldType": "WORD_GRAM",
59
+ "searchLabel": "entityName",
60
+ "searchTier": 1
59
61
  },
60
62
  "type": "string",
61
63
  "name": "name",
datahub/metadata/schemas/MLModelProperties.avsc CHANGED
@@ -87,7 +87,8 @@
87
87
  "boostScore": 10.0,
88
88
  "enableAutocomplete": true,
89
89
  "fieldType": "WORD_GRAM",
90
- "queryByDefault": true
90
+ "queryByDefault": true,
91
+ "searchTier": 1
91
92
  },
92
93
  "type": [
93
94
  "null",
@@ -100,7 +101,8 @@
100
101
  {
101
102
  "Searchable": {
102
103
  "fieldType": "TEXT",
103
- "hasValuesFieldName": "hasDescription"
104
+ "hasValuesFieldName": "hasDescription",
105
+ "searchTier": 2
104
106
  },
105
107
  "type": [
106
108
  "null",
datahub/metadata/schemas/MLPrimaryKeyKey.avsc CHANGED
@@ -40,7 +40,9 @@
40
40
  "fieldNameAliases": [
41
41
  "_entityName"
42
42
  ],
43
- "fieldType": "WORD_GRAM"
43
+ "fieldType": "WORD_GRAM",
44
+ "searchLabel": "entityName",
45
+ "searchTier": 1
44
46
  },
45
47
  "type": "string",
46
48
  "name": "name",