acryl-datahub-cloud 0.3.7.2__py3-none-any.whl → 0.3.7.4__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- acryl_datahub_cloud/_codegen_config.json +1 -1
- acryl_datahub_cloud/datahub_reporting/datahub_dataset.py +7 -0
- acryl_datahub_cloud/datahub_reporting/forms.py +5 -1
- acryl_datahub_cloud-0.3.7.4.dist-info/METADATA +77 -0
- {acryl_datahub_cloud-0.3.7.2.dist-info → acryl_datahub_cloud-0.3.7.4.dist-info}/RECORD +8 -8
- {acryl_datahub_cloud-0.3.7.2.dist-info → acryl_datahub_cloud-0.3.7.4.dist-info}/WHEEL +1 -1
- acryl_datahub_cloud-0.3.7.2.dist-info/METADATA +0 -78
- {acryl_datahub_cloud-0.3.7.2.dist-info → acryl_datahub_cloud-0.3.7.4.dist-info}/entry_points.txt +0 -0
- {acryl_datahub_cloud-0.3.7.2.dist-info → acryl_datahub_cloud-0.3.7.4.dist-info}/top_level.txt +0 -0
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import datetime
|
|
2
2
|
import logging
|
|
3
|
+
import os
|
|
3
4
|
import pathlib
|
|
4
5
|
import tempfile
|
|
5
6
|
import time
|
|
@@ -205,6 +206,12 @@ class DataHubBasedS3Dataset:
|
|
|
205
206
|
self.file_writer.close()
|
|
206
207
|
if self.local_file_path:
|
|
207
208
|
self.opened_files.append(self.local_file_path)
|
|
209
|
+
if not os.path.exists(self.local_file_path):
|
|
210
|
+
logger.info(f"File not found: {self.local_file_path}")
|
|
211
|
+
return
|
|
212
|
+
if os.path.getsize(self.local_file_path) == 0:
|
|
213
|
+
logger.info(f"File is empty: {self.local_file_path}")
|
|
214
|
+
return
|
|
208
215
|
if self.config.store_platform == "s3":
|
|
209
216
|
self.upload_file_to_s3()
|
|
210
217
|
yield from self._register_dataset()
|
|
@@ -137,11 +137,15 @@ class DataHubReportingFormsSource(Source):
|
|
|
137
137
|
lambda x: self.report.increment_forms_scanned(),
|
|
138
138
|
):
|
|
139
139
|
dataset.append(row)
|
|
140
|
-
|
|
140
|
+
num_workunits = 0
|
|
141
141
|
for mcp in dataset.commit():
|
|
142
142
|
assert mcp.entityUrn, "MCP must have a URN"
|
|
143
143
|
dataset_urn = mcp.entityUrn
|
|
144
144
|
yield mcp.as_workunit()
|
|
145
|
+
num_workunits += 1
|
|
146
|
+
if num_workunits == 0:
|
|
147
|
+
logger.info("No form reporting to be done")
|
|
148
|
+
return
|
|
145
149
|
logger.info(
|
|
146
150
|
f"Reporting file created at {dataset.get_remote_file_uri(dataset_uri_prefix=dataset_uri_prefix, date=datetime.date.today())}"
|
|
147
151
|
)
|
|
@@ -0,0 +1,77 @@
|
|
|
1
|
+
Metadata-Version: 2.1
|
|
2
|
+
Name: acryl-datahub-cloud
|
|
3
|
+
Version: 0.3.7.4
|
|
4
|
+
Requires-Dist: avro-gen3==0.7.16
|
|
5
|
+
Requires-Dist: acryl-datahub
|
|
6
|
+
Provides-Extra: datahub-lineage-features
|
|
7
|
+
Requires-Dist: pydantic<2; extra == "datahub-lineage-features"
|
|
8
|
+
Requires-Dist: opensearch-py==2.4.2; extra == "datahub-lineage-features"
|
|
9
|
+
Requires-Dist: pyarrow; extra == "datahub-lineage-features"
|
|
10
|
+
Requires-Dist: duckdb; extra == "datahub-lineage-features"
|
|
11
|
+
Requires-Dist: pandas; extra == "datahub-lineage-features"
|
|
12
|
+
Provides-Extra: datahub-reporting-forms
|
|
13
|
+
Requires-Dist: boto3; extra == "datahub-reporting-forms"
|
|
14
|
+
Requires-Dist: pydantic<2; extra == "datahub-reporting-forms"
|
|
15
|
+
Requires-Dist: pyarrow; extra == "datahub-reporting-forms"
|
|
16
|
+
Requires-Dist: duckdb; extra == "datahub-reporting-forms"
|
|
17
|
+
Requires-Dist: pandas; extra == "datahub-reporting-forms"
|
|
18
|
+
Provides-Extra: datahub-reporting-extract-graph
|
|
19
|
+
Requires-Dist: boto3; extra == "datahub-reporting-extract-graph"
|
|
20
|
+
Requires-Dist: pydantic<2; extra == "datahub-reporting-extract-graph"
|
|
21
|
+
Requires-Dist: opensearch-py==2.4.2; extra == "datahub-reporting-extract-graph"
|
|
22
|
+
Requires-Dist: pyarrow; extra == "datahub-reporting-extract-graph"
|
|
23
|
+
Requires-Dist: duckdb; extra == "datahub-reporting-extract-graph"
|
|
24
|
+
Requires-Dist: pandas; extra == "datahub-reporting-extract-graph"
|
|
25
|
+
Provides-Extra: datahub-reporting-extract-sql
|
|
26
|
+
Requires-Dist: boto3; extra == "datahub-reporting-extract-sql"
|
|
27
|
+
Requires-Dist: pydantic<2; extra == "datahub-reporting-extract-sql"
|
|
28
|
+
Requires-Dist: pyarrow; extra == "datahub-reporting-extract-sql"
|
|
29
|
+
Requires-Dist: duckdb; extra == "datahub-reporting-extract-sql"
|
|
30
|
+
Requires-Dist: pandas; extra == "datahub-reporting-extract-sql"
|
|
31
|
+
Provides-Extra: datahub-usage-feature-reporting
|
|
32
|
+
Requires-Dist: boto3; extra == "datahub-usage-feature-reporting"
|
|
33
|
+
Requires-Dist: pydantic<2; extra == "datahub-usage-feature-reporting"
|
|
34
|
+
Requires-Dist: polars; extra == "datahub-usage-feature-reporting"
|
|
35
|
+
Requires-Dist: pyarrow; extra == "datahub-usage-feature-reporting"
|
|
36
|
+
Requires-Dist: elasticsearch==7.13.4; extra == "datahub-usage-feature-reporting"
|
|
37
|
+
Requires-Dist: opensearch-py==2.4.2; extra == "datahub-usage-feature-reporting"
|
|
38
|
+
Requires-Dist: duckdb; extra == "datahub-usage-feature-reporting"
|
|
39
|
+
Requires-Dist: scipy; extra == "datahub-usage-feature-reporting"
|
|
40
|
+
Requires-Dist: numpy<2; extra == "datahub-usage-feature-reporting"
|
|
41
|
+
Requires-Dist: pandas; extra == "datahub-usage-feature-reporting"
|
|
42
|
+
Provides-Extra: acryl-cs-issues
|
|
43
|
+
Requires-Dist: jinja2; extra == "acryl-cs-issues"
|
|
44
|
+
Requires-Dist: zenpy; extra == "acryl-cs-issues"
|
|
45
|
+
Requires-Dist: openai; extra == "acryl-cs-issues"
|
|
46
|
+
Requires-Dist: slack-sdk; extra == "acryl-cs-issues"
|
|
47
|
+
Provides-Extra: all
|
|
48
|
+
Requires-Dist: boto3; extra == "all"
|
|
49
|
+
Requires-Dist: pydantic<2; extra == "all"
|
|
50
|
+
Requires-Dist: polars; extra == "all"
|
|
51
|
+
Requires-Dist: pyarrow; extra == "all"
|
|
52
|
+
Requires-Dist: jinja2; extra == "all"
|
|
53
|
+
Requires-Dist: zenpy; extra == "all"
|
|
54
|
+
Requires-Dist: elasticsearch==7.13.4; extra == "all"
|
|
55
|
+
Requires-Dist: opensearch-py==2.4.2; extra == "all"
|
|
56
|
+
Requires-Dist: openai; extra == "all"
|
|
57
|
+
Requires-Dist: slack-sdk; extra == "all"
|
|
58
|
+
Requires-Dist: duckdb; extra == "all"
|
|
59
|
+
Requires-Dist: scipy; extra == "all"
|
|
60
|
+
Requires-Dist: numpy<2; extra == "all"
|
|
61
|
+
Requires-Dist: pandas; extra == "all"
|
|
62
|
+
Provides-Extra: dev
|
|
63
|
+
Requires-Dist: boto3; extra == "dev"
|
|
64
|
+
Requires-Dist: pydantic<2; extra == "dev"
|
|
65
|
+
Requires-Dist: polars; extra == "dev"
|
|
66
|
+
Requires-Dist: pyarrow; extra == "dev"
|
|
67
|
+
Requires-Dist: jinja2; extra == "dev"
|
|
68
|
+
Requires-Dist: zenpy; extra == "dev"
|
|
69
|
+
Requires-Dist: elasticsearch==7.13.4; extra == "dev"
|
|
70
|
+
Requires-Dist: opensearch-py==2.4.2; extra == "dev"
|
|
71
|
+
Requires-Dist: openai; extra == "dev"
|
|
72
|
+
Requires-Dist: slack-sdk; extra == "dev"
|
|
73
|
+
Requires-Dist: acryl-datahub[dev]; extra == "dev"
|
|
74
|
+
Requires-Dist: duckdb; extra == "dev"
|
|
75
|
+
Requires-Dist: scipy; extra == "dev"
|
|
76
|
+
Requires-Dist: numpy<2; extra == "dev"
|
|
77
|
+
Requires-Dist: pandas; extra == "dev"
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
acryl_datahub_cloud/__init__.py,sha256=axrMXkn0RW80YmuZgwUP_YQImcv6L28duZLWnW-gaNM,521
|
|
2
|
-
acryl_datahub_cloud/_codegen_config.json,sha256=
|
|
2
|
+
acryl_datahub_cloud/_codegen_config.json,sha256=VJplr6_hfiAZiGVFAdf3qC2bitM3vUgVJFklqUgXLU0,554
|
|
3
3
|
acryl_datahub_cloud/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
4
4
|
acryl_datahub_cloud/acryl_cs_issues/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
5
5
|
acryl_datahub_cloud/acryl_cs_issues/acryl_customer.py,sha256=uFjR2SqGS34y09-S9WqOqNGY8nOq6ptGf4y9781i8Z4,25230
|
|
@@ -9,11 +9,11 @@ acryl_datahub_cloud/acryl_cs_issues/acryl_zendesk.py,sha256=eFS08VOzrUf9xgVmUEGx
|
|
|
9
9
|
acryl_datahub_cloud/acryl_cs_issues/models.py,sha256=6VvZjA-gInrG71kTzwN7y4lP-QsLcxhknVxNT2-cQEo,13459
|
|
10
10
|
acryl_datahub_cloud/acryl_cs_issues/source.py,sha256=eYOjkQIhrligWaY_t7GcClg6OeXpI8DSgAXk18P7NZE,16000
|
|
11
11
|
acryl_datahub_cloud/datahub_reporting/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
12
|
-
acryl_datahub_cloud/datahub_reporting/datahub_dataset.py,sha256=
|
|
12
|
+
acryl_datahub_cloud/datahub_reporting/datahub_dataset.py,sha256=ziaV56W07-gX6aoDu_MgeQL1OB0V4Cij6yoHYwOj-h4,19291
|
|
13
13
|
acryl_datahub_cloud/datahub_reporting/datahub_form_reporting.py,sha256=LO_6cKW21LF4FlVd6HtKSdzejPaZuBSICRIcik6qPEU,19727
|
|
14
14
|
acryl_datahub_cloud/datahub_reporting/extract_graph.py,sha256=gnz81aNjDZMRXF1yQmPUBZz1kXwRTwwEOjCEy0W9Ap4,7760
|
|
15
15
|
acryl_datahub_cloud/datahub_reporting/extract_sql.py,sha256=5CakspY_EyDJCjbDH3P3TNE0JS8DVlhem-11RNHEkiY,9401
|
|
16
|
-
acryl_datahub_cloud/datahub_reporting/forms.py,sha256=
|
|
16
|
+
acryl_datahub_cloud/datahub_reporting/forms.py,sha256=rOw3aZZ8XRb4JPrfMte8-Xq34snis6bp0gpx7Hf7uw4,5940
|
|
17
17
|
acryl_datahub_cloud/datahub_reporting/forms_config.py,sha256=3uzFKriUoWBNXLVMaEtnx74fLuujV59kDQwi1HSxFNI,2102
|
|
18
18
|
acryl_datahub_cloud/datahub_usage_reporting/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
19
19
|
acryl_datahub_cloud/datahub_usage_reporting/usage_feature_patch_builder.py,sha256=gR9neaHfi0JMQmAKMlgJCEuZIni7cdPFApGOKa5Pn4Y,14406
|
|
@@ -373,8 +373,8 @@ acryl_datahub_cloud/metadata/schemas/UsageFeatures.avsc,sha256=B7mqUWVwduvWSP9zp
|
|
|
373
373
|
acryl_datahub_cloud/metadata/schemas/VersionInfo.avsc,sha256=9gMcZ8tjuhgcZiq2gOAp_EOV9q9jvuOgfph6m6v_X7c,1189
|
|
374
374
|
acryl_datahub_cloud/metadata/schemas/ViewProperties.avsc,sha256=3HhcbH5493dJUnEUtFMYMVfbYQ52aDedm5L4j77Nym4,1032
|
|
375
375
|
acryl_datahub_cloud/metadata/schemas/__init__.py,sha256=uvLNC3VyCkWA_v8e9FdA1leFf46NFKDD0AajCfihepI,581
|
|
376
|
-
acryl_datahub_cloud-0.3.7.
|
|
377
|
-
acryl_datahub_cloud-0.3.7.
|
|
378
|
-
acryl_datahub_cloud-0.3.7.
|
|
379
|
-
acryl_datahub_cloud-0.3.7.
|
|
380
|
-
acryl_datahub_cloud-0.3.7.
|
|
376
|
+
acryl_datahub_cloud-0.3.7.4.dist-info/METADATA,sha256=GfiTono4CtzJfXQyvWd5cQEe6HZn1N8O7RWD1uXin2c,3847
|
|
377
|
+
acryl_datahub_cloud-0.3.7.4.dist-info/WHEEL,sha256=PZUExdf71Ui_so67QXpySuHtCi3-J3wvF4ORK6k_S8U,91
|
|
378
|
+
acryl_datahub_cloud-0.3.7.4.dist-info/entry_points.txt,sha256=pnIeD0q0iBd34tcq2N2vtgVOlos5oGp-bQ8bTvfNUd8,879
|
|
379
|
+
acryl_datahub_cloud-0.3.7.4.dist-info/top_level.txt,sha256=EwgCxfX-DzJANwxj-Mx_j4TOfAFhmc_FgMbRPzWsoZs,20
|
|
380
|
+
acryl_datahub_cloud-0.3.7.4.dist-info/RECORD,,
|
|
@@ -1,78 +0,0 @@
|
|
|
1
|
-
Metadata-Version: 2.1
|
|
2
|
-
Name: acryl-datahub-cloud
|
|
3
|
-
Version: 0.3.7.2
|
|
4
|
-
Requires-Dist: avro-gen3 ==0.7.16
|
|
5
|
-
Requires-Dist: acryl-datahub
|
|
6
|
-
Provides-Extra: acryl-cs-issues
|
|
7
|
-
Requires-Dist: jinja2 ; extra == 'acryl-cs-issues'
|
|
8
|
-
Requires-Dist: slack-sdk ; extra == 'acryl-cs-issues'
|
|
9
|
-
Requires-Dist: openai ; extra == 'acryl-cs-issues'
|
|
10
|
-
Requires-Dist: zenpy ; extra == 'acryl-cs-issues'
|
|
11
|
-
Provides-Extra: all
|
|
12
|
-
Requires-Dist: pandas ; extra == 'all'
|
|
13
|
-
Requires-Dist: duckdb ; extra == 'all'
|
|
14
|
-
Requires-Dist: opensearch-py ==2.4.2 ; extra == 'all'
|
|
15
|
-
Requires-Dist: zenpy ; extra == 'all'
|
|
16
|
-
Requires-Dist: boto3 ; extra == 'all'
|
|
17
|
-
Requires-Dist: slack-sdk ; extra == 'all'
|
|
18
|
-
Requires-Dist: openai ; extra == 'all'
|
|
19
|
-
Requires-Dist: elasticsearch ==7.13.4 ; extra == 'all'
|
|
20
|
-
Requires-Dist: scipy ; extra == 'all'
|
|
21
|
-
Requires-Dist: pydantic <2 ; extra == 'all'
|
|
22
|
-
Requires-Dist: numpy <2 ; extra == 'all'
|
|
23
|
-
Requires-Dist: jinja2 ; extra == 'all'
|
|
24
|
-
Requires-Dist: pyarrow ; extra == 'all'
|
|
25
|
-
Requires-Dist: polars ; extra == 'all'
|
|
26
|
-
Provides-Extra: datahub-lineage-features
|
|
27
|
-
Requires-Dist: pyarrow ; extra == 'datahub-lineage-features'
|
|
28
|
-
Requires-Dist: duckdb ; extra == 'datahub-lineage-features'
|
|
29
|
-
Requires-Dist: pydantic <2 ; extra == 'datahub-lineage-features'
|
|
30
|
-
Requires-Dist: opensearch-py ==2.4.2 ; extra == 'datahub-lineage-features'
|
|
31
|
-
Requires-Dist: pandas ; extra == 'datahub-lineage-features'
|
|
32
|
-
Provides-Extra: datahub-reporting-extract-graph
|
|
33
|
-
Requires-Dist: pyarrow ; extra == 'datahub-reporting-extract-graph'
|
|
34
|
-
Requires-Dist: duckdb ; extra == 'datahub-reporting-extract-graph'
|
|
35
|
-
Requires-Dist: pydantic <2 ; extra == 'datahub-reporting-extract-graph'
|
|
36
|
-
Requires-Dist: opensearch-py ==2.4.2 ; extra == 'datahub-reporting-extract-graph'
|
|
37
|
-
Requires-Dist: pandas ; extra == 'datahub-reporting-extract-graph'
|
|
38
|
-
Requires-Dist: boto3 ; extra == 'datahub-reporting-extract-graph'
|
|
39
|
-
Provides-Extra: datahub-reporting-extract-sql
|
|
40
|
-
Requires-Dist: pyarrow ; extra == 'datahub-reporting-extract-sql'
|
|
41
|
-
Requires-Dist: duckdb ; extra == 'datahub-reporting-extract-sql'
|
|
42
|
-
Requires-Dist: pydantic <2 ; extra == 'datahub-reporting-extract-sql'
|
|
43
|
-
Requires-Dist: pandas ; extra == 'datahub-reporting-extract-sql'
|
|
44
|
-
Requires-Dist: boto3 ; extra == 'datahub-reporting-extract-sql'
|
|
45
|
-
Provides-Extra: datahub-reporting-forms
|
|
46
|
-
Requires-Dist: pyarrow ; extra == 'datahub-reporting-forms'
|
|
47
|
-
Requires-Dist: duckdb ; extra == 'datahub-reporting-forms'
|
|
48
|
-
Requires-Dist: pydantic <2 ; extra == 'datahub-reporting-forms'
|
|
49
|
-
Requires-Dist: pandas ; extra == 'datahub-reporting-forms'
|
|
50
|
-
Requires-Dist: boto3 ; extra == 'datahub-reporting-forms'
|
|
51
|
-
Provides-Extra: datahub-usage-feature-reporting
|
|
52
|
-
Requires-Dist: pandas ; extra == 'datahub-usage-feature-reporting'
|
|
53
|
-
Requires-Dist: duckdb ; extra == 'datahub-usage-feature-reporting'
|
|
54
|
-
Requires-Dist: opensearch-py ==2.4.2 ; extra == 'datahub-usage-feature-reporting'
|
|
55
|
-
Requires-Dist: boto3 ; extra == 'datahub-usage-feature-reporting'
|
|
56
|
-
Requires-Dist: elasticsearch ==7.13.4 ; extra == 'datahub-usage-feature-reporting'
|
|
57
|
-
Requires-Dist: scipy ; extra == 'datahub-usage-feature-reporting'
|
|
58
|
-
Requires-Dist: pydantic <2 ; extra == 'datahub-usage-feature-reporting'
|
|
59
|
-
Requires-Dist: numpy <2 ; extra == 'datahub-usage-feature-reporting'
|
|
60
|
-
Requires-Dist: pyarrow ; extra == 'datahub-usage-feature-reporting'
|
|
61
|
-
Requires-Dist: polars ; extra == 'datahub-usage-feature-reporting'
|
|
62
|
-
Provides-Extra: dev
|
|
63
|
-
Requires-Dist: acryl-datahub[dev] ; extra == 'dev'
|
|
64
|
-
Requires-Dist: duckdb ; extra == 'dev'
|
|
65
|
-
Requires-Dist: opensearch-py ==2.4.2 ; extra == 'dev'
|
|
66
|
-
Requires-Dist: zenpy ; extra == 'dev'
|
|
67
|
-
Requires-Dist: boto3 ; extra == 'dev'
|
|
68
|
-
Requires-Dist: slack-sdk ; extra == 'dev'
|
|
69
|
-
Requires-Dist: openai ; extra == 'dev'
|
|
70
|
-
Requires-Dist: elasticsearch ==7.13.4 ; extra == 'dev'
|
|
71
|
-
Requires-Dist: pyarrow ; extra == 'dev'
|
|
72
|
-
Requires-Dist: scipy ; extra == 'dev'
|
|
73
|
-
Requires-Dist: pydantic <2 ; extra == 'dev'
|
|
74
|
-
Requires-Dist: numpy <2 ; extra == 'dev'
|
|
75
|
-
Requires-Dist: jinja2 ; extra == 'dev'
|
|
76
|
-
Requires-Dist: pandas ; extra == 'dev'
|
|
77
|
-
Requires-Dist: polars ; extra == 'dev'
|
|
78
|
-
|
{acryl_datahub_cloud-0.3.7.2.dist-info → acryl_datahub_cloud-0.3.7.4.dist-info}/entry_points.txt
RENAMED
|
File without changes
|
{acryl_datahub_cloud-0.3.7.2.dist-info → acryl_datahub_cloud-0.3.7.4.dist-info}/top_level.txt
RENAMED
|
File without changes
|