acryl-datahub-cloud 0.3.7.3__py3-none-any.whl → 0.3.7.5__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "acryl-datahub-cloud",
3
- "version": "0.3.7.3",
3
+ "version": "0.3.7.5",
4
4
  "install_requires": [
5
5
  "avro-gen3==0.7.16",
6
6
  "acryl-datahub"
@@ -1,5 +1,6 @@
1
1
  import datetime
2
2
  import logging
3
+ import os
3
4
  import pathlib
4
5
  import tempfile
5
6
  import time
@@ -205,6 +206,12 @@ class DataHubBasedS3Dataset:
205
206
  self.file_writer.close()
206
207
  if self.local_file_path:
207
208
  self.opened_files.append(self.local_file_path)
209
+ if not os.path.exists(self.local_file_path):
210
+ logger.info(f"File not found: {self.local_file_path}")
211
+ return
212
+ if os.path.getsize(self.local_file_path) == 0:
213
+ logger.info(f"File is empty: {self.local_file_path}")
214
+ return
208
215
  if self.config.store_platform == "s3":
209
216
  self.upload_file_to_s3()
210
217
  yield from self._register_dataset()
@@ -137,11 +137,15 @@ class DataHubReportingFormsSource(Source):
137
137
  lambda x: self.report.increment_forms_scanned(),
138
138
  ):
139
139
  dataset.append(row)
140
-
140
+ num_workunits = 0
141
141
  for mcp in dataset.commit():
142
142
  assert mcp.entityUrn, "MCP must have a URN"
143
143
  dataset_urn = mcp.entityUrn
144
144
  yield mcp.as_workunit()
145
+ num_workunits += 1
146
+ if num_workunits == 0:
147
+ logger.info("No form reporting to be done")
148
+ return
145
149
  logger.info(
146
150
  f"Reporting file created at {dataset.get_remote_file_uri(dataset_uri_prefix=dataset_uri_prefix, date=datetime.date.today())}"
147
151
  )
@@ -9,7 +9,7 @@ from datahub.configuration import ConfigModel
9
9
  class ElasticSearchClientConfig(ConfigModel):
10
10
  host: str = os.getenv("ELASTICSEARCH_HOST", "localhost")
11
11
  port: int = int(os.getenv("ELASTICSEARCH_PORT", "9200"))
12
- use_ssl: bool = bool(os.getenv("ELASTICSEARCH_USE_SSL", ""))
12
+ use_ssl: bool = os.getenv("ELASTICSEARCH_USE_SSL", "").lower() == "true"
13
13
  verify_certs: bool = False
14
14
  ca_certs: Optional[str] = None
15
15
  client_cert: Optional[str] = None
@@ -0,0 +1,77 @@
1
+ Metadata-Version: 2.1
2
+ Name: acryl-datahub-cloud
3
+ Version: 0.3.7.5
4
+ Requires-Dist: avro-gen3==0.7.16
5
+ Requires-Dist: acryl-datahub
6
+ Provides-Extra: datahub-lineage-features
7
+ Requires-Dist: pandas; extra == "datahub-lineage-features"
8
+ Requires-Dist: pydantic<2; extra == "datahub-lineage-features"
9
+ Requires-Dist: duckdb; extra == "datahub-lineage-features"
10
+ Requires-Dist: opensearch-py==2.4.2; extra == "datahub-lineage-features"
11
+ Requires-Dist: pyarrow; extra == "datahub-lineage-features"
12
+ Provides-Extra: datahub-reporting-forms
13
+ Requires-Dist: boto3; extra == "datahub-reporting-forms"
14
+ Requires-Dist: pandas; extra == "datahub-reporting-forms"
15
+ Requires-Dist: pydantic<2; extra == "datahub-reporting-forms"
16
+ Requires-Dist: duckdb; extra == "datahub-reporting-forms"
17
+ Requires-Dist: pyarrow; extra == "datahub-reporting-forms"
18
+ Provides-Extra: datahub-reporting-extract-graph
19
+ Requires-Dist: boto3; extra == "datahub-reporting-extract-graph"
20
+ Requires-Dist: pandas; extra == "datahub-reporting-extract-graph"
21
+ Requires-Dist: pydantic<2; extra == "datahub-reporting-extract-graph"
22
+ Requires-Dist: duckdb; extra == "datahub-reporting-extract-graph"
23
+ Requires-Dist: opensearch-py==2.4.2; extra == "datahub-reporting-extract-graph"
24
+ Requires-Dist: pyarrow; extra == "datahub-reporting-extract-graph"
25
+ Provides-Extra: datahub-reporting-extract-sql
26
+ Requires-Dist: boto3; extra == "datahub-reporting-extract-sql"
27
+ Requires-Dist: pandas; extra == "datahub-reporting-extract-sql"
28
+ Requires-Dist: pydantic<2; extra == "datahub-reporting-extract-sql"
29
+ Requires-Dist: duckdb; extra == "datahub-reporting-extract-sql"
30
+ Requires-Dist: pyarrow; extra == "datahub-reporting-extract-sql"
31
+ Provides-Extra: datahub-usage-feature-reporting
32
+ Requires-Dist: boto3; extra == "datahub-usage-feature-reporting"
33
+ Requires-Dist: pandas; extra == "datahub-usage-feature-reporting"
34
+ Requires-Dist: pydantic<2; extra == "datahub-usage-feature-reporting"
35
+ Requires-Dist: elasticsearch==7.13.4; extra == "datahub-usage-feature-reporting"
36
+ Requires-Dist: duckdb; extra == "datahub-usage-feature-reporting"
37
+ Requires-Dist: opensearch-py==2.4.2; extra == "datahub-usage-feature-reporting"
38
+ Requires-Dist: polars; extra == "datahub-usage-feature-reporting"
39
+ Requires-Dist: scipy; extra == "datahub-usage-feature-reporting"
40
+ Requires-Dist: numpy<2; extra == "datahub-usage-feature-reporting"
41
+ Requires-Dist: pyarrow; extra == "datahub-usage-feature-reporting"
42
+ Provides-Extra: acryl-cs-issues
43
+ Requires-Dist: zenpy; extra == "acryl-cs-issues"
44
+ Requires-Dist: slack-sdk; extra == "acryl-cs-issues"
45
+ Requires-Dist: openai; extra == "acryl-cs-issues"
46
+ Requires-Dist: jinja2; extra == "acryl-cs-issues"
47
+ Provides-Extra: all
48
+ Requires-Dist: boto3; extra == "all"
49
+ Requires-Dist: pandas; extra == "all"
50
+ Requires-Dist: pydantic<2; extra == "all"
51
+ Requires-Dist: elasticsearch==7.13.4; extra == "all"
52
+ Requires-Dist: duckdb; extra == "all"
53
+ Requires-Dist: opensearch-py==2.4.2; extra == "all"
54
+ Requires-Dist: zenpy; extra == "all"
55
+ Requires-Dist: openai; extra == "all"
56
+ Requires-Dist: jinja2; extra == "all"
57
+ Requires-Dist: polars; extra == "all"
58
+ Requires-Dist: slack-sdk; extra == "all"
59
+ Requires-Dist: scipy; extra == "all"
60
+ Requires-Dist: numpy<2; extra == "all"
61
+ Requires-Dist: pyarrow; extra == "all"
62
+ Provides-Extra: dev
63
+ Requires-Dist: boto3; extra == "dev"
64
+ Requires-Dist: numpy<2; extra == "dev"
65
+ Requires-Dist: pandas; extra == "dev"
66
+ Requires-Dist: pydantic<2; extra == "dev"
67
+ Requires-Dist: elasticsearch==7.13.4; extra == "dev"
68
+ Requires-Dist: duckdb; extra == "dev"
69
+ Requires-Dist: opensearch-py==2.4.2; extra == "dev"
70
+ Requires-Dist: zenpy; extra == "dev"
71
+ Requires-Dist: openai; extra == "dev"
72
+ Requires-Dist: jinja2; extra == "dev"
73
+ Requires-Dist: polars; extra == "dev"
74
+ Requires-Dist: slack-sdk; extra == "dev"
75
+ Requires-Dist: scipy; extra == "dev"
76
+ Requires-Dist: acryl-datahub[dev]; extra == "dev"
77
+ Requires-Dist: pyarrow; extra == "dev"
@@ -1,5 +1,5 @@
1
1
  acryl_datahub_cloud/__init__.py,sha256=axrMXkn0RW80YmuZgwUP_YQImcv6L28duZLWnW-gaNM,521
2
- acryl_datahub_cloud/_codegen_config.json,sha256=0UvcWYxkmsZR0-ByrIWMFem2cx1vsXwOZYXQd5X_piA,554
2
+ acryl_datahub_cloud/_codegen_config.json,sha256=5lMrHgQmer6O5qrNcIlsk_CKATPbdrwOiBJvZFFHRxI,554
3
3
  acryl_datahub_cloud/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
4
4
  acryl_datahub_cloud/acryl_cs_issues/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
5
5
  acryl_datahub_cloud/acryl_cs_issues/acryl_customer.py,sha256=uFjR2SqGS34y09-S9WqOqNGY8nOq6ptGf4y9781i8Z4,25230
@@ -9,17 +9,17 @@ acryl_datahub_cloud/acryl_cs_issues/acryl_zendesk.py,sha256=eFS08VOzrUf9xgVmUEGx
9
9
  acryl_datahub_cloud/acryl_cs_issues/models.py,sha256=6VvZjA-gInrG71kTzwN7y4lP-QsLcxhknVxNT2-cQEo,13459
10
10
  acryl_datahub_cloud/acryl_cs_issues/source.py,sha256=eYOjkQIhrligWaY_t7GcClg6OeXpI8DSgAXk18P7NZE,16000
11
11
  acryl_datahub_cloud/datahub_reporting/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
12
- acryl_datahub_cloud/datahub_reporting/datahub_dataset.py,sha256=XAf2Y9KedWf55AQ-f6OlhetEVe-kHbC1FZHlQJO9hPg,19002
12
+ acryl_datahub_cloud/datahub_reporting/datahub_dataset.py,sha256=ziaV56W07-gX6aoDu_MgeQL1OB0V4Cij6yoHYwOj-h4,19291
13
13
  acryl_datahub_cloud/datahub_reporting/datahub_form_reporting.py,sha256=LO_6cKW21LF4FlVd6HtKSdzejPaZuBSICRIcik6qPEU,19727
14
14
  acryl_datahub_cloud/datahub_reporting/extract_graph.py,sha256=gnz81aNjDZMRXF1yQmPUBZz1kXwRTwwEOjCEy0W9Ap4,7760
15
15
  acryl_datahub_cloud/datahub_reporting/extract_sql.py,sha256=5CakspY_EyDJCjbDH3P3TNE0JS8DVlhem-11RNHEkiY,9401
16
- acryl_datahub_cloud/datahub_reporting/forms.py,sha256=Yv_dmwOqc89hoQ5M7FuIhMluZgJZ-bpN1IFxFtaY2RE,5778
16
+ acryl_datahub_cloud/datahub_reporting/forms.py,sha256=rOw3aZZ8XRb4JPrfMte8-Xq34snis6bp0gpx7Hf7uw4,5940
17
17
  acryl_datahub_cloud/datahub_reporting/forms_config.py,sha256=3uzFKriUoWBNXLVMaEtnx74fLuujV59kDQwi1HSxFNI,2102
18
18
  acryl_datahub_cloud/datahub_usage_reporting/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
19
19
  acryl_datahub_cloud/datahub_usage_reporting/usage_feature_patch_builder.py,sha256=gR9neaHfi0JMQmAKMlgJCEuZIni7cdPFApGOKa5Pn4Y,14406
20
20
  acryl_datahub_cloud/datahub_usage_reporting/usage_feature_reporter.py,sha256=l2gvqKqoAxb6BDGTsUyx1IFMc7Yef3-dgtTzYRKocUg,65282
21
21
  acryl_datahub_cloud/elasticsearch/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
22
- acryl_datahub_cloud/elasticsearch/config.py,sha256=INUqsdtDyJCk8x8Poq9GndeN6VI4hzx8JMmUVDFUMSU,1208
22
+ acryl_datahub_cloud/elasticsearch/config.py,sha256=6QNBOmoQZu1cJrDIBZyvZgdQt0QLfP82hdQkPtP-4HE,1220
23
23
  acryl_datahub_cloud/elasticsearch/graph_service.py,sha256=K4ykcSMxlrhlDrchhte3vEb1mcw8QkOmdIFSVSX4OVU,2788
24
24
  acryl_datahub_cloud/lineage_features/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
25
25
  acryl_datahub_cloud/lineage_features/source.py,sha256=DS9VgFXtGDRP98dHBE4KVoB2MDFE-4FSS5RlBuYZqRY,5519
@@ -373,8 +373,8 @@ acryl_datahub_cloud/metadata/schemas/UsageFeatures.avsc,sha256=B7mqUWVwduvWSP9zp
373
373
  acryl_datahub_cloud/metadata/schemas/VersionInfo.avsc,sha256=9gMcZ8tjuhgcZiq2gOAp_EOV9q9jvuOgfph6m6v_X7c,1189
374
374
  acryl_datahub_cloud/metadata/schemas/ViewProperties.avsc,sha256=3HhcbH5493dJUnEUtFMYMVfbYQ52aDedm5L4j77Nym4,1032
375
375
  acryl_datahub_cloud/metadata/schemas/__init__.py,sha256=uvLNC3VyCkWA_v8e9FdA1leFf46NFKDD0AajCfihepI,581
376
- acryl_datahub_cloud-0.3.7.3.dist-info/METADATA,sha256=umN36GWVmz5vnNpMY652OBAUyt6CIm91fYcz6FibZig,3931
377
- acryl_datahub_cloud-0.3.7.3.dist-info/WHEEL,sha256=R06PA3UVYHThwHvxuRWMqaGcr-PuniXahwjmQRFMEkY,91
378
- acryl_datahub_cloud-0.3.7.3.dist-info/entry_points.txt,sha256=pnIeD0q0iBd34tcq2N2vtgVOlos5oGp-bQ8bTvfNUd8,879
379
- acryl_datahub_cloud-0.3.7.3.dist-info/top_level.txt,sha256=EwgCxfX-DzJANwxj-Mx_j4TOfAFhmc_FgMbRPzWsoZs,20
380
- acryl_datahub_cloud-0.3.7.3.dist-info/RECORD,,
376
+ acryl_datahub_cloud-0.3.7.5.dist-info/METADATA,sha256=sbNBs8APv2-ljzfNjq-uO-X0ZdWcWVyeJ9eLqV3Sgwc,3847
377
+ acryl_datahub_cloud-0.3.7.5.dist-info/WHEEL,sha256=PZUExdf71Ui_so67QXpySuHtCi3-J3wvF4ORK6k_S8U,91
378
+ acryl_datahub_cloud-0.3.7.5.dist-info/entry_points.txt,sha256=pnIeD0q0iBd34tcq2N2vtgVOlos5oGp-bQ8bTvfNUd8,879
379
+ acryl_datahub_cloud-0.3.7.5.dist-info/top_level.txt,sha256=EwgCxfX-DzJANwxj-Mx_j4TOfAFhmc_FgMbRPzWsoZs,20
380
+ acryl_datahub_cloud-0.3.7.5.dist-info/RECORD,,
@@ -1,5 +1,5 @@
1
1
  Wheel-Version: 1.0
2
- Generator: setuptools (75.5.0)
2
+ Generator: setuptools (75.6.0)
3
3
  Root-Is-Purelib: true
4
4
  Tag: py3-none-any
5
5
 
@@ -1,78 +0,0 @@
1
- Metadata-Version: 2.1
2
- Name: acryl-datahub-cloud
3
- Version: 0.3.7.3
4
- Requires-Dist: avro-gen3 ==0.7.16
5
- Requires-Dist: acryl-datahub
6
- Provides-Extra: acryl-cs-issues
7
- Requires-Dist: jinja2 ; extra == 'acryl-cs-issues'
8
- Requires-Dist: openai ; extra == 'acryl-cs-issues'
9
- Requires-Dist: zenpy ; extra == 'acryl-cs-issues'
10
- Requires-Dist: slack-sdk ; extra == 'acryl-cs-issues'
11
- Provides-Extra: all
12
- Requires-Dist: slack-sdk ; extra == 'all'
13
- Requires-Dist: boto3 ; extra == 'all'
14
- Requires-Dist: jinja2 ; extra == 'all'
15
- Requires-Dist: pydantic <2 ; extra == 'all'
16
- Requires-Dist: scipy ; extra == 'all'
17
- Requires-Dist: pyarrow ; extra == 'all'
18
- Requires-Dist: pandas ; extra == 'all'
19
- Requires-Dist: elasticsearch ==7.13.4 ; extra == 'all'
20
- Requires-Dist: polars ; extra == 'all'
21
- Requires-Dist: duckdb ; extra == 'all'
22
- Requires-Dist: opensearch-py ==2.4.2 ; extra == 'all'
23
- Requires-Dist: numpy <2 ; extra == 'all'
24
- Requires-Dist: openai ; extra == 'all'
25
- Requires-Dist: zenpy ; extra == 'all'
26
- Provides-Extra: datahub-lineage-features
27
- Requires-Dist: duckdb ; extra == 'datahub-lineage-features'
28
- Requires-Dist: opensearch-py ==2.4.2 ; extra == 'datahub-lineage-features'
29
- Requires-Dist: pydantic <2 ; extra == 'datahub-lineage-features'
30
- Requires-Dist: pyarrow ; extra == 'datahub-lineage-features'
31
- Requires-Dist: pandas ; extra == 'datahub-lineage-features'
32
- Provides-Extra: datahub-reporting-extract-graph
33
- Requires-Dist: duckdb ; extra == 'datahub-reporting-extract-graph'
34
- Requires-Dist: opensearch-py ==2.4.2 ; extra == 'datahub-reporting-extract-graph'
35
- Requires-Dist: boto3 ; extra == 'datahub-reporting-extract-graph'
36
- Requires-Dist: pydantic <2 ; extra == 'datahub-reporting-extract-graph'
37
- Requires-Dist: pyarrow ; extra == 'datahub-reporting-extract-graph'
38
- Requires-Dist: pandas ; extra == 'datahub-reporting-extract-graph'
39
- Provides-Extra: datahub-reporting-extract-sql
40
- Requires-Dist: duckdb ; extra == 'datahub-reporting-extract-sql'
41
- Requires-Dist: boto3 ; extra == 'datahub-reporting-extract-sql'
42
- Requires-Dist: pydantic <2 ; extra == 'datahub-reporting-extract-sql'
43
- Requires-Dist: pyarrow ; extra == 'datahub-reporting-extract-sql'
44
- Requires-Dist: pandas ; extra == 'datahub-reporting-extract-sql'
45
- Provides-Extra: datahub-reporting-forms
46
- Requires-Dist: duckdb ; extra == 'datahub-reporting-forms'
47
- Requires-Dist: boto3 ; extra == 'datahub-reporting-forms'
48
- Requires-Dist: pydantic <2 ; extra == 'datahub-reporting-forms'
49
- Requires-Dist: pyarrow ; extra == 'datahub-reporting-forms'
50
- Requires-Dist: pandas ; extra == 'datahub-reporting-forms'
51
- Provides-Extra: datahub-usage-feature-reporting
52
- Requires-Dist: boto3 ; extra == 'datahub-usage-feature-reporting'
53
- Requires-Dist: pydantic <2 ; extra == 'datahub-usage-feature-reporting'
54
- Requires-Dist: scipy ; extra == 'datahub-usage-feature-reporting'
55
- Requires-Dist: pyarrow ; extra == 'datahub-usage-feature-reporting'
56
- Requires-Dist: pandas ; extra == 'datahub-usage-feature-reporting'
57
- Requires-Dist: elasticsearch ==7.13.4 ; extra == 'datahub-usage-feature-reporting'
58
- Requires-Dist: polars ; extra == 'datahub-usage-feature-reporting'
59
- Requires-Dist: duckdb ; extra == 'datahub-usage-feature-reporting'
60
- Requires-Dist: opensearch-py ==2.4.2 ; extra == 'datahub-usage-feature-reporting'
61
- Requires-Dist: numpy <2 ; extra == 'datahub-usage-feature-reporting'
62
- Provides-Extra: dev
63
- Requires-Dist: slack-sdk ; extra == 'dev'
64
- Requires-Dist: boto3 ; extra == 'dev'
65
- Requires-Dist: jinja2 ; extra == 'dev'
66
- Requires-Dist: pydantic <2 ; extra == 'dev'
67
- Requires-Dist: scipy ; extra == 'dev'
68
- Requires-Dist: pyarrow ; extra == 'dev'
69
- Requires-Dist: acryl-datahub[dev] ; extra == 'dev'
70
- Requires-Dist: pandas ; extra == 'dev'
71
- Requires-Dist: elasticsearch ==7.13.4 ; extra == 'dev'
72
- Requires-Dist: polars ; extra == 'dev'
73
- Requires-Dist: duckdb ; extra == 'dev'
74
- Requires-Dist: opensearch-py ==2.4.2 ; extra == 'dev'
75
- Requires-Dist: numpy <2 ; extra == 'dev'
76
- Requires-Dist: openai ; extra == 'dev'
77
- Requires-Dist: zenpy ; extra == 'dev'
78
-