acdc_aws_etl_pipeline 0.4.8__py3-none-any.whl → 0.5.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- acdc_aws_etl_pipeline/upload/metadata_submitter.py +4 -4
- {acdc_aws_etl_pipeline-0.4.8.dist-info → acdc_aws_etl_pipeline-0.5.0.dist-info}/METADATA +1 -1
- {acdc_aws_etl_pipeline-0.4.8.dist-info → acdc_aws_etl_pipeline-0.5.0.dist-info}/RECORD +4 -4
- {acdc_aws_etl_pipeline-0.4.8.dist-info → acdc_aws_etl_pipeline-0.5.0.dist-info}/WHEEL +0 -0
|
@@ -1,10 +1,13 @@
|
|
|
1
|
+
import os
|
|
2
|
+
# redefine to use local cache in /tmp
|
|
3
|
+
os.environ['XDG_CACHE_HOME'] = '/tmp/.cache'
|
|
4
|
+
|
|
1
5
|
import json
|
|
2
6
|
import boto3
|
|
3
7
|
from gen3.auth import Gen3Auth
|
|
4
8
|
from gen3.index import Gen3Index
|
|
5
9
|
from gen3.submission import Gen3Submission
|
|
6
10
|
import logging
|
|
7
|
-
import os
|
|
8
11
|
from datetime import datetime
|
|
9
12
|
|
|
10
13
|
logger = logging.getLogger(__name__)
|
|
@@ -418,9 +421,6 @@ def submit_metadata(
|
|
|
418
421
|
Notes:
|
|
419
422
|
Each file is split into size-friendly chunks before submit. Local and S3 files are supported.
|
|
420
423
|
"""
|
|
421
|
-
|
|
422
|
-
# redefine to use local cache in /tmp
|
|
423
|
-
os.environ['XDG_CACHE_HOME'] = '/tmp/.cache'
|
|
424
424
|
|
|
425
425
|
timestamp = datetime.now().strftime("%Y%d%m-%H%M%S")
|
|
426
426
|
log_dir = f"submission_logs/{timestamp}"
|
|
@@ -3,12 +3,12 @@ acdc_aws_etl_pipeline/ingest/ingest.py,sha256=0meo4Sq6o0EPvgfFu0QPMQ5ZEHDiSEMsVG
|
|
|
3
3
|
acdc_aws_etl_pipeline/upload/__init__.py,sha256=kRI1wozjK-b9YXMAPwzWHzm967ZiUAM6g8rRo4ONWtI,67
|
|
4
4
|
acdc_aws_etl_pipeline/upload/gen3datasubmitter.py,sha256=bu5d8IOsKFIA1uvvzaxb7YIKwBZKdP-0QvBt-gZMyUc,8625
|
|
5
5
|
acdc_aws_etl_pipeline/upload/metadata_deleter.py,sha256=T4q9xqSE2Beu3zluvAmKh7wJWcCFGz2AZ9h9ZcASfyA,63
|
|
6
|
-
acdc_aws_etl_pipeline/upload/metadata_submitter.py,sha256=
|
|
6
|
+
acdc_aws_etl_pipeline/upload/metadata_submitter.py,sha256=6brNnwh5rpyzIf13ZGC_srcP_U0GRtne_sWoiM5CMnw,21059
|
|
7
7
|
acdc_aws_etl_pipeline/upload/upload_synthdata_s3.py,sha256=Ge5TQzZkWnJNp-q0Ak-Yhv1h1eWLxg-PlWVHrd1m0B8,5155
|
|
8
8
|
acdc_aws_etl_pipeline/utils/athena_utils.py,sha256=QJlBe-07Hkq-BqmcxBu6ZtAmVfZSHuSY4dijcysgPH8,29560
|
|
9
9
|
acdc_aws_etl_pipeline/utils/dbt_utils.py,sha256=5XRFOwNNIeuW2sQuor3h_OZTuXGg6xv2AUYwj9bMAAM,2054
|
|
10
10
|
acdc_aws_etl_pipeline/utils/release_writer.py,sha256=vsxHJ6l-UWPpzeyEPHurX5iFgeCEQ-9FbySAbPNfTTM,7555
|
|
11
11
|
acdc_aws_etl_pipeline/validate/validate.py,sha256=fTa76YvixCWOGkAIuR7CZ2WryMJcpc2wvSOHLZDEknc,28159
|
|
12
|
-
acdc_aws_etl_pipeline-0.
|
|
13
|
-
acdc_aws_etl_pipeline-0.
|
|
14
|
-
acdc_aws_etl_pipeline-0.
|
|
12
|
+
acdc_aws_etl_pipeline-0.5.0.dist-info/METADATA,sha256=0onEHo9mX7jg7kr7RuUcF_cbiWwSV96h0xZgiJygwK4,2853
|
|
13
|
+
acdc_aws_etl_pipeline-0.5.0.dist-info/WHEEL,sha256=zp0Cn7JsFoX2ATtOhtaFYIiE2rmFAD4OcMhtUki8W3U,88
|
|
14
|
+
acdc_aws_etl_pipeline-0.5.0.dist-info/RECORD,,
|
|
File without changes
|