acdc_aws_etl_pipeline 0.4.0__py3-none-any.whl → 0.4.2__py3-none-any.whl

acdc_aws_etl_pipeline/upload/metadata_submitter.py

@@ -365,7 +365,7 @@ def create_gen3_submission_class(api_key: dict, api_endpoint: str):
     submit = None
 
     try:
-        with tempfile.NamedTemporaryFile(mode='w', delete=False, suffix=".json") as tmp_file:
+        with tempfile.NamedTemporaryFile(mode='w', delete=False, suffix=".json", dir="/tmp") as tmp_file:
             json.dump(api_key, tmp_file)
             tmp_api_key_path = tmp_file.name
         auth = Gen3Auth(refresh_file=tmp_api_key_path)
@@ -393,6 +393,7 @@ def submit_metadata(
     max_size_kb: int = 50,
     exclude_nodes: list = None,
     max_retries: int = 5,
+    write_submission_results_path: str = None
 ):
     """
     Submit a set of metadata JSON files to a Gen3 data commons endpoint, in order.
@@ -490,16 +491,17 @@ def submit_metadata(
                     res = submit.submit_record("program1", project_id, jsn)
                     
                     # writing submission results as log json
-                    log_filename = os.path.join(
-                        log_dir, f"{project_id}_{node}_split{index + 1}_of_{n_json_data}.json"
-                    )
-                    abs_log_filename = os.path.abspath(log_filename)
-                    with open(abs_log_filename, "a") as f:
-                        json.dump(res, f)
-                        f.write("\n")
-                    logger.info(
-                        f"Wrote submission response to log file: {abs_log_filename}"
-                    )
+                    if write_submission_results_path is not None:
+                        log_filename = os.path.join(
+                            log_dir, f"{project_id}_{node}_split{index + 1}_of_{n_json_data}.json"
+                        )
+                        abs_log_filename = os.path.abspath(log_filename)
+                        with open(abs_log_filename, "a") as f:
+                            json.dump(res, f)
+                            f.write("\n")
+                        logger.info(
+                            f"Wrote submission response to log file: {abs_log_filename}"
+                        )
 
                     logger.info(
                         f"\033[92m[SUCCESS]\033[0m | Project: {project_id:<10} | "

{acdc_aws_etl_pipeline-0.4.0.dist-info → acdc_aws_etl_pipeline-0.4.2.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: acdc_aws_etl_pipeline
-Version: 0.4.0
+Version: 0.4.2
 Summary: Tools for ACDC ETL pipeline
 Author: JoshuaHarris391
 Author-email: harjo391@gmail.com

{acdc_aws_etl_pipeline-0.4.0.dist-info → acdc_aws_etl_pipeline-0.4.2.dist-info}/RECORD

@@ -3,12 +3,12 @@ acdc_aws_etl_pipeline/ingest/ingest.py,sha256=0meo4Sq6o0EPvgfFu0QPMQ5ZEHDiSEMsVG
 acdc_aws_etl_pipeline/upload/__init__.py,sha256=kRI1wozjK-b9YXMAPwzWHzm967ZiUAM6g8rRo4ONWtI,67
 acdc_aws_etl_pipeline/upload/gen3datasubmitter.py,sha256=bu5d8IOsKFIA1uvvzaxb7YIKwBZKdP-0QvBt-gZMyUc,8625
 acdc_aws_etl_pipeline/upload/metadata_deleter.py,sha256=T4q9xqSE2Beu3zluvAmKh7wJWcCFGz2AZ9h9ZcASfyA,63
-acdc_aws_etl_pipeline/upload/metadata_submitter.py,sha256=LvYkBZ3UDMDH0Wahv6q8aUhvcYxiZuKs_e--pTJHPpY,20152
+acdc_aws_etl_pipeline/upload/metadata_submitter.py,sha256=9ms1X3rjP4INGfNSr4GsM6FkTOfT3JRF3PxAN6pisFE,20316
 acdc_aws_etl_pipeline/upload/upload_synthdata_s3.py,sha256=Ge5TQzZkWnJNp-q0Ak-Yhv1h1eWLxg-PlWVHrd1m0B8,5155
 acdc_aws_etl_pipeline/utils/athena_utils.py,sha256=QJlBe-07Hkq-BqmcxBu6ZtAmVfZSHuSY4dijcysgPH8,29560
 acdc_aws_etl_pipeline/utils/dbt_utils.py,sha256=5XRFOwNNIeuW2sQuor3h_OZTuXGg6xv2AUYwj9bMAAM,2054
 acdc_aws_etl_pipeline/utils/release_writer.py,sha256=vsxHJ6l-UWPpzeyEPHurX5iFgeCEQ-9FbySAbPNfTTM,7555
 acdc_aws_etl_pipeline/validate/validate.py,sha256=fTa76YvixCWOGkAIuR7CZ2WryMJcpc2wvSOHLZDEknc,28159
-acdc_aws_etl_pipeline-0.4.0.dist-info/METADATA,sha256=7wvJ9ecYnnIH_GJF_xGoD0pCOmXJHb_8EldD8fKfTxA,2853
-acdc_aws_etl_pipeline-0.4.0.dist-info/WHEEL,sha256=zp0Cn7JsFoX2ATtOhtaFYIiE2rmFAD4OcMhtUki8W3U,88
-acdc_aws_etl_pipeline-0.4.0.dist-info/RECORD,,
+acdc_aws_etl_pipeline-0.4.2.dist-info/METADATA,sha256=Kzc0tDdxnD6OBHLZnOwbzw1MEwCdLG521YPFPttK1Jw,2853
+acdc_aws_etl_pipeline-0.4.2.dist-info/WHEEL,sha256=zp0Cn7JsFoX2ATtOhtaFYIiE2rmFAD4OcMhtUki8W3U,88
+acdc_aws_etl_pipeline-0.4.2.dist-info/RECORD,,