acdc_aws_etl_pipeline 0.4.1__tar.gz → 0.4.2__tar.gz

{acdc_aws_etl_pipeline-0.4.1 → acdc_aws_etl_pipeline-0.4.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: acdc_aws_etl_pipeline
-Version: 0.4.1
+Version: 0.4.2
 Summary: Tools for ACDC ETL pipeline
 Author: JoshuaHarris391
 Author-email: harjo391@gmail.com

{acdc_aws_etl_pipeline-0.4.1 → acdc_aws_etl_pipeline-0.4.2}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "acdc_aws_etl_pipeline"
-version = "0.4.1"
+version = "0.4.2"
 description = "Tools for ACDC ETL pipeline"
 authors = ["JoshuaHarris391 <harjo391@gmail.com>"]
 readme = "README.md"

{acdc_aws_etl_pipeline-0.4.1 → acdc_aws_etl_pipeline-0.4.2}/src/acdc_aws_etl_pipeline/upload/metadata_submitter.py

@@ -393,6 +393,7 @@ def submit_metadata(
     max_size_kb: int = 50,
     exclude_nodes: list = None,
     max_retries: int = 5,
+    write_submission_results_path: str = None
 ):
     """
     Submit a set of metadata JSON files to a Gen3 data commons endpoint, in order.
@@ -490,16 +491,17 @@ def submit_metadata(
                     res = submit.submit_record("program1", project_id, jsn)
                     
                     # writing submission results as log json
-                    log_filename = os.path.join(
-                        log_dir, f"{project_id}_{node}_split{index + 1}_of_{n_json_data}.json"
-                    )
-                    abs_log_filename = os.path.abspath(log_filename)
-                    with open(abs_log_filename, "a") as f:
-                        json.dump(res, f)
-                        f.write("\n")
-                    logger.info(
-                        f"Wrote submission response to log file: {abs_log_filename}"
-                    )
+                    if write_submission_results_path is not None:
+                        log_filename = os.path.join(
+                            log_dir, f"{project_id}_{node}_split{index + 1}_of_{n_json_data}.json"
+                        )
+                        abs_log_filename = os.path.abspath(log_filename)
+                        with open(abs_log_filename, "a") as f:
+                            json.dump(res, f)
+                            f.write("\n")
+                        logger.info(
+                            f"Wrote submission response to log file: {abs_log_filename}"
+                        )
 
                     logger.info(
                         f"\033[92m[SUCCESS]\033[0m | Project: {project_id:<10} | "