acdc_aws_etl_pipeline 0.4.5__py3-none-any.whl → 0.4.7__py3-none-any.whl

acdc_aws_etl_pipeline/upload/metadata_submitter.py

@@ -419,110 +419,123 @@ def submit_metadata(
         Each file is split into size-friendly chunks before submit. Local and S3 files are supported.
     """
     
+    # redefine to use local cache in /tmp
+    os.environ['XDG_CACHE_HOME'] = '/tmp/.cache'
+
     timestamp = datetime.now().strftime("%Y%d%m-%H%M%S")
     log_dir = f"submission_logs/{timestamp}"
     os.makedirs(log_dir, exist_ok=True)
-    
+
     if exclude_nodes is None:
         exclude_nodes = ["project", "program", "acknowledgement", "publication"]
 
     logger.info("Starting metadata submission process.")
     logger.info(f"Creating Gen3Submission class for endpoint: {api_endpoint}")
-    submit = create_gen3_submission_class(api_key, api_endpoint)
-
-    if is_s3_uri(data_import_order_path):
-        logger.info(f"Reading import order from S3: {data_import_order_path}")
-        import_order = read_data_import_order_txt_s3(data_import_order_path, boto3_session)
-        logger.debug(f"Import order from S3: {import_order}")
-    else:
-        logger.info(f"Reading import order from file: {data_import_order_path}")
-        import_order = read_data_import_order_txt(data_import_order_path, exclude_nodes)
-        logger.debug(f"Import order from file: {import_order}")
-
-    # Map node name to file for fast access and avoid repeatedly scanning file_list
-    file_map = {get_node_from_file_path(file): file for file in file_list}
-
-    for node in import_order:
-        if node in exclude_nodes:
-            logger.info(f"Skipping node '{node}' (in exclude list).")
-            continue
-        file = file_map.get(node)
-        if not file:
-            logger.info(f"Skipping node '{node}' (not present in file list).")
-            continue
-
-        logger.info(f"Processing file '{file}' for node '{node}'.")
-
-        try:
-            if is_s3_uri(file):
-                logger.info(f"Reading JSON data for node '{node}' from S3 file: {file}")
-                json_data = read_metadata_json_s3(file, boto3_session)
-            else:
-                logger.info(f"Reading JSON data for node '{node}' from local file: {file}")
-                json_data = read_metadata_json(file)
-        except Exception as e:
-            logger.error(f"Error reading JSON for node '{node}' from {file}: {e}")
-            continue
-
-        if not json_data:
-            logger.info(f"Skipping node '{node}' due to errors in reading JSON.")
-            continue
-
-        split_json_list = split_json_objects(json_data, max_size_kb=max_size_kb)
-        n_json_data = len(split_json_list)
-        logger.info(
-            f"--- Starting submission process for node '{node}' ({n_json_data} chunks) ---"
-        )
 
-        for index, jsn in enumerate(split_json_list):
-            progress_str = f"{index + 1}/{n_json_data}"
-
-            for attempt in range(max_retries + 1):
-                try:
-                    log_msg = (
-                        f"[SUBMIT]  | Project: {project_id:<10} | Node: {node:<12} | "
-                        f"Split: {progress_str:<5}"
-                        if attempt == 0 else
-                        f"[RETRY]   | Project: {project_id:<10} | Node: {node:<12} | "
-                        f"Split: {progress_str:<5} | Attempt: {attempt}/{max_retries}"
-                    )
-                    logger.info(log_msg) if attempt == 0 else logger.warning(log_msg)
-
-                    res = submit.submit_record("program1", project_id, jsn)
-                    
-                    # writing submission results as log json
-                    if write_submission_results_path is not None:
-                        log_filename = os.path.join(
-                            log_dir, f"{project_id}_{node}_split{index + 1}_of_{n_json_data}.json"
+    try:
+        submit = create_gen3_submission_class(api_key, api_endpoint)
+
+        if is_s3_uri(data_import_order_path):
+            logger.info(f"Reading import order from S3: {data_import_order_path}")
+            import_order = read_data_import_order_txt_s3(data_import_order_path, boto3_session)
+            logger.debug(f"Import order from S3: {import_order}")
+        else:
+            logger.info(f"Reading import order from file: {data_import_order_path}")
+            import_order = read_data_import_order_txt(data_import_order_path, exclude_nodes)
+            logger.debug(f"Import order from file: {import_order}")
+
+        file_map = {get_node_from_file_path(file): file for file in file_list}
+
+        for node in import_order:
+            if node in exclude_nodes:
+                logger.info(f"Skipping node '{node}' (in exclude list).")
+                continue
+            file = file_map.get(node)
+            if not file:
+                logger.info(f"Skipping node '{node}' (not present in file list).")
+                continue
+
+            logger.info(f"Processing file '{file}' for node '{node}'.")
+
+            try:
+                if is_s3_uri(file):
+                    logger.info(f"Reading JSON data for node '{node}' from S3 file: {file}")
+                    json_data = read_metadata_json_s3(file, boto3_session)
+                else:
+                    logger.info(f"Reading JSON data for node '{node}' from local file: {file}")
+                    json_data = read_metadata_json(file)
+            except Exception as e:
+                logger.error(f"Error reading JSON for node '{node}' from {file}: {e}")
+                raise Exception(f"Failed to read JSON metadata for node '{node}' from {file}: {e}")
+
+            split_json_list = split_json_objects(json_data, max_size_kb=max_size_kb)
+            n_json_data = len(split_json_list)
+            logger.info(
+                f"--- Starting submission process for node '{node}' ({n_json_data} chunks) ---"
+            )
+
+            for index, jsn in enumerate(split_json_list):
+                progress_str = f"{index + 1}/{n_json_data}"
+
+                submission_success = False
+                last_exception = None
+                for attempt in range(max_retries + 1):
+                    try:
+                        log_msg = (
+                            f"[SUBMIT]  | Project: {project_id:<10} | Node: {node:<12} | "
+                            f"Split: {progress_str:<5}"
+                            if attempt == 0 else
+                            f"[RETRY]   | Project: {project_id:<10} | Node: {node:<12} | "
+                            f"Split: {progress_str:<5} | Attempt: {attempt}/{max_retries}"
                         )
-                        abs_log_filename = os.path.abspath(log_filename)
-                        with open(abs_log_filename, "a") as f:
-                            json.dump(res, f)
-                            f.write("\n")
+                        logger.info(log_msg) if attempt == 0 else logger.warning(log_msg)
+
+                        res = submit.submit_record("program1", project_id, jsn)
+
+                        if write_submission_results_path is not None:
+                            log_filename = os.path.join(
+                                log_dir, f"{project_id}_{node}_split{index + 1}_of_{n_json_data}.json"
+                            )
+                            abs_log_filename = os.path.abspath(log_filename)
+                            with open(abs_log_filename, "a") as f:
+                                json.dump(res, f)
+                                f.write("\n")
+                            logger.info(
+                                f"Wrote submission response to log file: {abs_log_filename}"
+                            )
+
                         logger.info(
-                            f"Wrote submission response to log file: {abs_log_filename}"
+                            f"\033[92m[SUCCESS]\033[0m | Project: {project_id:<10} | "
+                            f"Node: {node:<12} | Split: {progress_str:<5}"
                         )
+                        submission_success = True
+                        break  # Success
 
-                    logger.info(
-                        f"\033[92m[SUCCESS]\033[0m | Project: {project_id:<10} | "
-                        f"Node: {node:<12} | Split: {progress_str:<5}"
+                    except Exception as e:
+                        last_exception = e
+                        logger.error(
+                            f"Error submitting chunk {progress_str} for node '{node}': {e}"
+                        )
+                        if attempt < max_retries:
+                            import time
+                            time.sleep(0.2)
+                        else:
+                            logger.critical(
+                                f"\033[91m[FAILED]\033[0m  | Project: {project_id:<10} | "
+                                f"Node: {node:<12} | Split: {progress_str:<5} | Error: {e}"
+                            )
+
+                if not submission_success:
+                    # After retries, still failed
+                    raise Exception(
+                        f"Failed to submit chunk {progress_str} for node '{node}' after {max_retries + 1} attempts. "
+                        f"Last error: {last_exception}"
                     )
-                    break  # Successful, move to next chunk
 
-                except Exception as e:
-                    logger.error(
-                        f"Error submitting chunk {progress_str} for node '{node}': {e}"
-                    )
-                    if attempt < max_retries:
-                        import time
-                        time.sleep(0.2)
-                    else:
-                        logger.critical(
-                            f"\033[91m[FAILED]\033[0m  | Project: {project_id:<10} | "
-                            f"Node: {node:<12} | Split: {progress_str:<5} | Error: {e}"
-                        )
-                        raise
+            logger.info(f"Finished submitting node '{node}'.")
 
-        logger.info(f"Finished submitting node '{node}'.")
+        logger.info("--- Submission process complete ---")
 
-    logger.info("--- Submission process complete ---")
+    except Exception as exc:
+        logger.exception(f"Critical error during submission process: {exc}")
+        raise

{acdc_aws_etl_pipeline-0.4.5.dist-info → acdc_aws_etl_pipeline-0.4.7.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: acdc_aws_etl_pipeline
-Version: 0.4.5
+Version: 0.4.7
 Summary: Tools for ACDC ETL pipeline
 Author: JoshuaHarris391
 Author-email: harjo391@gmail.com

{acdc_aws_etl_pipeline-0.4.5.dist-info → acdc_aws_etl_pipeline-0.4.7.dist-info}/RECORD

@@ -3,12 +3,12 @@ acdc_aws_etl_pipeline/ingest/ingest.py,sha256=0meo4Sq6o0EPvgfFu0QPMQ5ZEHDiSEMsVG
 acdc_aws_etl_pipeline/upload/__init__.py,sha256=kRI1wozjK-b9YXMAPwzWHzm967ZiUAM6g8rRo4ONWtI,67
 acdc_aws_etl_pipeline/upload/gen3datasubmitter.py,sha256=bu5d8IOsKFIA1uvvzaxb7YIKwBZKdP-0QvBt-gZMyUc,8625
 acdc_aws_etl_pipeline/upload/metadata_deleter.py,sha256=T4q9xqSE2Beu3zluvAmKh7wJWcCFGz2AZ9h9ZcASfyA,63
-acdc_aws_etl_pipeline/upload/metadata_submitter.py,sha256=9ms1X3rjP4INGfNSr4GsM6FkTOfT3JRF3PxAN6pisFE,20316
+acdc_aws_etl_pipeline/upload/metadata_submitter.py,sha256=agRDalSU8cyNQ0rm0BTm-U8pdWe93w3roQ-kPBhpxw0,21071
 acdc_aws_etl_pipeline/upload/upload_synthdata_s3.py,sha256=Ge5TQzZkWnJNp-q0Ak-Yhv1h1eWLxg-PlWVHrd1m0B8,5155
 acdc_aws_etl_pipeline/utils/athena_utils.py,sha256=QJlBe-07Hkq-BqmcxBu6ZtAmVfZSHuSY4dijcysgPH8,29560
 acdc_aws_etl_pipeline/utils/dbt_utils.py,sha256=5XRFOwNNIeuW2sQuor3h_OZTuXGg6xv2AUYwj9bMAAM,2054
 acdc_aws_etl_pipeline/utils/release_writer.py,sha256=vsxHJ6l-UWPpzeyEPHurX5iFgeCEQ-9FbySAbPNfTTM,7555
 acdc_aws_etl_pipeline/validate/validate.py,sha256=fTa76YvixCWOGkAIuR7CZ2WryMJcpc2wvSOHLZDEknc,28159
-acdc_aws_etl_pipeline-0.4.5.dist-info/METADATA,sha256=AclICNhbZYTn6D5sMY6bvpWRIZTaXHHnsynyzgRW-Po,2853
-acdc_aws_etl_pipeline-0.4.5.dist-info/WHEEL,sha256=zp0Cn7JsFoX2ATtOhtaFYIiE2rmFAD4OcMhtUki8W3U,88
-acdc_aws_etl_pipeline-0.4.5.dist-info/RECORD,,
+acdc_aws_etl_pipeline-0.4.7.dist-info/METADATA,sha256=2EoWAI7fk6Naqd2g-YAslv-JdSS0cxyssO32tUsh9yE,2853
+acdc_aws_etl_pipeline-0.4.7.dist-info/WHEEL,sha256=zp0Cn7JsFoX2ATtOhtaFYIiE2rmFAD4OcMhtUki8W3U,88
+acdc_aws_etl_pipeline-0.4.7.dist-info/RECORD,,