acdc_aws_etl_pipeline 0.4.3__tar.gz → 0.4.6__tar.gz

{acdc_aws_etl_pipeline-0.4.3 → acdc_aws_etl_pipeline-0.4.6}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: acdc_aws_etl_pipeline
-Version: 0.4.3
+Version: 0.4.6
 Summary: Tools for ACDC ETL pipeline
 Author: JoshuaHarris391
 Author-email: harjo391@gmail.com

{acdc_aws_etl_pipeline-0.4.3 → acdc_aws_etl_pipeline-0.4.6}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "acdc_aws_etl_pipeline"
-version = "0.4.3"
+version = "0.4.6"
 description = "Tools for ACDC ETL pipeline"
 authors = ["JoshuaHarris391 <harjo391@gmail.com>"]
 readme = "README.md"

{acdc_aws_etl_pipeline-0.4.3 → acdc_aws_etl_pipeline-0.4.6}/src/acdc_aws_etl_pipeline/upload/metadata_submitter.py

@@ -418,111 +418,121 @@ def submit_metadata(
     Notes:
         Each file is split into size-friendly chunks before submit. Local and S3 files are supported.
     """
-    
+
     timestamp = datetime.now().strftime("%Y%d%m-%H%M%S")
     log_dir = f"submission_logs/{timestamp}"
     os.makedirs(log_dir, exist_ok=True)
-    
+
     if exclude_nodes is None:
         exclude_nodes = ["project", "program", "acknowledgement", "publication"]
 
     logger.info("Starting metadata submission process.")
     logger.info(f"Creating Gen3Submission class for endpoint: {api_endpoint}")
-    submit = create_gen3_submission_class(api_key, api_endpoint)
-
-    if is_s3_uri(data_import_order_path):
-        logger.info(f"Reading import order from S3: {data_import_order_path}")
-        import_order = read_data_import_order_txt_s3(data_import_order_path, boto3_session)
-        logger.debug(f"Import order from S3: {import_order}")
-    else:
-        logger.info(f"Reading import order from file: {data_import_order_path}")
-        import_order = read_data_import_order_txt(data_import_order_path, exclude_nodes)
-        logger.debug(f"Import order from file: {import_order}")
-
-    # Map node name to file for fast access and avoid repeatedly scanning file_list
-    file_map = {get_node_from_file_path(file): file for file in file_list}
-
-    for node in import_order:
-        if node in exclude_nodes:
-            logger.info(f"Skipping node '{node}' (in exclude list).")
-            continue
-        file = file_map.get(node)
-        if not file:
-            logger.info(f"Skipping node '{node}' (not present in file list).")
-            continue
-
-        logger.info(f"Processing file '{file}' for node '{node}'.")
-
-        try:
-            if is_s3_uri(file):
-                logger.info(f"Reading JSON data for node '{node}' from S3 file: {file}")
-                json_data = read_metadata_json_s3(file, boto3_session)
-            else:
-                logger.info(f"Reading JSON data for node '{node}' from local file: {file}")
-                json_data = read_metadata_json(file)
-        except Exception as e:
-            logger.error(f"Error reading JSON for node '{node}' from {file}: {e}")
-            continue
-
-        if not json_data:
-            logger.info(f"Skipping node '{node}' due to errors in reading JSON.")
-            continue
-
-        split_json_list = split_json_objects(json_data, max_size_kb=max_size_kb)
-        n_json_data = len(split_json_list)
-        logger.info(
-            f"--- Starting submission process for node '{node}' ({n_json_data} chunks) ---"
-        )
 
-        for index, jsn in enumerate(split_json_list):
-            progress_str = f"{index + 1}/{n_json_data}"
-
-            for attempt in range(max_retries + 1):
-                try:
-                    log_msg = (
-                        f"[SUBMIT]  | Project: {project_id:<10} | Node: {node:<12} | "
-                        f"Split: {progress_str:<5}"
-                        if attempt == 0 else
-                        f"[RETRY]   | Project: {project_id:<10} | Node: {node:<12} | "
-                        f"Split: {progress_str:<5} | Attempt: {attempt}/{max_retries}"
-                    )
-                    logger.info(log_msg) if attempt == 0 else logger.warning(log_msg)
-
-                    res = submit.submit_record("program1", project_id, jsn)
-                    
-                    # writing submission results as log json
-                    if write_submission_results_path is not None:
-                        log_filename = os.path.join(
-                            log_dir, f"{project_id}_{node}_split{index + 1}_of_{n_json_data}.json"
+    try:
+        submit = create_gen3_submission_class(api_key, api_endpoint)
+
+        if is_s3_uri(data_import_order_path):
+            logger.info(f"Reading import order from S3: {data_import_order_path}")
+            import_order = read_data_import_order_txt_s3(data_import_order_path, boto3_session)
+            logger.debug(f"Import order from S3: {import_order}")
+        else:
+            logger.info(f"Reading import order from file: {data_import_order_path}")
+            import_order = read_data_import_order_txt(data_import_order_path, exclude_nodes)
+            logger.debug(f"Import order from file: {import_order}")
+
+        file_map = {get_node_from_file_path(file): file for file in file_list}
+
+        for node in import_order:
+            if node in exclude_nodes:
+                logger.info(f"Skipping node '{node}' (in exclude list).")
+                continue
+            file = file_map.get(node)
+            if not file:
+                logger.info(f"Skipping node '{node}' (not present in file list).")
+                continue
+
+            logger.info(f"Processing file '{file}' for node '{node}'.")
+
+            try:
+                if is_s3_uri(file):
+                    logger.info(f"Reading JSON data for node '{node}' from S3 file: {file}")
+                    json_data = read_metadata_json_s3(file, boto3_session)
+                else:
+                    logger.info(f"Reading JSON data for node '{node}' from local file: {file}")
+                    json_data = read_metadata_json(file)
+            except Exception as e:
+                logger.error(f"Error reading JSON for node '{node}' from {file}: {e}")
+                raise Exception(f"Failed to read JSON metadata for node '{node}' from {file}: {e}")
+
+            split_json_list = split_json_objects(json_data, max_size_kb=max_size_kb)
+            n_json_data = len(split_json_list)
+            logger.info(
+                f"--- Starting submission process for node '{node}' ({n_json_data} chunks) ---"
+            )
+
+            for index, jsn in enumerate(split_json_list):
+                progress_str = f"{index + 1}/{n_json_data}"
+
+                submission_success = False
+                last_exception = None
+                for attempt in range(max_retries + 1):
+                    try:
+                        log_msg = (
+                            f"[SUBMIT]  | Project: {project_id:<10} | Node: {node:<12} | "
+                            f"Split: {progress_str:<5}"
+                            if attempt == 0 else
+                            f"[RETRY]   | Project: {project_id:<10} | Node: {node:<12} | "
+                            f"Split: {progress_str:<5} | Attempt: {attempt}/{max_retries}"
                         )
-                        abs_log_filename = os.path.abspath(log_filename)
-                        with open(abs_log_filename, "a") as f:
-                            json.dump(res, f)
-                            f.write("\n")
+                        logger.info(log_msg) if attempt == 0 else logger.warning(log_msg)
+
+                        res = submit.submit_record("program1", project_id, jsn)
+
+                        if write_submission_results_path is not None:
+                            log_filename = os.path.join(
+                                log_dir, f"{project_id}_{node}_split{index + 1}_of_{n_json_data}.json"
+                            )
+                            abs_log_filename = os.path.abspath(log_filename)
+                            with open(abs_log_filename, "a") as f:
+                                json.dump(res, f)
+                                f.write("\n")
+                            logger.info(
+                                f"Wrote submission response to log file: {abs_log_filename}"
+                            )
+
                         logger.info(
-                            f"Wrote submission response to log file: {abs_log_filename}"
+                            f"\033[92m[SUCCESS]\033[0m | Project: {project_id:<10} | "
+                            f"Node: {node:<12} | Split: {progress_str:<5}"
                         )
+                        submission_success = True
+                        break  # Success
 
-                    logger.info(
-                        f"\033[92m[SUCCESS]\033[0m | Project: {project_id:<10} | "
-                        f"Node: {node:<12} | Split: {progress_str:<5}"
+                    except Exception as e:
+                        last_exception = e
+                        logger.error(
+                            f"Error submitting chunk {progress_str} for node '{node}': {e}"
+                        )
+                        if attempt < max_retries:
+                            import time
+                            time.sleep(0.2)
+                        else:
+                            logger.critical(
+                                f"\033[91m[FAILED]\033[0m  | Project: {project_id:<10} | "
+                                f"Node: {node:<12} | Split: {progress_str:<5} | Error: {e}"
+                            )
+
+                if not submission_success:
+                    # After retries, still failed
+                    raise Exception(
+                        f"Failed to submit chunk {progress_str} for node '{node}' after {max_retries + 1} attempts. "
+                        f"Last error: {last_exception}"
                     )
-                    break  # Successful, move to next chunk
 
-                except Exception as e:
-                    logger.error(
-                        f"Error submitting chunk {progress_str} for node '{node}': {e}"
-                    )
-                    if attempt < max_retries:
-                        import time
-                        time.sleep(0.2)
-                    else:
-                        logger.critical(
-                            f"\033[91m[FAILED]\033[0m  | Project: {project_id:<10} | "
-                            f"Node: {node:<12} | Split: {progress_str:<5} | Error: {e}"
-                        )
-                        raise
+            logger.info(f"Finished submitting node '{node}'.")
 
-        logger.info(f"Finished submitting node '{node}'.")
+        logger.info("--- Submission process complete ---")
 
-    logger.info("--- Submission process complete ---")
+    except Exception as exc:
+        logger.exception(f"Critical error during submission process: {exc}")
+        raise