acdc_aws_etl_pipeline 0.7.6__tar.gz → 0.8.0__tar.gz

{acdc_aws_etl_pipeline-0.7.6 → acdc_aws_etl_pipeline-0.8.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: acdc_aws_etl_pipeline
-Version: 0.7.6
+Version: 0.8.0
 Summary: Tools for ACDC ETL pipeline
 Author: JoshuaHarris391
 Author-email: harjo391@gmail.com
@@ -32,19 +32,29 @@ Infrastructure and code for the ACDC ETL pipeline and data operations in AWS
 
 ## Documentation
 
-- [Dictionary deployment](docs/dictionary_deployment.md)
-- [Data ingestion](docs/data_ingestion.md)
-- [Data validation](docs/data_validation.md)
-- [Data transformation (dbt)](docs/data_transformation_dbt.md)
-- [Data releases](docs/write_data_release.md)
-- [Synthetic data generation](docs/synthetic_data_generation.md)
-- [REST API upload to sheepdog](docs/rest_api_sheepdog_upload.md)
-- [Data deletion](docs/data_deletion.md)
-- [IndexD file registration](docs/indexd_registration.md)
-- [Querying Athena](docs/querying_athena.md)
-- [Writing Athena queries to JSON](docs/write_athena_queries_to_json.md)
+### Core Configuration & Management
+- [Deployment Configuration Guide](docs/config.md)
+- [Dictionary Deployment](docs/dictionary_deployment.md)
+- [Service Management](docs/service_management.md)
+- [Kubernetes Utilities](docs/k8s_utilities.md)
 - [Troubleshooting](docs/troubleshooting.md)
 
+### Data Lifecycle & ETL
+- [Data Ingestion](docs/data_ingestion.md)
+- [Data Validation](docs/data_validation.md)
+- [Data Transformation (dbt)](docs/data_transformation_dbt.md)
+- [Data Releases](docs/write_data_release.md)
+- [Data Deletion](docs/data_deletion.md)
+
+### Metadata & Registry Operations
+- [REST API Upload to Sheepdog](docs/rest_api_sheepdog_upload.md)
+- [IndexD File Registration](docs/indexd_registration.md)
+- [Synthetic Data Generation](docs/synthetic_data_generation.md)
+
+### Analysis & Querying
+- [Querying Athena](docs/querying_athena.md)
+- [Writing Athena Queries to JSON](docs/write_athena_queries_to_json.md)
+
 ## Library and source code (`src/acdc_aws_etl_pipeline`)
 
 The Python package in [`src/acdc_aws_etl_pipeline`](src/acdc_aws_etl_pipeline) provides reusable utilities for ingestion, validation, uploads, and Athena/Glue operations used across the pipeline and services.

{acdc_aws_etl_pipeline-0.7.6 → acdc_aws_etl_pipeline-0.8.0}/README.md

@@ -3,19 +3,29 @@ Infrastructure and code for the ACDC ETL pipeline and data operations in AWS
 
 ## Documentation
 
-- [Dictionary deployment](docs/dictionary_deployment.md)
-- [Data ingestion](docs/data_ingestion.md)
-- [Data validation](docs/data_validation.md)
-- [Data transformation (dbt)](docs/data_transformation_dbt.md)
-- [Data releases](docs/write_data_release.md)
-- [Synthetic data generation](docs/synthetic_data_generation.md)
-- [REST API upload to sheepdog](docs/rest_api_sheepdog_upload.md)
-- [Data deletion](docs/data_deletion.md)
-- [IndexD file registration](docs/indexd_registration.md)
-- [Querying Athena](docs/querying_athena.md)
-- [Writing Athena queries to JSON](docs/write_athena_queries_to_json.md)
+### Core Configuration & Management
+- [Deployment Configuration Guide](docs/config.md)
+- [Dictionary Deployment](docs/dictionary_deployment.md)
+- [Service Management](docs/service_management.md)
+- [Kubernetes Utilities](docs/k8s_utilities.md)
 - [Troubleshooting](docs/troubleshooting.md)
 
+### Data Lifecycle & ETL
+- [Data Ingestion](docs/data_ingestion.md)
+- [Data Validation](docs/data_validation.md)
+- [Data Transformation (dbt)](docs/data_transformation_dbt.md)
+- [Data Releases](docs/write_data_release.md)
+- [Data Deletion](docs/data_deletion.md)
+
+### Metadata & Registry Operations
+- [REST API Upload to Sheepdog](docs/rest_api_sheepdog_upload.md)
+- [IndexD File Registration](docs/indexd_registration.md)
+- [Synthetic Data Generation](docs/synthetic_data_generation.md)
+
+### Analysis & Querying
+- [Querying Athena](docs/querying_athena.md)
+- [Writing Athena Queries to JSON](docs/write_athena_queries_to_json.md)
+
 ## Library and source code (`src/acdc_aws_etl_pipeline`)
 
 The Python package in [`src/acdc_aws_etl_pipeline`](src/acdc_aws_etl_pipeline) provides reusable utilities for ingestion, validation, uploads, and Athena/Glue operations used across the pipeline and services.

{acdc_aws_etl_pipeline-0.7.6 → acdc_aws_etl_pipeline-0.8.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "acdc_aws_etl_pipeline"
-version = "0.7.6"
+version = "0.8.0"
 description = "Tools for ACDC ETL pipeline"
 authors = ["JoshuaHarris391 <harjo391@gmail.com>"]
 readme = "README.md"

acdc_aws_etl_pipeline-0.8.0/src/acdc_aws_etl_pipeline/upload/metadata_deleter.py

@@ -0,0 +1,115 @@
+import os
+import logging
+from gen3.auth import Gen3Auth
+from gen3.submission import Gen3Submission
+from typing import Optional, List
+from acdc_aws_etl_pipeline.upload.metadata_submitter import (
+    create_boto3_session,
+    get_gen3_api_key_aws_secret,
+    infer_api_endpoint_from_jwt,
+    create_gen3_submission_class
+)
+
+logger = logging.getLogger(__name__)
+
+def delete_project_metadata(
+    import_order_file: str,
+    project_id: str,
+    aws_secret_name: str,
+    aws_profile: Optional[str] = None,
+    aws_region: str = "us-east-1",
+    exclude_nodes: Optional[List[str]] = None,
+    prompt_for_confirmation: bool = True,
+):
+    """
+    Deletes metadata json files from the gen3 api endpoint. Deletion depends on
+    a DataImportOrder.txt file, which defines the order of the nodes to be
+    deleted.
+
+    Args:
+        import_order_file (str): The path to the import order file
+        project_id (str): The ID of the project.
+        aws_secret_name (str): The name of the AWS secret containing the Gen3
+            API key.
+        aws_profile (str, optional): AWS profile name.
+        aws_region (str, optional): AWS region. Default is "ap-southeast-2".
+        exclude_nodes (list): A list of node names to exclude from the
+            deletion. Default is ["project", "program", "acknowledgement",
+            "publication"].
+        prompt_for_confirmation (bool): Whether to prompt for confirmation
+            before deletion.
+
+    Returns:
+        None
+    """
+
+    if aws_region is None:
+        aws_region = "ap-southeast-2"
+    if aws_profile is None:
+        aws_profile = "default"
+
+    if exclude_nodes is None:
+        exclude_nodes = [
+            "project",
+            "program",
+            "acknowledgement",
+            "publication",
+        ]
+
+    def get_import_order(import_order_file):
+        try:
+            with open(import_order_file, "r", encoding="utf-8") as f:
+                import_order = [line.rstrip() for line in f]
+                import_order = [
+                    node for node in import_order if node not in exclude_nodes
+                ]
+            return import_order
+        except FileNotFoundError:
+            logger.error(
+                "DataImportOrder.txt not found in %s", import_order_file
+            )
+            return []
+
+    ordered_import_nodes = get_import_order(import_order_file)
+
+    # AWS and Gen3 Authentication
+    session = create_boto3_session(aws_profile)
+    api_key_dict = get_gen3_api_key_aws_secret(
+        aws_secret_name, aws_region, session
+    )
+    if api_key_dict is None:
+        logger.error("API key not found in AWS secret %s", aws_secret_name)
+        return
+
+    sub = create_gen3_submission_class(api_key_dict)
+
+    final_ordered_import_nodes = [
+        node for node in ordered_import_nodes if node not in exclude_nodes
+    ]
+    final_ordered_import_nodes.reverse()  # Reverse the order for deletion
+
+    if prompt_for_confirmation:
+        confirm = input(
+            "Do you want to delete the metadata? (yes/no): "
+        ).strip().lower()
+        if confirm != "yes":
+            logger.info("Deletion cancelled by user.")
+            return
+
+    for node in final_ordered_import_nodes:
+        logger.info("[DELETE]  | Project: %-10s | Node: %-12s", project_id, node)
+        try:
+            sub.delete_nodes("program1", project_id, [node])
+            logger.info(
+                "\033[92m[SUCCESS]\033[0m | Project: %-10s | Node: %-12s",
+                project_id,
+                node,
+            )
+        except Exception as e:
+            logger.error(
+                "\033[91m[FAILED]\033[0m  | Project: %-10s | Node: %-12s | "
+                "Error: %s",
+                project_id,
+                node,
+                e,
+            )

{acdc_aws_etl_pipeline-0.7.6 → acdc_aws_etl_pipeline-0.8.0}/src/acdc_aws_etl_pipeline/upload/metadata_submitter.py

@@ -596,7 +596,9 @@ class MetadataSubmitter:
         self.aws_profile = aws_profile
         self.partition_cols = partition_cols or ["upload_datetime"]
         self.upload_to_database = upload_to_database
-        self.boto3_session = self._create_boto3_session()
+        self.boto3_session = None
+        if self.upload_to_database:
+            self.boto3_session = self._create_boto3_session()
         logger.info("MetadataSubmitter initialised.")
 
     def _create_gen3_submission_class(self):
@@ -996,10 +998,13 @@ class MetadataSubmitter:
         }
         return file_map
 
-    def submit_metadata(self) -> List[Dict[str, Any]]:
+    def submit_metadata(self, specific_node: Optional[str] = None) -> List[Dict[str, Any]]:
         """
         Submits metadata for each node defined in the data import order, except those in the exclude list.
         
+        Args:
+            specific_node (Optional[str]): If provided, only submits metadata for the specified node.
+        
         **Detailed Process:**
         1.  **Order Resolution:** The function reads the import order to determine the sequence of nodes.
         2.  **File Mapping:** It finds the matching `node.json` file for each node in the order.
@@ -1023,6 +1028,11 @@ class MetadataSubmitter:
         file_map = self._create_file_map()
 
         logger.info("Starting metadata submission.")
+        
+        if specific_node:
+            if specific_node not in data_import_order:
+                raise ValueError(f"Node '{specific_node}' not found in data import order.")
+            data_import_order = [specific_node]
 
         for node in data_import_order:
             if node in self.exclude_nodes:

acdc_aws_etl_pipeline-0.7.6/src/acdc_aws_etl_pipeline/upload/metadata_deleter.py

@@ -1,2 +0,0 @@
-from acdc_aws_etl_pipeline.upload.metadata_submitter import *
-