acdc_aws_etl_pipeline 0.3.7__tar.gz → 0.3.9__tar.gz

{acdc_aws_etl_pipeline-0.3.7 → acdc_aws_etl_pipeline-0.3.9}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: acdc_aws_etl_pipeline
-Version: 0.3.7
+Version: 0.3.9
 Summary: Tools for ACDC ETL pipeline
 Author: JoshuaHarris391
 Author-email: harjo391@gmail.com

{acdc_aws_etl_pipeline-0.3.7 → acdc_aws_etl_pipeline-0.3.9}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "acdc_aws_etl_pipeline"
-version = "0.3.7"
+version = "0.3.9"
 description = "Tools for ACDC ETL pipeline"
 authors = ["JoshuaHarris391 <harjo391@gmail.com>"]
 readme = "README.md"

{acdc_aws_etl_pipeline-0.3.7 → acdc_aws_etl_pipeline-0.3.9}/src/acdc_aws_etl_pipeline/utils/athena_utils.py

@@ -13,6 +13,8 @@ import pytz
 import base64
 import numpy as np
 from decimal import Decimal
+from gen3_validator.dict import DataDictionary
+from acdc_aws_etl_pipeline.validate.validate import load_schema_from_s3_uri
 
 logger = logging.getLogger(__name__)
 
@@ -688,9 +690,37 @@ def write_gold_json_to_s3(
     logger.info(f"Object created at s3://{s3_bucket}/{s3_object_key}")
 
 
+def construct_data_import_order(s3_uri) -> list:
+    schema_dict = load_schema_from_s3_uri(s3_uri)
+    dd = DataDictionary(schema_dict)
+    dd.schema = schema_dict
+    dd.calculate_node_order()
+    return dd.node_order
+
 def write_release_jsons_to_s3(s3_bucket, release_id, study_id, table_name, json_data):
+    """
+    Write a JSON string to a specific S3 location for a given release and study.
+
+    Args:
+        s3_bucket (str): The S3 bucket where the file will be uploaded.
+        release_id (str): Release identifier used in the S3 key path.
+        study_id (str): Study identifier used in the S3 key path.
+        table_name (str): Table name (used for naming the .json file).
+        json_data (str): JSON data (as a string) to be uploaded.
+
+    Returns:
+        str: The output directory path in S3 where the file was written.
+
+    Raises:
+        Exception: Any exception raised by boto3.client('s3').put_object.
+
+    Example:
+        >>> output_dir = write_release_jsons_to_s3('my-bucket', 'release123', 'study1', 'gold_foo', '{"x":1}')
+    """
     s3 = boto3.client('s3')
-    s3_object_key = f"release_jsons/{release_id}/{study_id}/{table_name}.json"
+    output_dir = f"release_jsons/{release_id}/{study_id}"
+    s3_object_key = f"{output_dir}/{table_name}.json"
     logger.info(f"Writing JSON data to S3 bucket: {s3_bucket}, object key: {s3_object_key}")
     s3.put_object(Body=json_data, Bucket=s3_bucket, Key=s3_object_key)
-    logger.info(f"Object created at s3://{s3_bucket}/{s3_object_key}")
+    logger.info(f"Object created at s3://{s3_bucket}/{s3_object_key}")
+    return output_dir