acdc_aws_etl_pipeline 0.3.6__tar.gz → 0.3.8__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (12) hide show
  1. {acdc_aws_etl_pipeline-0.3.6 → acdc_aws_etl_pipeline-0.3.8}/PKG-INFO +7 -2
  2. {acdc_aws_etl_pipeline-0.3.6 → acdc_aws_etl_pipeline-0.3.8}/README.md +6 -1
  3. {acdc_aws_etl_pipeline-0.3.6 → acdc_aws_etl_pipeline-0.3.8}/pyproject.toml +1 -1
  4. {acdc_aws_etl_pipeline-0.3.6 → acdc_aws_etl_pipeline-0.3.8}/src/acdc_aws_etl_pipeline/utils/release_writer.py +3 -0
  5. {acdc_aws_etl_pipeline-0.3.6 → acdc_aws_etl_pipeline-0.3.8}/src/acdc_aws_etl_pipeline/__init__.py +0 -0
  6. {acdc_aws_etl_pipeline-0.3.6 → acdc_aws_etl_pipeline-0.3.8}/src/acdc_aws_etl_pipeline/ingest/ingest.py +0 -0
  7. {acdc_aws_etl_pipeline-0.3.6 → acdc_aws_etl_pipeline-0.3.8}/src/acdc_aws_etl_pipeline/upload/__init__.py +0 -0
  8. {acdc_aws_etl_pipeline-0.3.6 → acdc_aws_etl_pipeline-0.3.8}/src/acdc_aws_etl_pipeline/upload/gen3datasubmitter.py +0 -0
  9. {acdc_aws_etl_pipeline-0.3.6 → acdc_aws_etl_pipeline-0.3.8}/src/acdc_aws_etl_pipeline/upload/upload_synthdata_s3.py +0 -0
  10. {acdc_aws_etl_pipeline-0.3.6 → acdc_aws_etl_pipeline-0.3.8}/src/acdc_aws_etl_pipeline/utils/athena_utils.py +0 -0
  11. {acdc_aws_etl_pipeline-0.3.6 → acdc_aws_etl_pipeline-0.3.8}/src/acdc_aws_etl_pipeline/utils/dbt_utils.py +0 -0
  12. {acdc_aws_etl_pipeline-0.3.6 → acdc_aws_etl_pipeline-0.3.8}/src/acdc_aws_etl_pipeline/validate/validate.py +0 -0
{acdc_aws_etl_pipeline-0.3.6 → acdc_aws_etl_pipeline-0.3.8}/PKG-INFO RENAMED
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: acdc_aws_etl_pipeline
3
- Version: 0.3.6
3
+ Version: 0.3.8
4
4
  Summary: Tools for ACDC ETL pipeline
5
5
  Author: JoshuaHarris391
6
6
  Author-email: harjo391@gmail.com
@@ -47,11 +47,16 @@ e.g. to testing
47
47
  bash services/dictionary/pull_dict.sh <raw_dictionary_url>
48
48
  bash services/dictionary/upload_dictionary.py <local_dictionary_path> <s3_target_uri>
49
49
 
50
- # implementation
50
+ # Deploying to test
51
51
  VERSION=v0.6.3
52
52
  bash services/dictionary/pull_dict.sh "https://raw.githubusercontent.com/AustralianBioCommons/acdc-schema-json/refs/tags/${VERSION}/dictionary/prod_dict/acdc_schema.json"
53
53
  python3 services/dictionary/upload_dictionary.py "services/dictionary/schemas/acdc_schema_${VERSION}.json" s3://gen3schema-cad-uat-biocommons.org.au/cad.json
54
54
 
55
+
56
+ # Deploying to staging
57
+ VERSION=v0.7.7
58
+ bash services/dictionary/pull_dict.sh "https://raw.githubusercontent.com/AustralianBioCommons/acdc-schema-json/refs/tags/${VERSION}/dictionary/prod_dict/acdc_schema.json"
59
+ python3 services/dictionary/upload_dictionary.py "services/dictionary/schemas/acdc_schema_${VERSION}.json" s3://gen3schema-cad-staging-biocommons.org.au/cad.json
55
60
  ```
56
61
 
57
62
  ## Generating synthetic metadata
{acdc_aws_etl_pipeline-0.3.6 → acdc_aws_etl_pipeline-0.3.8}/README.md RENAMED
@@ -21,11 +21,16 @@ e.g. to testing
21
21
  bash services/dictionary/pull_dict.sh <raw_dictionary_url>
22
22
  bash services/dictionary/upload_dictionary.py <local_dictionary_path> <s3_target_uri>
23
23
 
24
- # implementation
24
+ # Deploying to test
25
25
  VERSION=v0.6.3
26
26
  bash services/dictionary/pull_dict.sh "https://raw.githubusercontent.com/AustralianBioCommons/acdc-schema-json/refs/tags/${VERSION}/dictionary/prod_dict/acdc_schema.json"
27
27
  python3 services/dictionary/upload_dictionary.py "services/dictionary/schemas/acdc_schema_${VERSION}.json" s3://gen3schema-cad-uat-biocommons.org.au/cad.json
28
28
 
29
+
30
+ # Deploying to staging
31
+ VERSION=v0.7.7
32
+ bash services/dictionary/pull_dict.sh "https://raw.githubusercontent.com/AustralianBioCommons/acdc-schema-json/refs/tags/${VERSION}/dictionary/prod_dict/acdc_schema.json"
33
+ python3 services/dictionary/upload_dictionary.py "services/dictionary/schemas/acdc_schema_${VERSION}.json" s3://gen3schema-cad-staging-biocommons.org.au/cad.json
29
34
  ```
30
35
 
31
36
  ## Generating synthetic metadata
{acdc_aws_etl_pipeline-0.3.6 → acdc_aws_etl_pipeline-0.3.8}/pyproject.toml RENAMED
@@ -1,6 +1,6 @@
1
1
  [tool.poetry]
2
2
  name = "acdc_aws_etl_pipeline"
3
- version = "0.3.6"
3
+ version = "0.3.8"
4
4
  description = "Tools for ACDC ETL pipeline"
5
5
  authors = ["JoshuaHarris391 <harjo391@gmail.com>"]
6
6
  readme = "README.md"
{acdc_aws_etl_pipeline-0.3.6 → acdc_aws_etl_pipeline-0.3.8}/src/acdc_aws_etl_pipeline/utils/release_writer.py RENAMED
@@ -148,6 +148,9 @@ def main():
148
148
 
149
149
  dbt_models = get_model_names(args.dbt_schema_path)
150
150
  athena_query = AthenaQuery(athena_config)
151
+
152
+ logger.info(f"Creating release table: {args.release_db}.{args.release_table}")
153
+ athena_query.create_release_table()
151
154
 
152
155
  logger.info(f"Processing DBT models from schema: {dbt_models}")
153
156