acdc_aws_etl_pipeline 0.3.5__tar.gz → 0.3.7__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (12) hide show
  1. {acdc_aws_etl_pipeline-0.3.5 → acdc_aws_etl_pipeline-0.3.7}/PKG-INFO +13 -4
  2. {acdc_aws_etl_pipeline-0.3.5 → acdc_aws_etl_pipeline-0.3.7}/README.md +12 -3
  3. {acdc_aws_etl_pipeline-0.3.5 → acdc_aws_etl_pipeline-0.3.7}/pyproject.toml +1 -1
  4. {acdc_aws_etl_pipeline-0.3.5 → acdc_aws_etl_pipeline-0.3.7}/src/acdc_aws_etl_pipeline/utils/release_writer.py +3 -0
  5. {acdc_aws_etl_pipeline-0.3.5 → acdc_aws_etl_pipeline-0.3.7}/src/acdc_aws_etl_pipeline/__init__.py +0 -0
  6. {acdc_aws_etl_pipeline-0.3.5 → acdc_aws_etl_pipeline-0.3.7}/src/acdc_aws_etl_pipeline/ingest/ingest.py +0 -0
  7. {acdc_aws_etl_pipeline-0.3.5 → acdc_aws_etl_pipeline-0.3.7}/src/acdc_aws_etl_pipeline/upload/__init__.py +0 -0
  8. {acdc_aws_etl_pipeline-0.3.5 → acdc_aws_etl_pipeline-0.3.7}/src/acdc_aws_etl_pipeline/upload/gen3datasubmitter.py +0 -0
  9. {acdc_aws_etl_pipeline-0.3.5 → acdc_aws_etl_pipeline-0.3.7}/src/acdc_aws_etl_pipeline/upload/upload_synthdata_s3.py +0 -0
  10. {acdc_aws_etl_pipeline-0.3.5 → acdc_aws_etl_pipeline-0.3.7}/src/acdc_aws_etl_pipeline/utils/athena_utils.py +0 -0
  11. {acdc_aws_etl_pipeline-0.3.5 → acdc_aws_etl_pipeline-0.3.7}/src/acdc_aws_etl_pipeline/utils/dbt_utils.py +0 -0
  12. {acdc_aws_etl_pipeline-0.3.5 → acdc_aws_etl_pipeline-0.3.7}/src/acdc_aws_etl_pipeline/validate/validate.py +0 -0
{acdc_aws_etl_pipeline-0.3.5 → acdc_aws_etl_pipeline-0.3.7}/PKG-INFO RENAMED
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: acdc_aws_etl_pipeline
3
- Version: 0.3.5
3
+ Version: 0.3.7
4
4
  Summary: Tools for ACDC ETL pipeline
5
5
  Author: JoshuaHarris391
6
6
  Author-email: harjo391@gmail.com
@@ -47,11 +47,16 @@ e.g. to testing
47
47
  bash services/dictionary/pull_dict.sh <raw_dictionary_url>
48
48
  bash services/dictionary/upload_dictionary.py <local_dictionary_path> <s3_target_uri>
49
49
 
50
- # implementation
51
- VERSION=v0.6.1
50
+ # Deploying to test
51
+ VERSION=v0.6.3
52
52
  bash services/dictionary/pull_dict.sh "https://raw.githubusercontent.com/AustralianBioCommons/acdc-schema-json/refs/tags/${VERSION}/dictionary/prod_dict/acdc_schema.json"
53
53
  python3 services/dictionary/upload_dictionary.py "services/dictionary/schemas/acdc_schema_${VERSION}.json" s3://gen3schema-cad-uat-biocommons.org.au/cad.json
54
54
 
55
+
56
+ # Deploying to staging
57
+ VERSION=v0.7.7
58
+ bash services/dictionary/pull_dict.sh "https://raw.githubusercontent.com/AustralianBioCommons/acdc-schema-json/refs/tags/${VERSION}/dictionary/prod_dict/acdc_schema.json"
59
+ python3 services/dictionary/upload_dictionary.py "services/dictionary/schemas/acdc_schema_${VERSION}.json" s3://gen3schema-cad-staging-biocommons.org.au/cad.json
55
60
  ```
56
61
 
57
62
  ## Generating synthetic metadata
@@ -66,5 +71,9 @@ bash services/synthetic_data/generate_synth_metadata.sh --studies "AusDiab_Simul
66
71
  - Run this script to upload synthetic metadata to sheepdog
67
72
 
68
73
  ```python
69
- python3 services/synthetic_data/upload_synth_metadata_sheepdog.py
74
+ # to see argumments
75
+ python3 services/synthetic_data/upload_synth_metadata_sheepdog.py -h
76
+
77
+ # to upload metadata for version v0.6.3
78
+ python3 services/synthetic_data/upload_synth_metadata_sheepdog.py --version v0.6.3
70
79
  ```
{acdc_aws_etl_pipeline-0.3.5 → acdc_aws_etl_pipeline-0.3.7}/README.md RENAMED
@@ -21,11 +21,16 @@ e.g. to testing
21
21
  bash services/dictionary/pull_dict.sh <raw_dictionary_url>
22
22
  bash services/dictionary/upload_dictionary.py <local_dictionary_path> <s3_target_uri>
23
23
 
24
- # implementation
25
- VERSION=v0.6.1
24
+ # Deploying to test
25
+ VERSION=v0.6.3
26
26
  bash services/dictionary/pull_dict.sh "https://raw.githubusercontent.com/AustralianBioCommons/acdc-schema-json/refs/tags/${VERSION}/dictionary/prod_dict/acdc_schema.json"
27
27
  python3 services/dictionary/upload_dictionary.py "services/dictionary/schemas/acdc_schema_${VERSION}.json" s3://gen3schema-cad-uat-biocommons.org.au/cad.json
28
28
 
29
+
30
+ # Deploying to staging
31
+ VERSION=v0.7.7
32
+ bash services/dictionary/pull_dict.sh "https://raw.githubusercontent.com/AustralianBioCommons/acdc-schema-json/refs/tags/${VERSION}/dictionary/prod_dict/acdc_schema.json"
33
+ python3 services/dictionary/upload_dictionary.py "services/dictionary/schemas/acdc_schema_${VERSION}.json" s3://gen3schema-cad-staging-biocommons.org.au/cad.json
29
34
  ```
30
35
 
31
36
  ## Generating synthetic metadata
@@ -40,5 +45,9 @@ bash services/synthetic_data/generate_synth_metadata.sh --studies "AusDiab_Simul
40
45
  - Run this script to upload synthetic metadata to sheepdog
41
46
 
42
47
  ```python
43
- python3 services/synthetic_data/upload_synth_metadata_sheepdog.py
48
+ # to see argumments
49
+ python3 services/synthetic_data/upload_synth_metadata_sheepdog.py -h
50
+
51
+ # to upload metadata for version v0.6.3
52
+ python3 services/synthetic_data/upload_synth_metadata_sheepdog.py --version v0.6.3
44
53
  ```
{acdc_aws_etl_pipeline-0.3.5 → acdc_aws_etl_pipeline-0.3.7}/pyproject.toml RENAMED
@@ -1,6 +1,6 @@
1
1
  [tool.poetry]
2
2
  name = "acdc_aws_etl_pipeline"
3
- version = "0.3.5"
3
+ version = "0.3.7"
4
4
  description = "Tools for ACDC ETL pipeline"
5
5
  authors = ["JoshuaHarris391 <harjo391@gmail.com>"]
6
6
  readme = "README.md"
{acdc_aws_etl_pipeline-0.3.5 → acdc_aws_etl_pipeline-0.3.7}/src/acdc_aws_etl_pipeline/utils/release_writer.py RENAMED
@@ -148,6 +148,9 @@ def main():
148
148
 
149
149
  dbt_models = get_model_names(args.dbt_schema_path)
150
150
  athena_query = AthenaQuery(athena_config)
151
+
152
+ logger.info(f"Creating release table: {args.release_db}.{args.release_table}")
153
+ athena_query.create_release_table()
151
154
 
152
155
  logger.info(f"Processing DBT models from schema: {dbt_models}")
153
156