acdc_aws_etl_pipeline 0.7.4__tar.gz → 0.7.6__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (14) hide show
  1. {acdc_aws_etl_pipeline-0.7.4 → acdc_aws_etl_pipeline-0.7.6}/PKG-INFO +1 -1
  2. {acdc_aws_etl_pipeline-0.7.4 → acdc_aws_etl_pipeline-0.7.6}/pyproject.toml +1 -1
  3. {acdc_aws_etl_pipeline-0.7.4 → acdc_aws_etl_pipeline-0.7.6}/src/acdc_aws_etl_pipeline/validate/validate.py +19 -0
  4. {acdc_aws_etl_pipeline-0.7.4 → acdc_aws_etl_pipeline-0.7.6}/README.md +0 -0
  5. {acdc_aws_etl_pipeline-0.7.4 → acdc_aws_etl_pipeline-0.7.6}/src/acdc_aws_etl_pipeline/__init__.py +0 -0
  6. {acdc_aws_etl_pipeline-0.7.4 → acdc_aws_etl_pipeline-0.7.6}/src/acdc_aws_etl_pipeline/ingest/ingest.py +0 -0
  7. {acdc_aws_etl_pipeline-0.7.4 → acdc_aws_etl_pipeline-0.7.6}/src/acdc_aws_etl_pipeline/upload/__init__.py +0 -0
  8. {acdc_aws_etl_pipeline-0.7.4 → acdc_aws_etl_pipeline-0.7.6}/src/acdc_aws_etl_pipeline/upload/gen3datasubmitter.py +0 -0
  9. {acdc_aws_etl_pipeline-0.7.4 → acdc_aws_etl_pipeline-0.7.6}/src/acdc_aws_etl_pipeline/upload/metadata_deleter.py +0 -0
  10. {acdc_aws_etl_pipeline-0.7.4 → acdc_aws_etl_pipeline-0.7.6}/src/acdc_aws_etl_pipeline/upload/metadata_submitter.py +0 -0
  11. {acdc_aws_etl_pipeline-0.7.4 → acdc_aws_etl_pipeline-0.7.6}/src/acdc_aws_etl_pipeline/upload/upload_synthdata_s3.py +0 -0
  12. {acdc_aws_etl_pipeline-0.7.4 → acdc_aws_etl_pipeline-0.7.6}/src/acdc_aws_etl_pipeline/utils/athena_utils.py +0 -0
  13. {acdc_aws_etl_pipeline-0.7.4 → acdc_aws_etl_pipeline-0.7.6}/src/acdc_aws_etl_pipeline/utils/dbt_utils.py +0 -0
  14. {acdc_aws_etl_pipeline-0.7.4 → acdc_aws_etl_pipeline-0.7.6}/src/acdc_aws_etl_pipeline/utils/release_writer.py +0 -0
{acdc_aws_etl_pipeline-0.7.4 → acdc_aws_etl_pipeline-0.7.6}/PKG-INFO RENAMED
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: acdc_aws_etl_pipeline
3
- Version: 0.7.4
3
+ Version: 0.7.6
4
4
  Summary: Tools for ACDC ETL pipeline
5
5
  Author: JoshuaHarris391
6
6
  Author-email: harjo391@gmail.com
{acdc_aws_etl_pipeline-0.7.4 → acdc_aws_etl_pipeline-0.7.6}/pyproject.toml RENAMED
@@ -1,6 +1,6 @@
1
1
  [tool.poetry]
2
2
  name = "acdc_aws_etl_pipeline"
3
- version = "0.7.4"
3
+ version = "0.7.6"
4
4
  description = "Tools for ACDC ETL pipeline"
5
5
  authors = ["JoshuaHarris391 <harjo391@gmail.com>"]
6
6
  readme = "README.md"
{acdc_aws_etl_pipeline-0.7.4 → acdc_aws_etl_pipeline-0.7.6}/src/acdc_aws_etl_pipeline/validate/validate.py RENAMED
@@ -615,9 +615,28 @@ def validate_pipeline(
615
615
  logger.info("Validating %s", filename)
616
616
  results.extend(gen3_validator.validate.validate_list_dict(obj, resolver.schema_resolved))
617
617
 
618
+ logger.info("Getting Schema Version")
619
+ schema_version = resolver.get_schema_version(schema = schema)
620
+
618
621
  full_validation_results_df = pd.DataFrame(results)
619
622
  full_validation_results_df["validation_id"] = latest_validation_id
620
623
  full_validation_results_df["study_id"] = study_id
624
+ full_validation_results_df["schema_version"] = schema_version
625
+
626
+ full_validation_results_df = full_validation_results_df[[
627
+ "validation_id",
628
+ "index",
629
+ "node",
630
+ "study_id",
631
+ "validation_result",
632
+ "invalid_key",
633
+ "schema_path",
634
+ "validator",
635
+ "validator_value",
636
+ "validation_error",
637
+ "schema_version"
638
+ ]]
639
+
621
640
  logger.info("Validation completed (%s rows).", len(full_validation_results_df))
622
641
  except Exception as e:
623
642
  logger.exception("Validation failed.")