acdc_aws_etl_pipeline 0.7.4__tar.gz → 0.7.5__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (14) hide show
  1. {acdc_aws_etl_pipeline-0.7.4 → acdc_aws_etl_pipeline-0.7.5}/PKG-INFO +1 -1
  2. {acdc_aws_etl_pipeline-0.7.4 → acdc_aws_etl_pipeline-0.7.5}/pyproject.toml +1 -1
  3. {acdc_aws_etl_pipeline-0.7.4 → acdc_aws_etl_pipeline-0.7.5}/src/acdc_aws_etl_pipeline/validate/validate.py +21 -0
  4. {acdc_aws_etl_pipeline-0.7.4 → acdc_aws_etl_pipeline-0.7.5}/README.md +0 -0
  5. {acdc_aws_etl_pipeline-0.7.4 → acdc_aws_etl_pipeline-0.7.5}/src/acdc_aws_etl_pipeline/__init__.py +0 -0
  6. {acdc_aws_etl_pipeline-0.7.4 → acdc_aws_etl_pipeline-0.7.5}/src/acdc_aws_etl_pipeline/ingest/ingest.py +0 -0
  7. {acdc_aws_etl_pipeline-0.7.4 → acdc_aws_etl_pipeline-0.7.5}/src/acdc_aws_etl_pipeline/upload/__init__.py +0 -0
  8. {acdc_aws_etl_pipeline-0.7.4 → acdc_aws_etl_pipeline-0.7.5}/src/acdc_aws_etl_pipeline/upload/gen3datasubmitter.py +0 -0
  9. {acdc_aws_etl_pipeline-0.7.4 → acdc_aws_etl_pipeline-0.7.5}/src/acdc_aws_etl_pipeline/upload/metadata_deleter.py +0 -0
  10. {acdc_aws_etl_pipeline-0.7.4 → acdc_aws_etl_pipeline-0.7.5}/src/acdc_aws_etl_pipeline/upload/metadata_submitter.py +0 -0
  11. {acdc_aws_etl_pipeline-0.7.4 → acdc_aws_etl_pipeline-0.7.5}/src/acdc_aws_etl_pipeline/upload/upload_synthdata_s3.py +0 -0
  12. {acdc_aws_etl_pipeline-0.7.4 → acdc_aws_etl_pipeline-0.7.5}/src/acdc_aws_etl_pipeline/utils/athena_utils.py +0 -0
  13. {acdc_aws_etl_pipeline-0.7.4 → acdc_aws_etl_pipeline-0.7.5}/src/acdc_aws_etl_pipeline/utils/dbt_utils.py +0 -0
  14. {acdc_aws_etl_pipeline-0.7.4 → acdc_aws_etl_pipeline-0.7.5}/src/acdc_aws_etl_pipeline/utils/release_writer.py +0 -0
{acdc_aws_etl_pipeline-0.7.4 → acdc_aws_etl_pipeline-0.7.5}/PKG-INFO RENAMED
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: acdc_aws_etl_pipeline
3
- Version: 0.7.4
3
+ Version: 0.7.5
4
4
  Summary: Tools for ACDC ETL pipeline
5
5
  Author: JoshuaHarris391
6
6
  Author-email: harjo391@gmail.com
{acdc_aws_etl_pipeline-0.7.4 → acdc_aws_etl_pipeline-0.7.5}/pyproject.toml RENAMED
@@ -1,6 +1,6 @@
1
1
  [tool.poetry]
2
2
  name = "acdc_aws_etl_pipeline"
3
- version = "0.7.4"
3
+ version = "0.7.5"
4
4
  description = "Tools for ACDC ETL pipeline"
5
5
  authors = ["JoshuaHarris391 <harjo391@gmail.com>"]
6
6
  readme = "README.md"
{acdc_aws_etl_pipeline-0.7.4 → acdc_aws_etl_pipeline-0.7.5}/src/acdc_aws_etl_pipeline/validate/validate.py RENAMED
@@ -615,9 +615,30 @@ def validate_pipeline(
615
615
  logger.info("Validating %s", filename)
616
616
  results.extend(gen3_validator.validate.validate_list_dict(obj, resolver.schema_resolved))
617
617
 
618
+ logger.info("Getting Schema Version")
619
+ schema_resolved = resolver.schema_resolved
620
+ schema_version = resolver.get_schema_version(schema = schema_resolved)
621
+
618
622
  full_validation_results_df = pd.DataFrame(results)
619
623
  full_validation_results_df["validation_id"] = latest_validation_id
620
624
  full_validation_results_df["study_id"] = study_id
625
+ full_validation_results_df["schema_version"] = schema_version
626
+
627
+ full_validation_results_df = full_validation_results_df[[
628
+ "validation_id",
629
+ "index",
630
+ "node",
631
+ "study_id",
632
+ "validation_result",
633
+ "invalid_key",
634
+ "schema_path",
635
+ "validator",
636
+ "validator_value",
637
+ "validation_error",
638
+ "schema_version"
639
+ ]]
640
+
641
+
621
642
  logger.info("Validation completed (%s rows).", len(full_validation_results_df))
622
643
  except Exception as e:
623
644
  logger.exception("Validation failed.")