gen3-dataops-toolkit 3.7.0__tar.gz → 3.7.2__tar.gz

{gen3_dataops_toolkit-3.7.0 → gen3_dataops_toolkit-3.7.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: gen3-dataops-toolkit
-Version: 3.7.0
+Version: 3.7.2
 Summary: Gen3 DataOps toolkit (g3dt): operate SSM-published Gen3 data pipeline environments
 License: Apache-2.0
 Author: JoshuaHarris391

{gen3_dataops_toolkit-3.7.0 → gen3_dataops_toolkit-3.7.2}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "gen3-dataops-toolkit"
-version = "3.7.0"
+version = "3.7.2"
 description = "Gen3 DataOps toolkit (g3dt): operate SSM-published Gen3 data pipeline environments"
 authors = ["JoshuaHarris391 <harjo391@gmail.com>"]
 readme = "README.md"

{gen3_dataops_toolkit-3.7.0 → gen3_dataops_toolkit-3.7.2}/src/g3dt/services/synthetic_data/upload_synth_metadata_sheepdog.py

@@ -13,6 +13,19 @@ DEFAULT_REGION = "ap-southeast-2"
 DEFAULT_PROFILE = None  # ambient credentials unless --aws-profile is given
 DEFAULT_SUBMISSION_SIZE_KB = 50
 
+# Nodes never submitted from a synthetic batch. The first four are the
+# structural nodes MetadataSubmitter always excludes; core_metadata_collection
+# is excluded here too because the simulator fills its date-time fields with
+# random words, which Sheepdog rejects (400) — and synthetic data has no use
+# for it anyway.
+EXCLUDE_NODES = [
+    "program",
+    "project",
+    "acknowledgement",
+    "publication",
+    "core_metadata_collection",
+]
+
 def submit_synthetic_metadata(
     base_dir,
     project_id,
@@ -46,17 +59,20 @@ def submit_synthetic_metadata(
 
         logger.info("Found %d metadata files for submission.", len(file_list))
 
+        # upload_to_database=False: this flow only submits to Sheepdog, so the
+        # audit-table settings (database/table/athena_s3_output) are unused.
         submitter = MetadataSubmitter(
             metadata_file_list=file_list,
             api_key=api_key,
             project_id=project_id,
             data_import_order_path=data_import_order_path,
+            database=None,
+            table=None,
+            athena_s3_output=None,
             max_size_kb=max_submission_size_kb,
+            exclude_nodes=EXCLUDE_NODES,
             aws_profile=aws_profile,
             upload_to_database=False,
-            dataset_root=None,
-            database=None,
-            table=None
         )
         submitter.submit_metadata()
         logger.info("Finished submitting for project_id=%s", project_id)
@@ -132,7 +148,15 @@ def main():
         "Looking for project subdirectories in base directory: %s",
         args.base_dir
     )
-    project_id_list = os.listdir(args.base_dir)
+    # Only real directories are projects: the batch dir also holds files like
+    # .g3dt-provenance.json (written by `g3dt synth generate`), which must not
+    # be treated as a project to submit.
+    project_id_list = sorted(
+        entry
+        for entry in os.listdir(args.base_dir)
+        if os.path.isdir(os.path.join(args.base_dir, entry))
+        and not entry.startswith(".")
+    )
     logger.info(
         "Found %d project subdirectories: %s",
         len(project_id_list),