gen3-dataops-toolkit 3.7.0__tar.gz → 3.7.1__tar.gz

{gen3_dataops_toolkit-3.7.0 → gen3_dataops_toolkit-3.7.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: gen3-dataops-toolkit
-Version: 3.7.0
+Version: 3.7.1
 Summary: Gen3 DataOps toolkit (g3dt): operate SSM-published Gen3 data pipeline environments
 License: Apache-2.0
 Author: JoshuaHarris391

{gen3_dataops_toolkit-3.7.0 → gen3_dataops_toolkit-3.7.1}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "gen3-dataops-toolkit"
-version = "3.7.0"
+version = "3.7.1"
 description = "Gen3 DataOps toolkit (g3dt): operate SSM-published Gen3 data pipeline environments"
 authors = ["JoshuaHarris391 <harjo391@gmail.com>"]
 readme = "README.md"

{gen3_dataops_toolkit-3.7.0 → gen3_dataops_toolkit-3.7.1}/src/g3dt/services/synthetic_data/upload_synth_metadata_sheepdog.py

@@ -46,17 +46,19 @@ def submit_synthetic_metadata(
 
         logger.info("Found %d metadata files for submission.", len(file_list))
 
+        # upload_to_database=False: this flow only submits to Sheepdog, so the
+        # audit-table settings (database/table/athena_s3_output) are unused.
         submitter = MetadataSubmitter(
             metadata_file_list=file_list,
             api_key=api_key,
             project_id=project_id,
             data_import_order_path=data_import_order_path,
+            database=None,
+            table=None,
+            athena_s3_output=None,
             max_size_kb=max_submission_size_kb,
             aws_profile=aws_profile,
             upload_to_database=False,
-            dataset_root=None,
-            database=None,
-            table=None
         )
         submitter.submit_metadata()
         logger.info("Finished submitting for project_id=%s", project_id)
@@ -132,7 +134,15 @@ def main():
         "Looking for project subdirectories in base directory: %s",
         args.base_dir
     )
-    project_id_list = os.listdir(args.base_dir)
+    # Only real directories are projects: the batch dir also holds files like
+    # .g3dt-provenance.json (written by `g3dt synth generate`), which must not
+    # be treated as a project to submit.
+    project_id_list = sorted(
+        entry
+        for entry in os.listdir(args.base_dir)
+        if os.path.isdir(os.path.join(args.base_dir, entry))
+        and not entry.startswith(".")
+    )
     logger.info(
         "Found %d project subdirectories: %s",
         len(project_id_list),