gen3-dataops-toolkit 4.1.0__tar.gz → 4.2.0__tar.gz

{gen3_dataops_toolkit-4.1.0 → gen3_dataops_toolkit-4.2.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: gen3-dataops-toolkit
-Version: 4.1.0
+Version: 4.2.0
 Summary: Gen3 DataOps toolkit (g3dt): operate SSM-published Gen3 data pipeline environments
 License: Apache-2.0
 Author: JoshuaHarris391

{gen3_dataops_toolkit-4.1.0 → gen3_dataops_toolkit-4.2.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "gen3-dataops-toolkit"
-version = "4.1.0"
+version = "4.2.0"
 description = "Gen3 DataOps toolkit (g3dt): operate SSM-published Gen3 data pipeline environments"
 authors = ["JoshuaHarris391 <harjo391@gmail.com>"]
 readme = "README.md"
@@ -44,7 +44,7 @@ moto = ">=5.0"
 optional = true
 
 [tool.poetry.group.synth.dependencies]
-gen3-metadata-simulator = "^0.3.0"
+gen3-metadata-simulator = "^0.5.3"
 
 [build-system]
 requires = ["poetry-core>=2.0.0,<3.0.0"]

{gen3_dataops_toolkit-4.1.0 → gen3_dataops_toolkit-4.2.0}/src/g3dt/cli/delete_cmds.py

@@ -7,6 +7,13 @@ the bare names (a specific version like ``0.9.8``, resolved via an Athena GUID
 lookup, or ``all`` for every version). A bare study with no version anywhere
 is refused (exit 2).
 
+``--synthetic`` switches to registry-free mode for synthetic data: each
+``--studies`` name is the Gen3 project code itself (no SSM study registry —
+synthetic projects are never registered), bare names default to version
+``all``, and a specific version is matched verbatim against the records'
+``data_version`` property via GraphQL rather than Athena receipts (synthetic
+uploads write none).
+
 Every command confirms before acting. Production always requires typing the
 target id, even with ``--yes``. Deleting ALL versions always prompts, even with
 ``--yes``. Confirmation happens locally before any EC2 dispatch (SSM has no
@@ -61,12 +68,30 @@ def _normalise_version(raw: str, where: str) -> str:
     return match.group(1)
 
 
-def _parse_study_specs(studies: str, fallback, env: str):
+def _synthetic_version(raw: str) -> str:
+    """Canonicalise a synthetic version token: ``all`` (any case) or verbatim.
+
+    Synthetic versions are matched exactly against the records' ``data_version``
+    property, and the natural label there is the dictionary version WITH its
+    leading ``v`` (batch dirs are ``~/.g3dt/synth_metadata/v1.3.0/...``) — so
+    unlike ``_normalise_version`` nothing is stripped. A version that matches
+    no records is reported by the worker (skip + hint), not silently absorbed.
+    """
+    token = raw.strip()
+    return "all" if token.lower() == "all" else token
+
+
+def _parse_study_specs(studies: str, fallback, env: str, synthetic: bool = False):
     """Turn ``--studies`` into ``[(resolved_study_key, version), ...]``.
 
     Each comma-separated entry is ``name`` or ``name:version``. A bare name
     takes *fallback* (the ``--version`` default); *fallback* is ``None`` when
-    ``--version`` was not given, which makes a bare name a usage error.
+    ``--version`` was not given, which makes a bare name a usage error —
+    except with *synthetic*, where a bare name defaults to ``all`` (the whole
+    point of the flag is "wipe the synthetic project").
+
+    With *synthetic* the raw name IS the Gen3 project code: no study-registry
+    lookup, and version tokens pass through :func:`_synthetic_version`.
 
     Every entry is validated before anything is dispatched, so a typo in the
     last study cannot leave the earlier ones already deleted.
@@ -102,9 +127,15 @@ def _parse_study_specs(studies: str, fallback, env: str):
             raise typer.Exit(2)
 
         if sep:
-            version = _normalise_version(raw_version, f"for study '{name}'")
+            version = (
+                _synthetic_version(raw_version)
+                if synthetic
+                else _normalise_version(raw_version, f"for study '{name}'")
+            )
         elif fallback is not None:
             version = fallback
+        elif synthetic:
+            version = "all"
         else:
             typer.secho(
                 f"No version for study '{name}': add ':<version>' to it "
@@ -115,7 +146,7 @@ def _parse_study_specs(studies: str, fallback, env: str):
             )
             raise typer.Exit(2)
 
-        specs.append((study_of(name, env).key, version))
+        specs.append((name if synthetic else study_of(name, env).key, version))
 
     if not specs:
         typer.secho("--studies is empty.", fg=typer.colors.RED, err=True)
@@ -141,6 +172,20 @@ def metadata(
              "':version', e.g. 0.9.8, or 'all' for every version.",
     ),
     node: Optional[str] = typer.Option(None, "--node", help="Delete only this node type."),
+    synthetic: bool = typer.Option(
+        False,
+        "--synthetic",
+        help="Registry-free synthetic-data mode: each --studies name is the "
+             "Gen3 project id itself (no SSM study registry). Bare names "
+             "default to version 'all'; a specific version matches records' "
+             "data_version property verbatim.",
+    ),
+    program_id: Optional[str] = typer.Option(
+        None,
+        "--program-id",
+        help="Gen3 program for --synthetic (default: program1). "
+             "Invalid without --synthetic.",
+    ),
     yes: bool = typer.Option(
         False, "--yes", "-y", help="Skip the non-prod prompt (specific-version only)."
     ),
@@ -156,12 +201,24 @@ def metadata(
 
       g3dt delete metadata --studies "ausdiab:0.7.5,cdah:0.8.1" --env staging
       g3dt delete metadata --studies "ausdiab:all,cdah" --version 0.9.8 --env staging
+      g3dt delete metadata --studies "synthetic_dataset_1,synthetic_dataset_2" --env test --synthetic
     """
     env = resolve.active_env(env)
-    fallback = (
-        _normalise_version(version, "for --version") if version is not None else None
-    )
-    specs = _parse_study_specs(studies, fallback, env)
+    if program_id is not None and not synthetic:
+        typer.secho(
+            "--program-id is only valid with --synthetic (registered studies "
+            "carry their program in the study registry).",
+            fg=typer.colors.RED,
+            err=True,
+        )
+        raise typer.Exit(2)
+    if version is None:
+        fallback = None
+    elif synthetic:
+        fallback = _synthetic_version(version)
+    else:
+        fallback = _normalise_version(version, "for --version")
+    specs = _parse_study_specs(studies, fallback, env, synthetic=synthetic)
     versions = [v for _, v in specs]
 
     # The typed production confirmation stays the study keys alone: short
@@ -171,13 +228,17 @@ def metadata(
     uniform = len(set(versions)) == 1
     any_all = "all" in versions
 
+    prefix = "synthetic " if synthetic else ""
     if uniform and versions[0] == "all":
-        action = "deletion of ALL VERSIONS"
+        action = f"{prefix}deletion of ALL VERSIONS"
     elif uniform:
-        action = f"deletion of v{versions[0]}"
+        # Synthetic versions are verbatim data_version values (often already
+        # v-prefixed); Athena versions are canonical x.y.z, displayed with v.
+        shown = versions[0] if synthetic else f"v{versions[0]}"
+        action = f"{prefix}deletion of {shown}"
     else:
         plan = ", ".join(f"{key}:{v}" for key, v in specs)
-        action = f"deletion of per-study versions [{plan}]"
+        action = f"{prefix}deletion of per-study versions [{plan}]"
 
     # Deleting every version is the most destructive path: always prompt (pass
     # assume_yes=False so --yes can't bypass it; prod still types the target).
@@ -200,6 +261,11 @@ def metadata(
             ]
         if node:
             a += ["--node", node]
+        if synthetic:
+            # Program is always passed explicitly: the shell default makes it
+            # optional on the wire, but an explicit value keeps the contract
+            # visible in logs and SSM command history.
+            a += ["--synthetic", "--program-id", program_id or "program1"]
         return a
 
     def remote_cli(env_name):
@@ -212,6 +278,12 @@ def metadata(
         a.append("--yes")
         if node:
             a += ["--node", node]
+        if synthetic:
+            # Without this the remote re-entry would re-parse --studies
+            # against the study registry on the box and exit 2.
+            a.append("--synthetic")
+            if program_id is not None:
+                a += ["--program-id", program_id]
         return a
 
     dispatch.run_or_dispatch(

{gen3_dataops_toolkit-4.1.0 → gen3_dataops_toolkit-4.2.0}/src/g3dt/cli/synth.py

@@ -370,6 +370,13 @@ def generate(
         None, "--version", "-v",
         help="Version label for output dir (default: env dictionary_version).",
     ),
+    data_version: Optional[str] = typer.Option(
+        None, "--data-version",
+        help="Stamp every generated record's data_version property with this "
+             "value (requires the dictionary to declare data_version). Makes "
+             "the batch deletable later with "
+             "'delete metadata --synthetic --version <value>'.",
+    ),
 ) -> None:
     """Generate synthetic metadata locally with gen3-metadata-simulator.
 
@@ -431,6 +438,8 @@ def generate(
         args += ["--num-records", num_records]
     if seed is not None:
         args += ["--seed", str(seed)]
+    if data_version:
+        args += ["--data-version", data_version]
     env_vars = script_env(e, ver)
     if effective_provider is Provider.llm:
         env_vars.update(

{gen3_dataops_toolkit-4.1.0 → gen3_dataops_toolkit-4.2.0}/src/g3dt/config.py

@@ -40,6 +40,7 @@ import re
 from dataclasses import dataclass
 from pathlib import Path
 from typing import Dict, List, Optional, Tuple
+from urllib.parse import unquote, urlsplit
 
 import yaml
 
@@ -393,6 +394,8 @@ class EnvConfig:
     dictionary_version: str
     aws_profile: Optional[str]
     aws_secret_name: str
+    # Canonical scheme-less "bucket/key" (normalize_s3_location at resolve time;
+    # callers prepend s3:// themselves).
     schema_s3_uri: str
     domain: str
     app_name: str
@@ -421,6 +424,74 @@ def _app_or_default(rc, leaf: str, default: str) -> str:
     return (rc.get(f"app/{leaf}") or "").strip() or default
 
 
+#: The two S3 endpoint URL host styles. Path-style must be tried first: a bare
+#: ``s3.<region>.amazonaws.com`` host would otherwise match the virtual-hosted
+#: pattern with bucket "s3". The virtual-hosted bucket group is greedy so
+#: dotted bucket names keep their dots.
+_S3_PATH_STYLE_HOST = re.compile(r"^s3([.-][a-z0-9-]+)*\.amazonaws\.com$")
+_S3_VIRTUAL_HOST = re.compile(r"^(?P<bucket>.+)\.s3([.-][a-z0-9-]+)*\.amazonaws\.com$")
+
+#: Remediation appended to normalize_s3_location errors for the SSM app fact.
+_SCHEMA_S3_URI_HINT = (
+    "Fix gen3.schemaS3Uri in the CDK config (gen3-aws-data-pipeline) and "
+    "re-run `cdk deploy`."
+)
+
+
+def normalize_s3_location(
+    value: str, *, param: str = "app/schema_s3_uri", hint: Optional[str] = None
+) -> str:
+    """Canonicalize an operator-supplied S3 location to scheme-less ``bucket[/key]``.
+
+    Accepted forms: ``bucket/key`` (canonical), ``s3://bucket/key`` — a repeated
+    scheme (``s3://s3://...``) is tolerated, since that is exactly what a
+    scheme-carrying SSM value produces once callers prepend ``s3://`` — and the
+    two S3 endpoint URL styles (``https://<bucket>.s3.<region>.amazonaws.com/<key>``,
+    ``https://s3.<region>.amazonaws.com/<bucket>/<key>``); query strings on URL
+    forms (presigned links, ``?versionId=``) are dropped.
+
+    An object key is NOT required — ``resolve_env`` gates every command, so
+    shape is enforced at point of use (upload). The key is preserved verbatim,
+    trailing slash included, except for percent-decoding of URL forms.
+
+    :raises ConfigError: empty value, unrecognizable http(s) host (e.g. an AWS
+        console page URL), or a bucket segment containing ``:`` (the
+        ``s3:/bucket`` one-slash typo).
+    """
+    original = value.strip()
+
+    def _bad() -> ConfigError:
+        msg = (
+            f"Cannot interpret {param} value '{original}' as an S3 location. "
+            "Accepted forms: bucket/key, s3://bucket/key, or an S3 endpoint "
+            "URL (https://<bucket>.s3.<region>.amazonaws.com/<key> or "
+            "https://s3.<region>.amazonaws.com/<bucket>/<key>)."
+        )
+        return ConfigError(msg + (f" {hint}" if hint else ""))
+
+    v = original
+    if not v:
+        raise _bad()
+    while v.lower().startswith("s3://"):
+        v = v[len("s3://") :]
+    if v.lower().startswith(("https://", "http://")):
+        parts = urlsplit(v)
+        host = parts.hostname or ""
+        path = unquote(parts.path)
+        if _S3_PATH_STYLE_HOST.match(host):
+            v = path.lstrip("/")
+        else:
+            m = _S3_VIRTUAL_HOST.match(host)
+            if not m:
+                raise _bad()
+            key = path.lstrip("/")
+            v = f"{m.group('bucket')}/{key}" if key else m.group("bucket")
+    bucket = v.split("/", 1)[0]
+    if not bucket or ":" in bucket:
+        raise _bad()
+    return v
+
+
 def resolve_env(env: str, project: Optional[str] = None) -> EnvConfig:
     """Resolve one environment: app INPUT facts + CDK OUTPUT names, from SSM.
 
@@ -454,7 +525,9 @@ def resolve_env(env: str, project: Optional[str] = None) -> EnvConfig:
         dictionary_version=rc.app("dictionary_version"),
         aws_profile=profile,
         aws_secret_name=rc.app("aws_secret_name"),
-        schema_s3_uri=rc.app("schema_s3_uri"),
+        schema_s3_uri=normalize_s3_location(
+            rc.app("schema_s3_uri"), hint=_SCHEMA_S3_URI_HINT
+        ),
         domain=rc.app("domain"),
         app_name=rc.app("app_name"),
         namespace=rc.app("namespace"),

{gen3_dataops_toolkit-4.1.0 → gen3_dataops_toolkit-4.2.0}/src/g3dt/services/delete/delete_all_metadata_for_project.py

@@ -81,6 +81,21 @@ def main():
         required=True,
         help="Environment to use (selects AWS secret, profile, etc.)",
     )
+    parser.add_argument(
+        "--synthetic",
+        action="store_true",
+        default=False,
+        help=(
+            "Registry-free mode for synthetic data: --study is used directly "
+            "as the Gen3 project code (no SSM study registry lookup), with "
+            "the program from --program-id."
+        ),
+    )
+    parser.add_argument(
+        "--program-id",
+        default="program1",
+        help="Gen3 program for --synthetic mode.",
+    )
     parser.add_argument(
         "--import-order",
         default="DataImportOrder.txt",
@@ -106,16 +121,21 @@ def main():
 
     # Env facts from SSM; the study registry from the marker or
     # SSM /{project}/{env}/studies/* (legacy studies.yaml fallback until 5.0).
+    # Synthetic projects are not registered studies: --synthetic skips the
+    # registry and takes --study as the Gen3 project code itself.
     try:
         env_cfg = g3dt_config.resolve_env(args.env)
-        study_cfg = g3dt_config.resolve_study(args.study, args.env)
+        if args.synthetic:
+            project_id = args.study
+            program_id = args.program_id
+        else:
+            study_cfg = g3dt_config.resolve_study(args.study, args.env)
+            project_id = study_cfg.project_id
+            program_id = study_cfg.program_id
     except g3dt_config.ConfigError as exc:
         logger.error(str(exc))
         sys.exit(1)
 
-    project_id = study_cfg.project_id
-    program_id = study_cfg.program_id
-
     aws_secret_name = env_cfg.aws_secret_name
     aws_profile = env_cfg.aws_profile
     aws_region = env_cfg.region

{gen3_dataops_toolkit-4.1.0 → gen3_dataops_toolkit-4.2.0}/src/g3dt/services/delete/delete_metadata.sh

@@ -10,21 +10,26 @@ SKIP_EXIT_CODE=3
 
 usage() {
     cat <<EOF
-Usage: $(basename "$0") --studies <name[:version|all],...> --env <environment> [--version <version|all>] [--node <node>]
+Usage: $(basename "$0") --studies <name[:version|all],...> --env <environment> [--version <version|all>] [--node <node>] [--synthetic] [--program-id <program>]
 
 Delete metadata for each study sequentially, in a single job.
 
 Arguments:
-  --studies   Comma-separated study config keys, each optionally qualified with
-              its own version (e.g. ausdiab_staging:0.7.5,cdah_staging:0.8.1,
-              or bare ausdiab_staging to take the --version default)
-  --env       Environment string passed to the Python worker (e.g. staging_ec2)
-  --version   Default version for bare --studies entries (e.g. 0.9.8), or 'all'
-  --node      (optional) Restrict deletion to a single node type
+  --studies    Comma-separated study config keys, each optionally qualified with
+               its own version (e.g. ausdiab_staging:0.7.5,cdah_staging:0.8.1,
+               or bare ausdiab_staging to take the --version default)
+  --env        Environment string passed to the Python worker (e.g. staging_ec2)
+  --version    Default version for bare --studies entries (e.g. 0.9.8), or 'all'
+  --node       (optional) Restrict deletion to a single node type
+  --synthetic  (optional) Registry-free synthetic-data mode: each study name is
+               the Gen3 project code itself (no SSM study registry lookup)
+  --program-id (optional) Gen3 program for --synthetic mode (default program1)
 
 Behaviour:
   * version 'all'          -> delete_all_metadata_for_project.py (deletes whole nodes)
   * version <x.y.z>        -> delete_metadata_by_guid.py (Athena GUID lookup for that version)
+  * --synthetic + 'all'    -> delete_all_metadata_for_project.py --synthetic
+  * --synthetic + version  -> delete_synth_metadata_by_version.py (GraphQL data_version filter)
 
   A study that exists but has no data at the requested version is skipped and the
   loop continues. Only genuine errors (Gen3/AWS failures) count as failures.
@@ -45,6 +50,8 @@ STUDIES=""
 ENV=""
 VERSION=""
 NODE=""
+SYNTHETIC=0
+PROGRAM_ID="program1"
 
 while [[ $# -gt 0 ]]; do
     case "$1" in
@@ -64,6 +71,14 @@ while [[ $# -gt 0 ]]; do
             NODE="$2"
             shift 2
             ;;
+        --synthetic)
+            SYNTHETIC=1
+            shift
+            ;;
+        --program-id)
+            PROGRAM_ID="$2"
+            shift 2
+            ;;
         *)
             echo "ERROR: Unknown argument: $1"
             usage
@@ -122,6 +137,7 @@ echo "Environment : ${ENV}"
 echo "Studies     : ${STUDIES}"
 echo "Version     : ${VERSION:-(per study, from --studies)}"
 [[ -n "$NODE" ]] && echo "Node        : ${NODE}"
+[[ $SYNTHETIC -eq 1 ]] && echo "Synthetic   : yes (program: ${PROGRAM_ID})"
 echo "Failure log : ${FAILED_LOG}"
 echo "============================================"
 echo ""
@@ -140,6 +156,12 @@ for i in "${!STUDY_NAMES[@]}"; do
     if [[ "$study_version_lc" == "all" ]]; then
         CMD=("${G3DT_PYTHON:-python3}" "${SCRIPT_DIR}/delete_all_metadata_for_project.py"
              --study "$study" --env "$ENV")
+        [[ $SYNTHETIC -eq 1 ]] && CMD+=(--synthetic --program-id "$PROGRAM_ID")
+        [[ -n "$NODE" ]] && CMD+=(--node "$NODE")
+    elif [[ $SYNTHETIC -eq 1 ]]; then
+        CMD=("${G3DT_PYTHON:-python3}" "${SCRIPT_DIR}/delete_synth_metadata_by_version.py"
+             --study "$study" --env "$ENV" --version "$study_version"
+             --program-id "$PROGRAM_ID" --skip-if-empty)
         [[ -n "$NODE" ]] && CMD+=(--node "$NODE")
     else
         CMD=("${G3DT_PYTHON:-python3}" "${SCRIPT_DIR}/delete_metadata_by_guid.py"

gen3_dataops_toolkit-4.2.0/src/g3dt/services/delete/delete_synth_metadata_by_version.py

@@ -0,0 +1,334 @@
+"""Registry-free, version-filtered deletion for synthetic metadata.
+
+Synthetic uploads write no Athena receipts (upload_synth_metadata_sheepdog
+submits with upload_to_database=False), so delete_metadata_by_guid.py's
+receipt lookup can never find them. This worker instead filters records
+server-side: per node (reverse DataImportOrder), it queries sheepdog GraphQL
+for records whose `data_version` property equals --version and deletes them.
+The --study value is used directly as the Gen3 project code — no SSM study
+registry involvement.
+
+A node whose schema does not declare `data_version` makes the GraphQL query
+error; that node is warned about and skipped. If nothing is deleted anywhere,
+the run exits with the skip code (3, with --skip-if-empty) plus a hint, so the
+bulk loop counts it as skipped rather than failed.
+"""
+import sys
+import time
+import logging
+import argparse
+
+from gen3.submission import Gen3SubmissionQueryError
+
+from g3dt.upload.metadata_submitter import (
+    create_boto3_session,
+    get_gen3_api_key_aws_secret,
+    create_gen3_submission_class,
+)
+from g3dt.upload.metadata_deleter import delete_node_records_by_property
+
+# ANSI colour codes (matching metadata_submitter.py style)
+GREEN = "\033[92m"
+RED = "\033[91m"
+YELLOW = "\033[93m"
+BLUE = "\033[94m"
+RESET = "\033[0m"
+
+EXCLUDE_NODES = [
+    "program",
+    "project",
+    "acknowledgement",
+    "publication",
+]
+
+# Exit code that signals "project exists but has no data at this version —
+# skipped". The bulk caller (services/delete/delete_metadata.sh) treats this as
+# a skip-and-continue rather than a failure. Only emitted with --skip-if-empty.
+SKIP_EXIT_CODE = 3
+
+
+def setup_logger():
+    logger = logging.getLogger()
+    logger.setLevel(logging.INFO)
+    if not logger.handlers:
+        handler = logging.StreamHandler(sys.stdout)
+        formatter = logging.Formatter(
+            '%(asctime)s - %(name)s - %(levelname)s - %(message)s'
+        )
+        handler.setFormatter(formatter)
+        logger.addHandler(handler)
+    return logger
+
+
+# Shared config resolution (SSM-backed) — see src/g3dt/config.py
+from g3dt import config as g3dt_config  # noqa: E402
+
+
+def load_import_order(import_order_path, exclude_nodes=None):
+    """
+    Reads the DataImportOrder.txt file and returns the node list
+    in deletion order (reversed, with excluded nodes removed).
+    """
+    if exclude_nodes is None:
+        exclude_nodes = EXCLUDE_NODES
+    with open(import_order_path, 'r', encoding='utf-8') as f:
+        nodes = [line.strip() for line in f if line.strip()]
+    nodes = [n for n in nodes if n not in exclude_nodes]
+    nodes.reverse()
+    return nodes
+
+
+def main():
+    logger = setup_logger()
+
+    parser = argparse.ArgumentParser(
+        description=(
+            "Delete synthetic Gen3 metadata records at one data version. "
+            "Filters each node (in reverse DataImportOrder) via sheepdog "
+            "GraphQL on the records' data_version property and deletes the "
+            "matches. --study is the Gen3 project code itself — no study "
+            "registry lookup."
+        ),
+        formatter_class=argparse.ArgumentDefaultsHelpFormatter,
+    )
+    parser.add_argument(
+        "--study",
+        required=True,
+        help="Gen3 project code (e.g. synthetic_dataset_1)",
+    )
+    parser.add_argument(
+        "--env",
+        required=True,
+        help="Environment to use (selects AWS secret, profile, etc.)",
+    )
+    parser.add_argument(
+        "--version",
+        required=True,
+        help=(
+            "Value the records' data_version property must equal, matched "
+            "verbatim (e.g. v1.3.0)"
+        ),
+    )
+    parser.add_argument(
+        "--program-id",
+        default="program1",
+        help="Gen3 program the project lives under.",
+    )
+    parser.add_argument(
+        "--import-order",
+        default="DataImportOrder.txt",
+        help="Path to DataImportOrder.txt",
+    )
+    parser.add_argument(
+        "--node",
+        default=None,
+        help=(
+            "Delete only a specific node (e.g. 'subject'). "
+            "If omitted, all nodes are processed in reverse "
+            "DataImportOrder."
+        ),
+    )
+    parser.add_argument(
+        "--prompt",
+        action="store_true",
+        default=False,
+        help="Prompt for confirmation before deleting.",
+    )
+    parser.add_argument(
+        "--batch-size",
+        type=int,
+        default=40,
+        help="Number of UUIDs per DELETE request.",
+    )
+    parser.add_argument(
+        "--batch-delay",
+        type=float,
+        default=0.5,
+        help="Seconds to pause between batches.",
+    )
+    parser.add_argument(
+        "--verbose",
+        action="store_true",
+        default=False,
+        help="Log full API response JSON for each request.",
+    )
+    parser.add_argument(
+        "--delay",
+        type=float,
+        default=1.0,
+        help="Seconds to wait between nodes.",
+    )
+    parser.add_argument(
+        "--skip-if-empty",
+        action="store_true",
+        default=False,
+        help=(
+            "If no records match the given version, exit with the skip "
+            "code (3) instead of 0. Used by the bulk delete loop to "
+            "skip-and-continue rather than treat it as a failure."
+        ),
+    )
+
+    args = parser.parse_args()
+
+    if args.verbose:
+        logger.setLevel(logging.DEBUG)
+
+    # Env facts from SSM. Deliberately NO resolve_study: synthetic projects
+    # are not in the study registry — the --study value IS the project code.
+    try:
+        env_cfg = g3dt_config.resolve_env(args.env)
+    except g3dt_config.ConfigError as exc:
+        logger.error(str(exc))
+        sys.exit(1)
+
+    project_id = args.study
+    program_id = args.program_id
+
+    aws_secret_name = env_cfg.aws_secret_name
+    aws_profile = env_cfg.aws_profile
+    aws_region = env_cfg.region
+
+    compound_project_id = f"{program_id}-{project_id}"
+
+    logger.info(
+        "Synthetic | Env: %s | Project: %s | Version: %s",
+        args.env,
+        compound_project_id,
+        args.version,
+    )
+
+    # AWS and Gen3 authentication
+    session = create_boto3_session(aws_profile=aws_profile)
+    api_key = get_gen3_api_key_aws_secret(
+        secret_name=aws_secret_name,
+        region_name=aws_region,
+        session=session,
+    )
+    sub = create_gen3_submission_class(api_key)
+
+    # Determine node list
+    if args.node:
+        nodes_to_delete = [args.node]
+        logger.info(
+            "%s[SINGLE NODE]%s Targeting node: %s",
+            BLUE, RESET, args.node,
+        )
+    else:
+        nodes_to_delete = load_import_order(args.import_order)
+        logger.info(
+            "Loaded %s nodes from %s (deletion order, "
+            "excluding %s)",
+            len(nodes_to_delete),
+            args.import_order,
+            EXCLUDE_NODES,
+        )
+
+    if args.prompt:
+        confirm = input(
+            f"Proceed with deletion for project "
+            f"{compound_project_id}, data_version {args.version}, "
+            f"{len(nodes_to_delete)} node(s)? (yes/no): "
+        ).strip().lower()
+        if confirm != "yes":
+            logger.info("Deletion cancelled by user.")
+            return
+
+    # Process each node
+    total_deleted = 0
+    total_skipped = 0
+    failed_nodes = []
+    total_nodes = len(nodes_to_delete)
+
+    for idx, node in enumerate(nodes_to_delete, start=1):
+        logger.info(
+            "%s[Node %d/%d]%s | Project: %-10s | "
+            "Node: %-25s | Querying...",
+            BLUE, idx, total_nodes, RESET,
+            compound_project_id, node,
+        )
+
+        try:
+            deleted = delete_node_records_by_property(
+                gen3_submission=sub,
+                program_id=program_id,
+                project_id=project_id,
+                node=node,
+                property_name="data_version",
+                property_value=args.version,
+                batch_size=args.batch_size,
+                batch_delay=args.batch_delay,
+                verbose=args.verbose,
+            )
+        except Gen3SubmissionQueryError as exc:
+            logger.warning(
+                "%s[SKIP]%s    | Node: %-25s | GraphQL rejected the "
+                "data_version filter (property not in this node's "
+                "schema?): %s",
+                YELLOW, RESET, node, exc,
+            )
+            total_skipped += 1
+            continue
+        except Exception as exc:
+            logger.error(
+                "%s[FAILED]%s  | Node: %-25s | %s",
+                RED, RESET, node, exc,
+            )
+            failed_nodes.append(node)
+            continue
+
+        if deleted == 0:
+            logger.info(
+                "%s[SKIP]%s    | Project: %-10s | "
+                "Node: %-25s | No records found",
+                YELLOW, RESET,
+                compound_project_id, node,
+            )
+            total_skipped += 1
+            continue
+
+        logger.info(
+            "%s[SUCCESS]%s | Project: %-10s | "
+            "Node: %-25s | Deleted: %s",
+            GREEN, RESET,
+            compound_project_id, node, deleted,
+        )
+        total_deleted += deleted
+
+        if idx < total_nodes:
+            time.sleep(args.delay)
+
+    logger.info(
+        "=========================================="
+    )
+    logger.info(
+        "Deletion complete. Total deleted: %s | "
+        "Nodes skipped: %s | Nodes failed: %s",
+        total_deleted,
+        total_skipped,
+        len(failed_nodes),
+    )
+
+    if failed_nodes:
+        logger.error(
+            "Failed nodes: %s", ", ".join(failed_nodes)
+        )
+        sys.exit(1)
+
+    # No records matched the requested version on any node. Usually the data
+    # was generated without the property (see `g3dt synth generate
+    # --data-version`), so surface the actionable hint rather than a silent
+    # "0 deleted".
+    if total_deleted == 0:
+        logger.warning(
+            "Data version '%s' not found for study '%s'. Ensure each data node "
+            "has a `data_version` property for versioning to work.",
+            args.version,
+            args.study,
+        )
+        if args.skip_if_empty:
+            sys.exit(SKIP_EXIT_CODE)
+
+
+if __name__ == "__main__":
+    main()

{gen3_dataops_toolkit-4.1.0 → gen3_dataops_toolkit-4.2.0}/src/g3dt/services/dictionary/upload_dictionary.py

@@ -4,6 +4,8 @@ import logging
 import sys
 from botocore.exceptions import ClientError
 
+from g3dt.config import ConfigError, normalize_s3_location
+
 logging.basicConfig(
     level=logging.INFO,
     format="%(asctime)s %(levelname)s %(message)s",
@@ -49,23 +51,26 @@ def upload_dict_to_s3(dict_file_path, s3_target_uri, dict_version, profile_name=
 
     Args:
         dict_file_path (str): Local dictionary path.
-        s3_target_uri (str): URI like s3://bucket/key.
+        s3_target_uri (str): Target like s3://bucket/key. Forgiving: bare
+            bucket/key, a doubled s3:// scheme, and S3 endpoint https URLs
+            are all accepted (see g3dt.config.normalize_s3_location).
         dict_version (str): Dictionary version string.
         profile_name (str, optional): AWS profile name.
 
     Returns:
         bool: True on success, False otherwise.
     """
-    if not s3_target_uri.startswith("s3://"):
-        logger.error(f"Invalid S3 URI: {s3_target_uri}")
+    try:
+        location = normalize_s3_location(s3_target_uri, param="s3_uri argument")
+    except ConfigError as e:
+        logger.error(str(e))
+        return False
+    if "/" not in location:
+        logger.error(f"S3 location missing an object key: {s3_target_uri}")
         return False
 
     try:
-        s3_path = s3_target_uri[len("s3://") :]
-        if "/" not in s3_path:
-            logger.error(f"S3 URI missing key: {s3_target_uri}")
-            return False
-        bucket, key = s3_path.split("/", 1)
+        bucket, key = location.split("/", 1)
         # Set S3 metadata key to "version" instead of "dict_version"
         extra_args = {"Metadata": {"version": dict_version or "unknown"}}
         s3_client = get_s3_client(profile_name)

{gen3_dataops_toolkit-4.1.0 → gen3_dataops_toolkit-4.2.0}/src/g3dt/services/synthetic_data/generate_synth_metadata.sh

@@ -34,6 +34,9 @@ Options:
                          'random' needs no key; 'llm' uses \$G3DT_LLM_PROVIDER /
                          \$G3DT_LLM_MODEL / \$LLM_API_KEY_FILE (set by g3dt).
   --seed N               RNG seed for reproducible output.
+  --data-version V       Stamp every record's data_version property with V
+                         (requires the dictionary to declare data_version;
+                         enables versioned deletion later).
   --output-root DIR      Root output dir. Default: ${DEFAULT_OUTPUT_ROOT}
   -h, --help             Show this help and exit.
 
@@ -57,6 +60,7 @@ STUDIES="${DEFAULT_STUDIES}"
 NUM_RECORDS="${DEFAULT_NUM_RECORDS}"
 PROVIDER="${DEFAULT_PROVIDER}"
 SEED=""
+DATA_VERSION=""
 OUTPUT_ROOT="${DEFAULT_OUTPUT_ROOT}"
 
 while [[ $# -gt 0 ]]; do
@@ -67,6 +71,7 @@ while [[ $# -gt 0 ]]; do
         --num-records)  NUM_RECORDS="$2"; shift 2 ;;
         --provider)     PROVIDER="$2"; shift 2 ;;
         --seed)         SEED="$2"; shift 2 ;;
+        --data-version) DATA_VERSION="$2"; shift 2 ;;
         --output-root)  OUTPUT_ROOT="$2"; shift 2 ;;
         -h|--help)      usage; exit 0 ;;
         *) echo "Unknown argument: $1" >&2; usage; exit 1 ;;
@@ -122,6 +127,9 @@ for i in "${!STUDY_ARRAY[@]}"; do
          --num-records "$N"
          --provider "$PROVIDER")
     [[ -n "$SEED" ]] && CMD+=(--seed "$SEED")
+    # --set pins a declared data property to a constant on every record; the
+    # simulator errors before generating if no node declares data_version.
+    [[ -n "$DATA_VERSION" ]] && CMD+=(--set "data_version=${DATA_VERSION}")
     if [[ "$PROVIDER" == "llm" ]]; then
         # Vendor/model resolved by g3dt (flags > SSM > default) and forwarded
         # as simulator flags, because the simulator's own precedence puts

{gen3_dataops_toolkit-4.1.0 → gen3_dataops_toolkit-4.2.0}/src/g3dt/upload/metadata_deleter.py

@@ -202,6 +202,84 @@ def delete_records_by_guid(
         )
 
 
+def delete_node_records_by_property(
+    gen3_submission: Gen3Submission,
+    program_id: str,
+    project_id: str,
+    node: str,
+    property_name: str,
+    property_value: str,
+    page_size: int = 100,
+    batch_size: int = 40,
+    batch_delay: float = 0.5,
+    verbose: bool = False,
+) -> int:
+    """
+    Deletes every record of one node whose ``property_name`` equals
+    ``property_value``, via the sheepdog GraphQL endpoint — no Athena
+    receipts required.
+
+    Pages with query -> delete -> re-query until the query returns no
+    records (the same pattern the SDK's own delete_nodes uses), so a
+    partially failed batch is simply retried on the next round. If two
+    consecutive rounds return the same first id, no progress is being
+    made (e.g. every deletion is failing) and a RuntimeError is raised.
+
+    GraphQL errors from the query (e.g. the node's schema does not
+    declare the property) propagate to the caller as
+    Gen3SubmissionQueryError — per-node tolerance is the caller's
+    decision, not this function's.
+
+    Args:
+        gen3_submission (Gen3Submission): An authenticated
+            Gen3Submission instance.
+        program_id (str): The Gen3 program name.
+        project_id (str): The Gen3 project name (bare, not compound).
+        node (str): The node type to delete records from.
+        property_name (str): Node property to filter on
+            (e.g. "data_version").
+        property_value (str): Exact value the property must equal.
+        page_size (int, optional): Records fetched per query round.
+        batch_size (int, optional): Passed to delete_records_by_guid.
+        batch_delay (float, optional): Passed to delete_records_by_guid.
+        verbose (bool, optional): Passed to delete_records_by_guid.
+
+    Returns:
+        int: Number of distinct records deleted.
+    """
+    compound_project_id = f"{program_id}-{project_id}"
+    seen = set()
+    first_uuid = ""
+    while True:
+        query_string = (
+            f'{{ {node} (first: {page_size}, '
+            f'project_id: "{compound_project_id}", '
+            f'{property_name}: "{property_value}") {{ id }} }}'
+        )
+        res = gen3_submission.query(query_string)
+        uuids = [x["id"] for x in res["data"][node]]
+        if not uuids:
+            break
+        if first_uuid == uuids[0]:
+            raise RuntimeError(
+                f"No progress deleting '{node}' records with "
+                f"{property_name}='{property_value}' — record "
+                f"{uuids[0]} keeps reappearing (deletions failing?)."
+            )
+        first_uuid = uuids[0]
+        seen.update(uuids)
+        delete_records_by_guid(
+            gen3_submission,
+            program_id,
+            project_id,
+            uuids,
+            batch_size=batch_size,
+            batch_delay=batch_delay,
+            verbose=verbose,
+        )
+    return len(seen)
+
+
 def delete_project_metadata(
     gen3_submission: Gen3Submission,
     program_id: str,