gen3-dataops-toolkit 4.2.0__tar.gz → 4.3.0__tar.gz

{gen3_dataops_toolkit-4.2.0 → gen3_dataops_toolkit-4.3.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: gen3-dataops-toolkit
-Version: 4.2.0
+Version: 4.3.0
 Summary: Gen3 DataOps toolkit (g3dt): operate SSM-published Gen3 data pipeline environments
 License: Apache-2.0
 Author: JoshuaHarris391
@@ -194,6 +194,20 @@ version until the CDK config catches up —
 `g3dt config diff --env <env> --file <wrapper>/config/<project>.<env>.json`
 reports exactly that gap and exits 1, so it can gate CI.
 
+### Deleting metadata: where the node order comes from
+
+`delete metadata` walks nodes children-before-parents. The order is resolved
+from the first available of: an explicit `--import-order <path|s3://…>`
+(failures are fatal — an explicit source is never silently skipped), the
+registered study's release bucket (`DataImportOrder.txt` next to the release's
+node JSONs), a `DataImportOrder.txt` in the current directory, or a
+topological sort derived from the dictionary itself — the `--dict-version`
+tag's bundle when given (downloaded to `~/.g3dt/schemas` if needed), else the
+env's deployed dictionary. Deriving matters when deleting data submitted
+under an older dictionary whose node layout differs from today's: pass
+`--dict-version <old-tag>`. With `--on ec2`, `--import-order` must be an
+`s3://` URI (a laptop path does not exist on the box).
+
 Synthetic data is only schema-valid against the dictionary that generated it, so
 `synth generate` records the dictionary version in each batch and `synth upload`
 refuses a batch that doesn't match the version being uploaded (override with

{gen3_dataops_toolkit-4.2.0 → gen3_dataops_toolkit-4.3.0}/README.md

@@ -160,6 +160,20 @@ version until the CDK config catches up —
 `g3dt config diff --env <env> --file <wrapper>/config/<project>.<env>.json`
 reports exactly that gap and exits 1, so it can gate CI.
 
+### Deleting metadata: where the node order comes from
+
+`delete metadata` walks nodes children-before-parents. The order is resolved
+from the first available of: an explicit `--import-order <path|s3://…>`
+(failures are fatal — an explicit source is never silently skipped), the
+registered study's release bucket (`DataImportOrder.txt` next to the release's
+node JSONs), a `DataImportOrder.txt` in the current directory, or a
+topological sort derived from the dictionary itself — the `--dict-version`
+tag's bundle when given (downloaded to `~/.g3dt/schemas` if needed), else the
+env's deployed dictionary. Deriving matters when deleting data submitted
+under an older dictionary whose node layout differs from today's: pass
+`--dict-version <old-tag>`. With `--on ec2`, `--import-order` must be an
+`s3://` URI (a laptop path does not exist on the box).
+
 Synthetic data is only schema-valid against the dictionary that generated it, so
 `synth generate` records the dictionary version in each batch and `synth upload`
 refuses a batch that doesn't match the version being uploaded (override with

{gen3_dataops_toolkit-4.2.0 → gen3_dataops_toolkit-4.3.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "gen3-dataops-toolkit"
-version = "4.2.0"
+version = "4.3.0"
 description = "Gen3 DataOps toolkit (g3dt): operate SSM-published Gen3 data pipeline environments"
 authors = ["JoshuaHarris391 <harjo391@gmail.com>"]
 readme = "README.md"

{gen3_dataops_toolkit-4.2.0 → gen3_dataops_toolkit-4.3.0}/src/g3dt/cli/delete_cmds.py

@@ -14,6 +14,12 @@ synthetic projects are never registered), bare names default to version
 ``data_version`` property via GraphQL rather than Athena receipts (synthetic
 uploads write none).
 
+The node deletion order comes from, in order: an explicit ``--import-order``
+(path or s3:// URI; failures are fatal, never silently skipped), the
+registered study's release bucket, a ``DataImportOrder.txt`` in the current
+directory, or a topological sort derived from the dictionary itself — the
+``--dict-version`` bundle when given, else the env's deployed dictionary.
+
 Every command confirms before acting. Production always requires typing the
 target id, even with ``--yes``. Deleting ALL versions always prompts, even with
 ``--yes``. Confirmation happens locally before any EC2 dispatch (SSM has no
@@ -186,6 +192,21 @@ def metadata(
         help="Gen3 program for --synthetic (default: program1). "
              "Invalid without --synthetic.",
     ),
+    import_order: Optional[str] = typer.Option(
+        None,
+        "--import-order",
+        help="Path or s3:// URI of DataImportOrder.txt. Default: auto — the "
+             "study's release bucket (registered studies), then "
+             "./DataImportOrder.txt, then derived from the dictionary. "
+             "With --on ec2 only s3:// URIs are accepted.",
+    ),
+    dict_version: Optional[str] = typer.Option(
+        None,
+        "--dict-version",
+        help="Dictionary git tag to derive the node order from (verbatim, "
+             "e.g. v1.3.0). Default: the env's deployed dictionary. Only "
+             "used when the order is derived.",
+    ),
     yes: bool = typer.Option(
         False, "--yes", "-y", help="Skip the non-prod prompt (specific-version only)."
     ),
@@ -212,6 +233,26 @@ def metadata(
             err=True,
         )
         raise typer.Exit(2)
+    if import_order and dict_version:
+        typer.secho(
+            "--import-order names the exact file; --dict-version derives one "
+            "— pass only one.",
+            fg=typer.colors.RED,
+            err=True,
+        )
+        raise typer.Exit(2)
+    # A laptop path forwarded to the EC2 box would resolve against the box's
+    # filesystem — at best a crash after confirmation, at worst a same-named
+    # DIFFERENT file ordering the delete. s3:// URIs (and --dict-version)
+    # resolve identically anywhere, so only those may travel.
+    if on == Target.ec2 and import_order and not import_order.startswith("s3://"):
+        typer.secho(
+            "--import-order with --on ec2 must be an s3:// URI (a local path "
+            "does not exist on the box). Upload the file, or run locally.",
+            fg=typer.colors.RED,
+            err=True,
+        )
+        raise typer.Exit(2)
     if version is None:
         fallback = None
     elif synthetic:
@@ -266,6 +307,10 @@ def metadata(
             # optional on the wire, but an explicit value keeps the contract
             # visible in logs and SSM command history.
             a += ["--synthetic", "--program-id", program_id or "program1"]
+        if import_order:
+            a += ["--import-order", import_order]
+        if dict_version:
+            a += ["--dict-version", dict_version]
         return a
 
     def remote_cli(env_name):
@@ -284,6 +329,11 @@ def metadata(
             a.append("--synthetic")
             if program_id is not None:
                 a += ["--program-id", program_id]
+        if import_order:
+            # Guaranteed s3:// by the pre-dispatch gate above.
+            a += ["--import-order", import_order]
+        if dict_version:
+            a += ["--dict-version", dict_version]
         return a
 
     dispatch.run_or_dispatch(

gen3_dataops_toolkit-4.3.0/src/g3dt/import_order.py

@@ -0,0 +1,265 @@
+"""Node import-order resolution for metadata deletion.
+
+Deleting Gen3 metadata must walk nodes children-before-parents, which the
+toolkit historically took from a ``DataImportOrder.txt`` in the caller's
+working directory — a silent dependency that crashed runs started anywhere
+else. This module resolves the order from an explicit chain of sources:
+
+1. an explicit ``--import-order`` (local path or ``s3://`` URI) — fatal if
+   unreadable, never silently skipped;
+2. the study's release bucket (registered studies only — releases ship a
+   ``DataImportOrder.txt`` next to the node JSONs);
+3. a ``DataImportOrder.txt`` in the current directory (legacy behavior);
+4. derivation from the dictionary itself: a topological sort over the raw
+   bundle's ``links`` (proven byte-identical to simulator-written order
+   files), reading either the deployed dictionary at ``schema_s3_uri`` or a
+   ``--dict-version`` bundle cached under ``~/.g3dt/schemas``.
+
+Deliberately NOT used: ``gen3_validator``'s ``DataDictionary.get_node_order``
+(it force-moves core_metadata_collection last, which a deletion reversal would
+delete FIRST — before the file nodes that link to it) and its
+``get_node_link`` (inspects only ``links[0]`` for subgroups).
+"""
+from __future__ import annotations
+
+import json
+import logging
+import os
+import urllib.error
+import urllib.request
+from collections import deque
+from pathlib import Path
+from typing import List, Optional, Sequence, Tuple
+
+logger = logging.getLogger(__name__)
+
+#: Bundle keys that are shared definitions, not nodes.
+_META_KEYS = frozenset(
+    {"_definitions.yaml", "_terms.yaml", "_settings.yaml", "root.yaml", "metaschema.yaml"}
+)
+
+_ORDER_FILENAME = "DataImportOrder.txt"
+
+
+class ImportOrderError(Exception):
+    """A node order could not be resolved; the message names the fix."""
+
+
+def derive_import_order(schema: dict) -> List[str]:
+    """Topologically sort a raw Gen3 dictionary bundle into submission order.
+
+    Works on the UNRESOLVED bundle ({"<node>.yaml": {...}, ...}): ``links``
+    blocks are self-contained (no $refs), so no schema resolution is needed.
+    Skips the shared ``_*.yaml`` definitions, entries without ``properties``,
+    and ``submittable: false`` nodes (which removes ``program``). Every
+    ``links`` entry is flattened — subgroup members and plain links alike —
+    into edges ``parent -> child``; edges to nodes outside the set are
+    dropped (this handles ``project -> program``).
+
+    Kahn's algorithm with sorted tie-breaking keeps the output deterministic
+    and byte-compatible with simulator-written DataImportOrder.txt files.
+    Members of a dependency cycle (impossible in a valid Gen3 dictionary) are
+    appended at the end with a warning rather than dropped.
+
+    Returns SUBMISSION order (parents first); callers reverse for deletion.
+    """
+    nodes = {}
+    for key, value in schema.items():
+        if key in _META_KEYS or not isinstance(value, dict):
+            continue
+        if "properties" not in value or value.get("submittable") is False:
+            continue
+        name = key[: -len(".yaml")] if key.endswith(".yaml") else key
+        nodes[name] = value
+
+    graph = {n: [] for n in nodes}
+    in_degree = {n: 0 for n in nodes}
+    for name, node in nodes.items():
+        for entry in node.get("links", []) or []:
+            members = entry.get("subgroup", [entry]) if isinstance(entry, dict) else []
+            for member in members:
+                parent = member.get("target_type")
+                if parent in nodes:
+                    graph[parent].append(name)
+                    in_degree[name] += 1
+
+    queue = deque(sorted(n for n, d in in_degree.items() if d == 0))
+    ordered: List[str] = []
+    while queue:
+        node = queue.popleft()
+        ordered.append(node)
+        for child in sorted(graph[node]):
+            in_degree[child] -= 1
+            if in_degree[child] == 0:
+                queue.append(child)
+        queue = deque(sorted(queue))
+
+    leftovers = sorted(n for n in nodes if n not in ordered)
+    if leftovers:
+        logger.warning(
+            "Dictionary link cycle detected; appending unordered node(s): %s "
+            "— deleting them may require a re-run.",
+            ", ".join(leftovers),
+        )
+        ordered.extend(leftovers)
+    return ordered
+
+
+def ensure_local_dictionary(env_cfg, version: str, schema_dir: Optional[Path] = None) -> Path:
+    """Return the local bundle for ``version``, downloading it if needed.
+
+    The cache is ``$G3DT_SCHEMA_DIR`` (else ``~/.g3dt/schemas``), shared with
+    pull_dict.sh. A cache hit requires a non-empty file that parses as JSON —
+    ``wget -O`` leaves a zero-byte file behind on a failed pull, so mere
+    existence is not trusted. Downloads are validated as JSON BEFORE being
+    written, to a temp name replaced atomically, so a failed or partial
+    download can never poison the cache.
+    """
+    from g3dt import config
+
+    if schema_dir is None:
+        schema_dir = Path(
+            os.environ.get("G3DT_SCHEMA_DIR", "~/.g3dt/schemas")
+        ).expanduser()
+    target = schema_dir / config.dictionary_filename(env_cfg, version)
+
+    if target.exists() and target.stat().st_size > 0:
+        try:
+            json.loads(target.read_text(encoding="utf-8"))
+            return target
+        except ValueError:
+            logger.warning("Cached dictionary %s is corrupt; re-downloading.", target)
+
+    url = config.dictionary_url(env_cfg, version)
+    logger.info("Downloading dictionary %s from %s", version, url)
+    try:
+        with urllib.request.urlopen(url) as resp:
+            body = resp.read()
+    except (urllib.error.HTTPError, urllib.error.URLError) as exc:
+        raise ImportOrderError(
+            f"Could not download dictionary tag '{version}' from {url}: {exc}. "
+            f"Check the tag exists in the schema repo, or pass --import-order."
+        )
+    try:
+        json.loads(body.decode("utf-8"))
+    except ValueError:
+        raise ImportOrderError(
+            f"Downloaded dictionary tag '{version}' from {url} is not valid "
+            f"JSON — is the tag/path right?"
+        )
+    schema_dir.mkdir(parents=True, exist_ok=True)
+    tmp = target.with_name(target.name + ".part")
+    tmp.write_bytes(body)
+    os.replace(tmp, target)
+    return target
+
+
+def to_deletion_order(nodes: Sequence[str], exclude_nodes: Sequence[str]) -> List[str]:
+    """Filter excluded nodes out of a submission order and reverse it."""
+    kept = [n for n in nodes if n not in exclude_nodes]
+    kept.reverse()
+    return kept
+
+
+def resolve_import_order(
+    *,
+    env_cfg,
+    session,
+    import_order: Optional[str] = None,
+    dict_version: Optional[str] = None,
+    study_cfg=None,
+    cwd: Optional[Path] = None,
+) -> Tuple[List[str], str]:
+    """Resolve the node SUBMISSION order for a delete, returning (nodes, source).
+
+    The chain (first hit wins): explicit ``import_order`` (fatal on failure —
+    an explicit source is never silently skipped), the registered study's
+    release bucket (``study_cfg.s3_metadata_path``; a listing error warns and
+    falls through), a ``DataImportOrder.txt`` in the current directory
+    (legacy behavior), then derivation from the dictionary — the
+    ``dict_version`` bundle when given, else the deployed dictionary at
+    ``s3://{env_cfg.schema_s3_uri}``.
+
+    ``source`` is a human-readable description of the winning step; workers
+    log it so operators can always see what ordered a destructive run.
+    """
+    from g3dt.upload.metadata_submitter import (
+        find_data_import_order_file_s3,
+        read_data_import_order_txt_s3,
+        read_metadata_json_s3,
+    )
+
+    # 1. Explicit flag: local path or s3:// URI. Failures are fatal.
+    if import_order:
+        if import_order.startswith("s3://"):
+            try:
+                nodes = read_data_import_order_txt_s3(import_order, session)
+            except Exception as exc:
+                raise ImportOrderError(
+                    f"Could not read --import-order {import_order}: {exc}"
+                )
+            return nodes, f"explicit --import-order {import_order}"
+        path = Path(import_order)
+        try:
+            lines = path.read_text(encoding="utf-8").splitlines()
+        except OSError as exc:
+            raise ImportOrderError(
+                f"Could not read --import-order {import_order}: {exc}"
+            )
+        return (
+            [line.strip() for line in lines if line.strip()],
+            f"explicit --import-order {path}",
+        )
+
+    # 2. Registered study: the release ships its own DataImportOrder.txt.
+    if study_cfg is not None and str(
+        getattr(study_cfg, "s3_metadata_path", "")
+    ).startswith("s3://"):
+        try:
+            uri = find_data_import_order_file_s3(
+                s3_uri=study_cfg.s3_metadata_path, session=session
+            )
+            return (
+                read_data_import_order_txt_s3(uri, session),
+                f"release bucket {uri}",
+            )
+        except Exception as exc:
+            logger.warning(
+                "No usable DataImportOrder.txt under %s (%s) — falling back.",
+                study_cfg.s3_metadata_path,
+                exc,
+            )
+
+    # 3. Legacy: a DataImportOrder.txt in the working directory.
+    local = (cwd or Path.cwd()) / _ORDER_FILENAME
+    if local.exists():
+        lines = local.read_text(encoding="utf-8").splitlines()
+        return (
+            [line.strip() for line in lines if line.strip()],
+            f"{_ORDER_FILENAME} in current directory ({local.resolve()})",
+        )
+
+    # 4. Derive from the dictionary.
+    if dict_version:
+        bundle_path = ensure_local_dictionary(env_cfg, dict_version)
+        schema = json.loads(bundle_path.read_text(encoding="utf-8"))
+        source = f"derived from dictionary {dict_version} ({bundle_path})"
+    else:
+        uri = f"s3://{env_cfg.schema_s3_uri}"
+        try:
+            schema = read_metadata_json_s3(uri, session)
+        except Exception as exc:
+            raise ImportOrderError(
+                f"No DataImportOrder.txt found and the deployed dictionary at "
+                f"{uri} (SSM app/schema_s3_uri) could not be read: {exc}. "
+                f"Pass --import-order <path|s3://...> or --dict-version <tag>."
+            )
+        source = f"derived from deployed dictionary {uri}"
+
+    nodes = derive_import_order(schema)
+    if not nodes:
+        raise ImportOrderError(
+            f"Deriving the node order produced no submittable nodes — is the "
+            f"object read for '{source}' a Gen3 dictionary bundle?"
+        )
+    return nodes, source

{gen3_dataops_toolkit-4.2.0 → gen3_dataops_toolkit-4.3.0}/src/g3dt/services/delete/delete_all_metadata_for_project.py

@@ -10,6 +10,11 @@ from g3dt.upload.metadata_submitter import (
 from g3dt.upload.metadata_deleter import (
     delete_project_metadata,
 )
+from g3dt.import_order import (
+    ImportOrderError,
+    resolve_import_order,
+    to_deletion_order,
+)
 
 # ANSI colour codes
 GREEN = "\033[92m"
@@ -43,20 +48,6 @@ def setup_logger():
 from g3dt import config as g3dt_config  # noqa: E402
 
 
-def load_import_order(import_order_path, exclude_nodes=None):
-    """
-    Reads the DataImportOrder.txt file and returns the node list
-    in deletion order (reversed, with excluded nodes removed).
-    """
-    if exclude_nodes is None:
-        exclude_nodes = EXCLUDE_NODES
-    with open(import_order_path, 'r', encoding='utf-8') as f:
-        nodes = [line.strip() for line in f if line.strip()]
-    nodes = [n for n in nodes if n not in exclude_nodes]
-    nodes.reverse()
-    return nodes
-
-
 def main():
     logger = setup_logger()
 
@@ -98,8 +89,21 @@ def main():
     )
     parser.add_argument(
         "--import-order",
-        default="DataImportOrder.txt",
-        help="Path to DataImportOrder.txt",
+        default=None,
+        help=(
+            "Path or s3:// URI of DataImportOrder.txt. Default: auto "
+            "(the study's release bucket, then ./DataImportOrder.txt, "
+            "then derived from the dictionary)."
+        ),
+    )
+    parser.add_argument(
+        "--dict-version",
+        default=None,
+        help=(
+            "Dictionary git tag to derive the node order from (verbatim, "
+            "e.g. v1.3.0). Default: the env's deployed dictionary. Only "
+            "used when the order is derived."
+        ),
     )
     parser.add_argument(
         "--node",
@@ -119,6 +123,12 @@ def main():
 
     args = parser.parse_args()
 
+    if args.import_order and args.dict_version:
+        parser.error(
+            "--import-order names the exact file; --dict-version derives one "
+            "— pass only one."
+        )
+
     # Env facts from SSM; the study registry from the marker or
     # SSM /{project}/{env}/studies/* (legacy studies.yaml fallback until 5.0).
     # Synthetic projects are not registered studies: --synthetic skips the
@@ -126,6 +136,7 @@ def main():
     try:
         env_cfg = g3dt_config.resolve_env(args.env)
         if args.synthetic:
+            study_cfg = None
             project_id = args.study
             program_id = args.program_id
         else:
@@ -162,12 +173,23 @@ def main():
             BLUE, RESET, args.node,
         )
     else:
-        nodes_to_delete = load_import_order(args.import_order)
+        try:
+            nodes, source = resolve_import_order(
+                env_cfg=env_cfg,
+                session=session,
+                import_order=args.import_order,
+                dict_version=args.dict_version,
+                study_cfg=study_cfg,
+            )
+        except ImportOrderError as exc:
+            logger.error(str(exc))
+            sys.exit(1)
+        nodes_to_delete = to_deletion_order(nodes, EXCLUDE_NODES)
         logger.info(
-            "Loaded %s nodes from %s (deletion order, "
+            "Import order source: %s (%s nodes, deletion order, "
             "excluding %s)",
+            source,
             len(nodes_to_delete),
-            args.import_order,
             EXCLUDE_NODES,
         )
 

{gen3_dataops_toolkit-4.2.0 → gen3_dataops_toolkit-4.3.0}/src/g3dt/services/delete/delete_metadata.sh

@@ -24,6 +24,11 @@ Arguments:
   --synthetic  (optional) Registry-free synthetic-data mode: each study name is
                the Gen3 project code itself (no SSM study registry lookup)
   --program-id (optional) Gen3 program for --synthetic mode (default program1)
+  --import-order (optional) Path or s3:// URI of DataImportOrder.txt. Default:
+               auto — the study's release bucket (registered studies), then
+               ./DataImportOrder.txt, then derived from the dictionary
+  --dict-version (optional) Dictionary git tag to derive the node order from
+               (default: the env's deployed dictionary)
 
 Behaviour:
   * version 'all'          -> delete_all_metadata_for_project.py (deletes whole nodes)
@@ -52,6 +57,8 @@ VERSION=""
 NODE=""
 SYNTHETIC=0
 PROGRAM_ID="program1"
+IMPORT_ORDER=""
+DICT_VERSION=""
 
 while [[ $# -gt 0 ]]; do
     case "$1" in
@@ -79,6 +86,14 @@ while [[ $# -gt 0 ]]; do
             PROGRAM_ID="$2"
             shift 2
             ;;
+        --import-order)
+            IMPORT_ORDER="$2"
+            shift 2
+            ;;
+        --dict-version)
+            DICT_VERSION="$2"
+            shift 2
+            ;;
         *)
             echo "ERROR: Unknown argument: $1"
             usage
@@ -157,17 +172,17 @@ for i in "${!STUDY_NAMES[@]}"; do
         CMD=("${G3DT_PYTHON:-python3}" "${SCRIPT_DIR}/delete_all_metadata_for_project.py"
              --study "$study" --env "$ENV")
         [[ $SYNTHETIC -eq 1 ]] && CMD+=(--synthetic --program-id "$PROGRAM_ID")
-        [[ -n "$NODE" ]] && CMD+=(--node "$NODE")
     elif [[ $SYNTHETIC -eq 1 ]]; then
         CMD=("${G3DT_PYTHON:-python3}" "${SCRIPT_DIR}/delete_synth_metadata_by_version.py"
              --study "$study" --env "$ENV" --version "$study_version"
              --program-id "$PROGRAM_ID" --skip-if-empty)
-        [[ -n "$NODE" ]] && CMD+=(--node "$NODE")
     else
         CMD=("${G3DT_PYTHON:-python3}" "${SCRIPT_DIR}/delete_metadata_by_guid.py"
              --study "$study" --env "$ENV" --version "$study_version" --skip-if-empty)
-        [[ -n "$NODE" ]] && CMD+=(--node "$NODE")
     fi
+    [[ -n "$NODE" ]] && CMD+=(--node "$NODE")
+    [[ -n "$IMPORT_ORDER" ]] && CMD+=(--import-order "$IMPORT_ORDER")
+    [[ -n "$DICT_VERSION" ]] && CMD+=(--dict-version "$DICT_VERSION")
 
     # Run the worker without aborting the loop on a non-zero exit.
     set +e

{gen3_dataops_toolkit-4.2.0 → gen3_dataops_toolkit-4.3.0}/src/g3dt/services/delete/delete_metadata_by_guid.py

@@ -13,6 +13,11 @@ from g3dt.upload.metadata_deleter import (
     query_metadata_upload_guids,
     delete_records_by_guid,
 )
+from g3dt.import_order import (
+    ImportOrderError,
+    resolve_import_order,
+    to_deletion_order,
+)
 
 # ANSI colour codes (matching metadata_submitter.py style)
 GREEN = "\033[92m"
@@ -51,20 +56,6 @@ def setup_logger():
 from g3dt import config as g3dt_config, resolver  # noqa: E402
 
 
-def load_import_order(import_order_path, exclude_nodes=None):
-    """
-    Reads the DataImportOrder.txt file and returns the node list
-    in deletion order (reversed, with excluded nodes removed).
-    """
-    if exclude_nodes is None:
-        exclude_nodes = EXCLUDE_NODES
-    with open(import_order_path, 'r', encoding='utf-8') as f:
-        nodes = [line.strip() for line in f if line.strip()]
-    nodes = [n for n in nodes if n not in exclude_nodes]
-    nodes.reverse()
-    return nodes
-
-
 def main():
     logger = setup_logger()
 
@@ -96,8 +87,21 @@ def main():
     )
     parser.add_argument(
         "--import-order",
-        default="DataImportOrder.txt",
-        help="Path to DataImportOrder.txt",
+        default=None,
+        help=(
+            "Path or s3:// URI of DataImportOrder.txt. Default: auto "
+            "(the study's release bucket, then ./DataImportOrder.txt, "
+            "then derived from the dictionary)."
+        ),
+    )
+    parser.add_argument(
+        "--dict-version",
+        default=None,
+        help=(
+            "Dictionary git tag to derive the node order from (verbatim, "
+            "e.g. v1.3.0). Default: the env's deployed dictionary. Only "
+            "used when the order is derived."
+        ),
     )
     parser.add_argument(
         "--node",
@@ -151,6 +155,12 @@ def main():
 
     args = parser.parse_args()
 
+    if args.import_order and args.dict_version:
+        parser.error(
+            "--import-order names the exact file; --dict-version derives one "
+            "— pass only one."
+        )
+
     if args.verbose:
         logger.setLevel(logging.DEBUG)
 
@@ -225,12 +235,23 @@ def main():
             BLUE, RESET, args.node,
         )
     else:
-        nodes_to_delete = load_import_order(args.import_order)
+        try:
+            nodes, source = resolve_import_order(
+                env_cfg=env_cfg,
+                session=session,
+                import_order=args.import_order,
+                dict_version=args.dict_version,
+                study_cfg=study_cfg,
+            )
+        except ImportOrderError as exc:
+            logger.error(str(exc))
+            sys.exit(1)
+        nodes_to_delete = to_deletion_order(nodes, EXCLUDE_NODES)
         logger.info(
-            "Loaded %s nodes from %s (deletion order, "
+            "Import order source: %s (%s nodes, deletion order, "
             "excluding %s)",
+            source,
             len(nodes_to_delete),
-            args.import_order,
             EXCLUDE_NODES,
         )
 

{gen3_dataops_toolkit-4.2.0 → gen3_dataops_toolkit-4.3.0}/src/g3dt/services/delete/delete_synth_metadata_by_version.py

@@ -26,6 +26,11 @@ from g3dt.upload.metadata_submitter import (
     create_gen3_submission_class,
 )
 from g3dt.upload.metadata_deleter import delete_node_records_by_property
+from g3dt.import_order import (
+    ImportOrderError,
+    resolve_import_order,
+    to_deletion_order,
+)
 
 # ANSI colour codes (matching metadata_submitter.py style)
 GREEN = "\033[92m"
@@ -64,20 +69,6 @@ def setup_logger():
 from g3dt import config as g3dt_config  # noqa: E402
 
 
-def load_import_order(import_order_path, exclude_nodes=None):
-    """
-    Reads the DataImportOrder.txt file and returns the node list
-    in deletion order (reversed, with excluded nodes removed).
-    """
-    if exclude_nodes is None:
-        exclude_nodes = EXCLUDE_NODES
-    with open(import_order_path, 'r', encoding='utf-8') as f:
-        nodes = [line.strip() for line in f if line.strip()]
-    nodes = [n for n in nodes if n not in exclude_nodes]
-    nodes.reverse()
-    return nodes
-
-
 def main():
     logger = setup_logger()
 
@@ -116,8 +107,21 @@ def main():
     )
     parser.add_argument(
         "--import-order",
-        default="DataImportOrder.txt",
-        help="Path to DataImportOrder.txt",
+        default=None,
+        help=(
+            "Path or s3:// URI of DataImportOrder.txt. Default: auto "
+            "(./DataImportOrder.txt, then derived from the dictionary — "
+            "synthetic projects have no release bucket)."
+        ),
+    )
+    parser.add_argument(
+        "--dict-version",
+        default=None,
+        help=(
+            "Dictionary git tag to derive the node order from (verbatim, "
+            "e.g. v1.3.0). Default: the env's deployed dictionary. Only "
+            "used when the order is derived."
+        ),
     )
     parser.add_argument(
         "--node",
@@ -171,6 +175,12 @@ def main():
 
     args = parser.parse_args()
 
+    if args.import_order and args.dict_version:
+        parser.error(
+            "--import-order names the exact file; --dict-version derives one "
+            "— pass only one."
+        )
+
     if args.verbose:
         logger.setLevel(logging.DEBUG)
 
@@ -215,12 +225,23 @@ def main():
             BLUE, RESET, args.node,
         )
     else:
-        nodes_to_delete = load_import_order(args.import_order)
+        try:
+            nodes, source = resolve_import_order(
+                env_cfg=env_cfg,
+                session=session,
+                import_order=args.import_order,
+                dict_version=args.dict_version,
+                study_cfg=None,  # synthetic projects are never registered
+            )
+        except ImportOrderError as exc:
+            logger.error(str(exc))
+            sys.exit(1)
+        nodes_to_delete = to_deletion_order(nodes, EXCLUDE_NODES)
         logger.info(
-            "Loaded %s nodes from %s (deletion order, "
+            "Import order source: %s (%s nodes, deletion order, "
             "excluding %s)",
+            source,
             len(nodes_to_delete),
-            args.import_order,
             EXCLUDE_NODES,
         )
 

{gen3_dataops_toolkit-4.2.0 → gen3_dataops_toolkit-4.3.0}/src/g3dt/utils/athena_utils.py

@@ -13,7 +13,6 @@ import pytz
 import base64
 import numpy as np
 from decimal import Decimal
-from gen3_validator.dict import DataDictionary
 
 logger = logging.getLogger(__name__)
 
@@ -916,14 +915,6 @@ def write_gold_json_to_s3(
     logger.info(f"Object created at s3://{s3_bucket}/{s3_object_key}")
 
 
-def construct_data_import_order(s3_uri) -> list:
-    from g3dt.validate.validate import load_schema_from_s3_uri
-    schema_dict = load_schema_from_s3_uri(s3_uri)
-    dd = DataDictionary(schema_dict)
-    dd.schema = schema_dict
-    dd.calculate_node_order()
-    return dd.node_order
-
 def write_release_jsons_to_s3(s3_bucket, release_id, study_id, table_name, json_data,
                               s3_client=None, key_prefix="release_jsons"):
     """