gen3-dataops-toolkit 2.0.0__tar.gz

gen3_dataops_toolkit-2.0.0/PKG-INFO

@@ -0,0 +1,125 @@
+Metadata-Version: 2.4
+Name: gen3-dataops-toolkit
+Version: 2.0.0
+Summary: Gen3 DataOps toolkit (g3dt): operate SSM-published Gen3 data pipeline environments
+License: Apache-2.0
+Author: JoshuaHarris391
+Author-email: harjo391@gmail.com
+Requires-Python: >=3.9.5,<4.0.0
+Classifier: License :: OSI Approved :: Apache Software License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Programming Language :: Python :: 3.14
+Requires-Dist: awswrangler (>=3.14.0,<4.0.0)
+Requires-Dist: boto3
+Requires-Dist: gen3 (>=4.27.4,<5.0.0)
+Requires-Dist: gen3-metadata (>=1.4.0,<2.0.0)
+Requires-Dist: gen3_validator (>=2.0.0,<3.0.0)
+Requires-Dist: numpy (<2.0.0)
+Requires-Dist: pyarrow (>=14.0.0,<19.0.0)
+Requires-Dist: pyjwt (>=2.10.1,<3.0.0)
+Requires-Dist: python-dotenv
+Requires-Dist: pytz (>=2025.2,<2026.0)
+Requires-Dist: pyyaml (>=6.0.2,<7.0.0)
+Requires-Dist: s3fs (==2025.10.0)
+Requires-Dist: tenacity (>=8.2,<10.0)
+Requires-Dist: typer (>=0.12)
+Requires-Dist: tzlocal (>=5.3.1,<6.0.0)
+Project-URL: Repository, https://github.com/AustralianBioCommons/gen3-dataops-toolkit
+Description-Content-Type: text/markdown
+
+# gen3-dataops-toolkit (`g3dt`)
+
+Operate Gen3 AWS data-pipeline environments from one pip-installable CLI.
+
+`g3dt` is the tooling half of the Gen3 DataOps platform: the
+[gen3-aws-data-pipeline](https://github.com/AustralianBioCommons/gen3-aws-data-pipeline)
+CDK app deploys a complete pipeline per project/environment and publishes every
+resource name to AWS SSM Parameter Store; `g3dt` resolves those names at
+runtime and gives operators one command surface for dictionary deploys,
+metadata upload/delete, indexd registration, EC2 job dispatch, and Kubernetes
+restarts. The dbt half of the platform lives in
+[gen3-dbt-template](https://github.com/AustralianBioCommons/gen3-dbt-template).
+
+**No AWS resource name is compiled into this package.** The same wheel
+operates any project: it is targeted purely by `--env`, the project's SSM tree
+(`/{project}/{env}/...`), and a tiny local bootstrap marker.
+
+## Install
+
+```bash
+pip install gen3-dataops-toolkit
+```
+
+## Bootstrap (the only local configuration)
+
+`g3dt` needs to know just the project and region — everything else comes from
+SSM. Create `~/.g3dt/g3dt.yaml`:
+
+```yaml
+project: etl                # your projectId
+region: ap-southeast-2
+default_env: test
+profiles:                   # optional: AWS named profile per env
+  test: etl_test            # (omit entirely on EC2/CodeBuild — ambient
+  staging: etl_staging      #  role credentials are used)
+studies:                    # optional: the project's study registry;
+  mystudy_test:             # alternatively upload it once per env to
+    project_id: MyStudy     # s3://<metadata-bucket>/config/studies.yaml
+    program_id: program1
+    s3_metadata_path: s3://my-bucket/metadata/mystudy/
+```
+
+Search order: `./g3dt.yaml` → `~/.g3dt/g3dt.yaml` → `/etc/g3dt/g3dt.yaml`
+(the EC2 job box's copy, written by CDK user-data). Env vars override:
+`G3DT_PROJECT`, `AWS_REGION`, `G3DT_DEFAULT_ENV`.
+
+## Quick start
+
+```bash
+g3dt config envs                 # environments with a deployed SSM tree
+g3dt config show --env test      # every resolved name — the safety check
+g3dt ec2 up --env test           # start the env's job box (SSM-managed)
+g3dt metadata upload --study mystudy --env test --on ec2
+g3dt jobs logs <run-id> --follow # live logs; laptop can sleep, job keeps going
+g3dt ec2 down --env test         # or let the auto-stop alarm handle it
+g3dt docs                        # the full operations overview
+```
+
+## How configuration works
+
+There are exactly two kinds of configuration:
+
+- **INPUTS** — human-authored values, committed as
+  `config/<projectId>.<env>.json` in the CDK repo and read only by
+  `cdk deploy`. To change a deployed setting (e.g. the dictionary version),
+  edit that file and redeploy — the value flows to SSM.
+- **OUTPUTS** — every resource name the CDK creates plus the mirrored Gen3
+  app facts, published to SSM under `/{project}/{env}/...` on deploy. `g3dt`
+  reads these live (cached one round-trip per invocation) and never stores
+  them locally.
+
+Because the CLI and the infrastructure read the same parameters, they cannot
+disagree — and because each environment has its own tree (including its own
+`ec2/instanceId`), running a job against the wrong environment's resources is
+structurally impossible.
+
+## Development
+
+```bash
+poetry install
+poetry run python3 -m pytest
+```
+
+## Provenance
+
+This toolkit was ported (working tree only) from
+[AustralianBioCommons/acdc-aws-etl-pipeline](https://github.com/AustralianBioCommons/acdc-aws-etl-pipeline),
+the ACDC ETL monolith, as part of the Gen3 DataOps platform refactor (2026).
+It starts at version **2.0.0**; versions ≤ 1.2.0 on PyPI are the legacy
+`acdc_aws_etl_pipeline` package, which continues to operate the legacy ACDC
+pipeline unchanged.
+

gen3_dataops_toolkit-2.0.0/README.md

@@ -0,0 +1,91 @@
+# gen3-dataops-toolkit (`g3dt`)
+
+Operate Gen3 AWS data-pipeline environments from one pip-installable CLI.
+
+`g3dt` is the tooling half of the Gen3 DataOps platform: the
+[gen3-aws-data-pipeline](https://github.com/AustralianBioCommons/gen3-aws-data-pipeline)
+CDK app deploys a complete pipeline per project/environment and publishes every
+resource name to AWS SSM Parameter Store; `g3dt` resolves those names at
+runtime and gives operators one command surface for dictionary deploys,
+metadata upload/delete, indexd registration, EC2 job dispatch, and Kubernetes
+restarts. The dbt half of the platform lives in
+[gen3-dbt-template](https://github.com/AustralianBioCommons/gen3-dbt-template).
+
+**No AWS resource name is compiled into this package.** The same wheel
+operates any project: it is targeted purely by `--env`, the project's SSM tree
+(`/{project}/{env}/...`), and a tiny local bootstrap marker.
+
+## Install
+
+```bash
+pip install gen3-dataops-toolkit
+```
+
+## Bootstrap (the only local configuration)
+
+`g3dt` needs to know just the project and region — everything else comes from
+SSM. Create `~/.g3dt/g3dt.yaml`:
+
+```yaml
+project: etl                # your projectId
+region: ap-southeast-2
+default_env: test
+profiles:                   # optional: AWS named profile per env
+  test: etl_test            # (omit entirely on EC2/CodeBuild — ambient
+  staging: etl_staging      #  role credentials are used)
+studies:                    # optional: the project's study registry;
+  mystudy_test:             # alternatively upload it once per env to
+    project_id: MyStudy     # s3://<metadata-bucket>/config/studies.yaml
+    program_id: program1
+    s3_metadata_path: s3://my-bucket/metadata/mystudy/
+```
+
+Search order: `./g3dt.yaml` → `~/.g3dt/g3dt.yaml` → `/etc/g3dt/g3dt.yaml`
+(the EC2 job box's copy, written by CDK user-data). Env vars override:
+`G3DT_PROJECT`, `AWS_REGION`, `G3DT_DEFAULT_ENV`.
+
+## Quick start
+
+```bash
+g3dt config envs                 # environments with a deployed SSM tree
+g3dt config show --env test      # every resolved name — the safety check
+g3dt ec2 up --env test           # start the env's job box (SSM-managed)
+g3dt metadata upload --study mystudy --env test --on ec2
+g3dt jobs logs <run-id> --follow # live logs; laptop can sleep, job keeps going
+g3dt ec2 down --env test         # or let the auto-stop alarm handle it
+g3dt docs                        # the full operations overview
+```
+
+## How configuration works
+
+There are exactly two kinds of configuration:
+
+- **INPUTS** — human-authored values, committed as
+  `config/<projectId>.<env>.json` in the CDK repo and read only by
+  `cdk deploy`. To change a deployed setting (e.g. the dictionary version),
+  edit that file and redeploy — the value flows to SSM.
+- **OUTPUTS** — every resource name the CDK creates plus the mirrored Gen3
+  app facts, published to SSM under `/{project}/{env}/...` on deploy. `g3dt`
+  reads these live (cached one round-trip per invocation) and never stores
+  them locally.
+
+Because the CLI and the infrastructure read the same parameters, they cannot
+disagree — and because each environment has its own tree (including its own
+`ec2/instanceId`), running a job against the wrong environment's resources is
+structurally impossible.
+
+## Development
+
+```bash
+poetry install
+poetry run python3 -m pytest
+```
+
+## Provenance
+
+This toolkit was ported (working tree only) from
+[AustralianBioCommons/acdc-aws-etl-pipeline](https://github.com/AustralianBioCommons/acdc-aws-etl-pipeline),
+the ACDC ETL monolith, as part of the Gen3 DataOps platform refactor (2026).
+It starts at version **2.0.0**; versions ≤ 1.2.0 on PyPI are the legacy
+`acdc_aws_etl_pipeline` package, which continues to operate the legacy ACDC
+pipeline unchanged.

gen3_dataops_toolkit-2.0.0/pyproject.toml

@@ -0,0 +1,50 @@
+[tool.poetry]
+name = "gen3-dataops-toolkit"
+version = "2.0.0"
+description = "Gen3 DataOps toolkit (g3dt): operate SSM-published Gen3 data pipeline environments"
+authors = ["JoshuaHarris391 <harjo391@gmail.com>"]
+readme = "README.md"
+license = "Apache-2.0"
+repository = "https://github.com/AustralianBioCommons/gen3-dataops-toolkit"
+packages = [{ include = "g3dt", from = "src" }]
+# Ship the services scripts (and any other non-.py assets) inside the wheel —
+# the toolkit must work from a bare `pip install`, no repo checkout.
+include = [{ path = "src/g3dt/services/**/*", format = ["sdist", "wheel"] }]
+
+[tool.poetry.dependencies]
+python = ">=3.9.5,<4.0.0"
+boto3 = "*"
+python-dotenv = "*"
+numpy = "<2.0.0"
+pyyaml = ">=6.0.2,<7.0.0"
+tzlocal = ">=5.3.1,<6.0.0"
+gen3 = ">=4.27.4, <5.0.0"
+gen3_validator = ">=2.0.0,<3.0.0"
+pytz = ">=2025.2,<2026.0"
+s3fs = "2025.10.0"
+pyjwt = ">=2.10.1,<3.0.0"
+tenacity = ">=8.2,<10.0"
+awswrangler = ">=3.14.0,<4.0.0"
+pyarrow = ">=14.0.0,<19.0.0"
+gen3-metadata = ">=1.4.0,<2.0.0"
+typer = ">=0.12"
+
+[tool.poetry.scripts]
+g3dt = "g3dt.cli.main:main"
+
+[tool.poetry.group.dev.dependencies]
+pytest = "*"
+moto = ">=5.0"
+
+# Optional: synthetic-data generator (LLM-realistic). Install with
+# `poetry install --with synth` (or `g3dt synth install-simulator`). Kept out
+# of the default install so core/EC2 environments stay lean.
+[tool.poetry.group.synth]
+optional = true
+
+[tool.poetry.group.synth.dependencies]
+gen3-metadata-simulator = "^0.3.0"
+
+[build-system]
+requires = ["poetry-core>=2.0.0,<3.0.0"]
+build-backend = "poetry.core.masonry.api"

gen3_dataops_toolkit-2.0.0/src/g3dt/cli/__init__.py

@@ -0,0 +1,5 @@
+"""The ``acdc`` operations CLI.
+
+A thin, self-documenting command layer over the project's existing service
+scripts and library functions. Run ``acdc --help`` or ``acdc docs`` to start.
+"""

gen3_dataops_toolkit-2.0.0/src/g3dt/cli/_internal/__init__.py

@@ -0,0 +1 @@
+"""Internal helpers for the acdc CLI (subprocess runner, safety guards, EC2 dispatch)."""