dmtri 0.1.0__py3-none-any.whl

dmtri/cli.py

@@ -0,0 +1,118 @@
+import argparse
+import sys
+import os
+from datetime import datetime, timezone
+import logging
+
+from dmtri.execution_plan import print_execution_plan
+from dmtri.utils import get_version, validate_and_normalize_args
+from dmtri.hooks.hooks import run_hook
+from dmtri.paths import COMMAND_PLAYBOOKS
+
+logging.basicConfig(level=logging.INFO, format="%(message)s")
+logger = logging.getLogger(__name__)
+
+PROJECT_URL = "https://github.com/EIDA/dmtri/issues"
+
+def main():
+    global_options = argparse.ArgumentParser(add_help=False)
+
+    parser = argparse.ArgumentParser(
+        description="Trigger data/metadata refresh or cleaning for multiple EIDA endpoints.",
+        parents=[global_options]
+    )
+    parser.add_argument(
+        "--version",
+        action="version",
+        version=f"%(prog)s version {get_version()} — See {PROJECT_URL} for updates or to report issues."
+    )
+
+    subparsers = parser.add_subparsers(dest="command")
+
+    shared = argparse.ArgumentParser(add_help=False, parents=[global_options])
+    shared.add_argument("-n", "--network", nargs="+", default=["*"], help="FDSN network code(s), supports wildcards")
+    shared.add_argument("-s", "--station", nargs="+", default=["*"], help="FDSN station code(s), supports wildcards")
+    shared.add_argument("-l", "--location", nargs="+", default=["*"], help="FDSN location code(s), supports wildcards")
+    shared.add_argument("-c", "--channel", nargs="+", default=["*"], help="FDSN channel code(s), supports wildcards")
+    shared.add_argument("-S", "--starttime", required=True, help="Start time (ISO 8601 format)")
+
+    now_str = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%S")
+    shared.add_argument("-E", "--endtime", default=now_str, help="End time (default: now in UTC)")
+    shared.add_argument("--type", default="data,metadata", help="Comma-separated types to process (data, metadata)")
+    shared.add_argument("--no-confirm", action="store_true", help="Skip confirmation prompt before executing")
+    shared.add_argument("--debug", action="store_true", help="Show verbose output from Ansible")
+
+    subparsers.add_parser("refresh", parents=[shared], help="Refresh data/metadata using configured playbooks.")
+    subparsers.add_parser("clean", parents=[shared], help="Clean outdated data/metadata using configured playbooks.")
+    subparsers.add_parser("track", help="Track job status via tracking playbook.")
+    doctor_parser = subparsers.add_parser("doctor", help="Check SSH connectivity to all inventory hosts")
+    doctor_parser.add_argument("-v", "--verbose", action="store_true", help="Show full Ansible output per host")
+
+    args = parser.parse_args()
+    if args.command == "track":
+        cfg_file = "ansible_verbose.cfg"
+    else:
+        cfg_file = "ansible_verbose.cfg" if getattr(args, "debug", False) else "ansible_quiet.cfg"
+
+    cfg_path = os.path.abspath(cfg_file)
+    os.environ["ANSIBLE_CONFIG"] = cfg_path
+
+    if args.command == "doctor":
+        from dmtri.doctor import run_diagnostics
+        return run_diagnostics(verbose=args.verbose)
+
+    elif args.command in ("refresh", "clean"):
+        start_dt, end_dt, types = validate_and_normalize_args(args, parser)
+
+        vars_to_pass = {
+            "network": args.network,
+            "station": args.station,
+            "location": args.location,
+            "channel": args.channel,
+            "starttime": args.starttime,
+            "endtime": args.endtime,
+            "type": [t.strip() for t in args.type.split(",")],
+            "debug": args.debug,
+        }
+
+        if args.command == "refresh":
+            selected_types = vars_to_pass["type"]
+            valid_types = set(COMMAND_PLAYBOOKS["refresh"].keys())
+
+            invalid = set(selected_types) - valid_types
+            if invalid:
+                print(f" Invalid --type: {', '.join(invalid)}. Valid types are: {', '.join(valid_types)}.")
+                sys.exit(1)
+
+            playbooks = []
+            for t in selected_types:
+                playbooks.extend(COMMAND_PLAYBOOKS["refresh"][t])
+        else:
+            playbooks = COMMAND_PLAYBOOKS.get("clean", [])
+
+        if not playbooks:
+            print(f"No playbooks configured for command '{args.command}'.")
+            sys.exit(1)
+
+        if not args.no_confirm:
+            for pb in playbooks:
+                print_execution_plan(args.command, vars_to_pass, pb)
+            proceed = input("\nDo you want to proceed with executing all playbooks? (y/N): ").strip().lower()
+            if proceed != "y":
+                print("Aborted by user.")
+                sys.exit(0)
+
+        for pb in playbooks:
+            run_hook(pb, vars_to_pass)
+
+    elif args.command == "track":
+        playbooks = COMMAND_PLAYBOOKS.get("track", [])
+        if not playbooks:
+            print("No playbooks configured for command 'track'.")
+            sys.exit(1)
+        for pb in playbooks:
+            run_hook(pb, {})
+
+    else:
+        parser.print_help()
+        sys.exit(1)

dmtri/doctor.py

@@ -0,0 +1,52 @@
+import logging
+import ansible_runner
+from dmtri.paths import INVENTORY_FILE
+
+logger = logging.getLogger(__name__)
+
+def run_diagnostics(verbose=False, retries=1):
+    logger.info(f"Running connectivity check using inventory: {INVENTORY_FILE}")
+
+    ok_hosts = set()
+    unreachable_hosts = set()
+
+    for attempt in range(retries):
+        logger.info(f"Attempt {attempt + 1} of {retries}")
+
+        r = ansible_runner.run(
+            private_data_dir=".",
+            inventory=str(INVENTORY_FILE.resolve()),
+            module="ping",
+            host_pattern="all",
+            quiet=not verbose,
+            envvars={
+                "ANSIBLE_CACHE_PLUGIN": "memory"  # Disable caching to avoid plugin warnings
+            }
+        )
+
+        for event in r.events:
+            if not isinstance(event, dict):
+                continue
+            data = event.get("event_data", {})
+            host = data.get("host")
+            if not host:
+                continue
+            if event["event"] == "runner_on_ok":
+                ok_hosts.add(host)
+            elif event["event"] == "runner_on_unreachable":
+                unreachable_hosts.add(host)
+
+        # Break early if all hosts responded
+        if ok_hosts and not unreachable_hosts:
+            break
+
+    logger.info("\nResults:")
+
+    if not ok_hosts and not unreachable_hosts:
+        logger.warning("No responses received. Are you connected to the VPN?")
+        return
+
+    for host in sorted(ok_hosts):
+        logger.info(f"[OK] {host}")
+    for host in sorted(unreachable_hosts):
+        logger.error(f"[UNREACHABLE] {host}")

dmtri/execution_plan.py

@@ -0,0 +1,59 @@
+import sys
+import yaml
+from pathlib import Path
+
+
+def get_playbook_metadata(playbook_path: str):
+    try:
+        with open(playbook_path) as f:
+            playbook = yaml.safe_load(f)
+
+        if not playbook or not isinstance(playbook, list) or not playbook[0]:
+            return {"error": "Empty playbook"}
+
+        metadata = {
+            "hosts": playbook[0].get("hosts", "unknown"),
+            "description": playbook[0].get("vars", {}).get("description", ""),
+            "tasks": []
+        }
+
+        for task in playbook[0].get("tasks", []):
+            desc = task.get("vars", {}).get("description", task.get("name", "Unnamed Task"))
+            metadata["tasks"].append(desc)
+
+        return metadata
+
+    except Exception as e:
+        return {"error": str(e)}
+
+
+
+    
+
+def print_execution_plan(command: str, variables: dict, playbook_path: str, metadata=None):
+    if metadata is None:
+        metadata = get_playbook_metadata(playbook_path)
+
+    print("\nExecution Plan")
+    print("-" * 40)
+    print(f"Command     : {command}")
+    print(f"Playbook    : {playbook_path}")
+
+    if "error" in metadata:
+        print(f"Warning     : {metadata['error']}")
+    else:
+        print(f"Hosts       : {metadata['hosts']}")
+        print(f"Description : {metadata['description']}")
+        print("Steps       :")
+        for idx, desc in enumerate(metadata['tasks'], start=1):
+            print(f"  {idx}. {desc}")
+
+    if variables:
+        print("\nVariables:")
+        for k, v in variables.items():
+            print(f"  {k:10}: {v}")
+
+    confirm = input("\nProceed with this execution plan? [y/N]: ").strip().lower()
+    if confirm != "y":
+        print("Aborted by user.")
+        sys.exit(0)

dmtri/hooks/hooks.py

@@ -0,0 +1,45 @@
+import ansible_runner
+import sys
+from pathlib import Path
+import logging
+from dmtri.paths import INVENTORY_FILE
+
+
+logger = logging.getLogger(__name__)
+def run_hook(playbook_name: Path, vars: dict,inventory: Path = INVENTORY_FILE) -> None:
+    """
+    Run an Ansible playbook using ansible_runner.run_command, bypassing the project layout.
+    """
+
+    # Build the extra vars string manually
+    extra_vars = []
+    for key, value in vars.items():
+        if isinstance(value, list):
+            value = ",".join(value)
+        extra_vars.extend(["-e", f"{key}='{value}'"])
+
+    cmd = [
+        "ansible-playbook",
+        str(playbook_name.resolve()),
+        "-i",
+        str(inventory.resolve()),
+        
+    ] + extra_vars
+
+
+    logger.info(f"Running: {' '.join(str(x) for x in cmd)}")
+
+
+    out, err, rc = ansible_runner.run_command(
+        executable_cmd="ansible-playbook",
+        cmdline_args=cmd[1:], 
+        input_fd=sys.stdin,
+        output_fd=sys.stdout,
+        error_fd=sys.stderr,
+    )
+
+    if rc != 0:
+        logger.error(f"Playbook {playbook_name} failed with return code {rc}")
+        sys.exit(rc)
+    
+    logger.info(f"Playbook {playbook_name} completed successfully")

dmtri/paths.py

@@ -0,0 +1,49 @@
+from pathlib import Path
+
+MODULE_DIR = Path(__file__).resolve().parent
+SRC_DIR = MODULE_DIR.parent
+
+# Directories
+PLAYBOOKS_DIR = SRC_DIR / "playbooks"
+INVENTORY_DIR = SRC_DIR / "inventory"
+
+# Inventory file
+INVENTORY_FILE = INVENTORY_DIR / "hosts.ini"
+
+# Specific playbooks
+PLAYBOOK_TRACK = PLAYBOOKS_DIR / "track.yml"
+PLAYBOOK_LIST_SDS = PLAYBOOKS_DIR / "list_sds_files.yml"
+
+# Refresh Data playbooks
+PLAYBOOK_SEEDPSD_REFRESH = PLAYBOOKS_DIR / "refresh" / "seedpsd_refresh.yml"
+PLAYBOOK_WFCATALOG_REFRESH = PLAYBOOKS_DIR / "refresh" / "wfcatalog_refresh.yml"
+PLAYBOOK_AVAILABILITY_REFRESH = PLAYBOOKS_DIR / "refresh" / "availability_refresh.yml"
+# Refresh metadata playbooks
+PLAYBOOK_METADATA_REFRESH = PLAYBOOKS_DIR / "refresh" / "seedpsd_metadata_refresh.yml"
+
+# Clean playbooks
+PLAYBOOK_SEEDPSD_CLEAN = PLAYBOOKS_DIR / "clean" /"seedpsd_clean.yml"
+PLAYBOOK_WFCATALOG_CLEAN = PLAYBOOKS_DIR / "clean" / "wfcatalog_clean.yml"
+PLAYBOOK_AVAILABILITY_CLEAN = PLAYBOOKS_DIR / "clean" / "availability_clean.yml"
+
+# Group playbooks by command
+COMMAND_PLAYBOOKS = {
+    "refresh": {
+        "data": [
+            PLAYBOOK_SEEDPSD_REFRESH,
+            PLAYBOOK_WFCATALOG_REFRESH,
+            PLAYBOOK_AVAILABILITY_REFRESH,
+        ],
+        "metadata": [
+            PLAYBOOK_METADATA_REFRESH,
+        ]
+    },
+    "clean": [
+        PLAYBOOK_SEEDPSD_CLEAN,
+        PLAYBOOK_WFCATALOG_CLEAN,
+        PLAYBOOK_AVAILABILITY_CLEAN,
+    ],
+    "track": [
+        PLAYBOOK_TRACK
+    ]
+}

dmtri/utils.py

@@ -0,0 +1,85 @@
+import os
+import sys
+from datetime import datetime, timezone
+from pathlib import Path
+import tomllib
+
+def get_version():
+    root = Path(__file__).resolve().parents[2]
+    pyproject_path = root / "pyproject.toml"
+    with open(pyproject_path, "rb") as f:
+        data = tomllib.load(f)
+        return data["project"]["version"]
+
+def maybe_warn_shell_globbing(args):
+    cwd_files = set(os.listdir("."))
+
+    def looks_globbed(value_list):
+        return any(v in cwd_files for v in value_list)
+
+    if looks_globbed(args.station):
+        print("⚠️  Error: --station appears to be expanded by your shell. Use --station '*'.", file=sys.stderr)
+        sys.exit(1)
+
+    if looks_globbed(args.network):
+        print("⚠️  Error: --network appears to be expanded by your shell. Use --network '*'.", file=sys.stderr)
+        sys.exit(1)
+
+def validate_datetime_range(start_dt, end_dt):
+    now = datetime.now(timezone.utc)
+
+
+    if start_dt > now:
+        print("Error: --starttime cannot be in the future.", file=sys.stderr)
+        sys.exit(1)
+
+    if end_dt > now:
+        print("Error: --endtime cannot be in the future.", file=sys.stderr)
+        sys.exit(1)
+
+    if end_dt <= start_dt:
+        print("Error: --endtime must be after --starttime.", file=sys.stderr)
+        sys.exit(1)
+
+    duration_days = (end_dt - start_dt).days
+    if duration_days > 7:
+        print(f"Warning: You are querying more than 7 days of data ({duration_days} days).")
+        proceed = input("Are you sure you want to continue? (y/N): ").strip().lower()
+        if proceed != "y":
+            print("Aborted by user.")
+            sys.exit(0)
+
+def parse_datetime_args(args):
+    try:
+        start_dt = datetime.fromisoformat(args.starttime)
+        end_dt = datetime.fromisoformat(args.endtime)
+
+        if start_dt.tzinfo is None:
+            start_dt = start_dt.replace(tzinfo=timezone.utc)
+        if end_dt.tzinfo is None:
+            end_dt = end_dt.replace(tzinfo=timezone.utc)
+
+        return start_dt, end_dt
+
+    except ValueError:
+        print("Error: Invalid date format. Use ISO 8601 (e.g. 2025-01-01T00:00:00)", file=sys.stderr)
+        sys.exit(1)
+
+def validate_types(args):
+    types = [t.strip() for t in args.type.split(",") if t.strip() in ("data", "metadata")]
+    if not types:
+        print("Error: --type must contain at least one of 'data' or 'metadata'", file=sys.stderr)
+        sys.exit(1)
+    return types
+
+def validate_and_normalize_args(args, parser):
+    if args.command is None:
+        parser.print_help()
+        sys.exit(1)
+
+    maybe_warn_shell_globbing(args)
+    start_dt, end_dt = parse_datetime_args(args)
+    validate_datetime_range(start_dt, end_dt)
+    types = validate_types(args)
+
+    return start_dt, end_dt, types

dmtri-0.1.0.dist-info/METADATA

@@ -0,0 +1,174 @@
+Metadata-Version: 2.4
+Name: dmtri
+Version: 0.1.0
+Summary: CLI tool for triggering datacenter metadata updates
+Author-email: Nikos Sokos <nsokos@noa.com>
+Requires-Python: >=3.8
+Description-Content-Type: text/markdown
+Requires-Dist: ansible>=6.7.0
+Requires-Dist: ansible-runner>=2.3.6
+Requires-Dist: appdirs>=1.4.4
+Requires-Dist: pytest>=8.3.5
+Requires-Dist: pytest-cov>=5.0.0
+Requires-Dist: requests
+Requires-Dist: tomli>=2.2.1
+
+# dmtri
+
+[![Run Tests](https://github.com/EIDA/dmtri/actions/workflows/pytest.yml/badge.svg)](https://github.com/EIDA/dmtri/actions/workflows/pytest.yml)
+![Coverage](badges/coverage.svg)
+
+`dmtri` is a command-line tool designed to trigger metadata and data refresh simulations across EIDA nodes using Ansible automation.
+
+---
+- `seedpsd` 
+- `wfcatalog` 
+- `availability` 
+
+It uses `.dmtri_jobs.json` per host to track async job state and integrates with `uv` and `ansible` for CLI automation.
+
+---
+
+## Quick Start
+
+You can use `dmtri` in two ways, depending on your preference:
+
+### Option 1: Using `uvx`
+
+You can run commands using:
+
+```bash
+uvx dmtri <command> [options]
+
+```
+
+---
+
+### Option 2: Global (Tool-based) — Ideal for global CLI use
+
+```bash
+uv tool install dmtri
+```
+
+This makes `dmtri` available globally:
+
+```bash
+dmtri <command> [options]
+
+```
+
+You can update it any time using:
+
+```bash
+uv tool upgrade dmtri
+```
+
+---
+
+### 2. Configure inventory
+
+Edit the file: `src/inventory/hosts.ini`
+
+Example:
+
+```ini
+[seedpsd_nodes]
+<SEEDPSD_HOST> ansible_user=<USER> ansible_ssh_pass="<PASSWORD>" ansible_connection=ssh ansible_become=false ansible_ssh_common_args='-o StrictHostKeyChecking=no'
+
+[seedpsd_nodes:vars]
+seedpsd_dir=/home/<USER>/seedpsd
+
+[wf_catalogue]
+<WFCATALOG_HOST> ansible_user=<USER> ansible_ssh_pass="<PASSWORD>" ansible_connection=ssh ansible_become=false ansible_ssh_common_args='-o StrictHostKeyChecking=no'
+
+[wf_catalogue:vars]
+collector_dir=/home/<USER>/Programs/wfcatalogue<VERSION>/wfcatalog/collector2
+
+[ws_availability]
+<AVAILABILITY_HOST> ansible_user=<USER> ansible_ssh_pass="<PASSWORD>" ansible_connection=ssh ansible_become=false ansible_ssh_common_args='-o StrictHostKeyChecking=no'
+
+[ws_availability:vars]
+availability_dir=/home/<USER>/Programs/ws-availability<VERSION>/views
+mongo_user=<MONGO_USERNAME>
+mongo_pass=<MONGO_PASSWORD>
+mongo_authdb=<MONGO_DATABASE>
+
+```
+
+---
+
+### 3. Run example jobs
+
+By default, this command runs **all refresh playbooks** (seedpsd, wfcatalog, and availability):
+
+```bash
+uv run dmtri refresh -e "network=HL station=ATH starttime=2024-01-01T00:00:00 endtime=2024-01-02T00:00:00"
+```
+
+---
+
+### 4. Track jobs across hosts
+
+```bash
+uv run dmtri track
+```
+
+Shows per-host job status:
+```
+Job ID: j123... Type: seedpsd  Station: ATH  Status: ✅ Finished
+Job ID: manual-... Type: availability  Status: ✅ Manual (not tracked)
+```
+
+---
+
+### 5. Troubleshooting with `dmtri doctor`
+
+Check SSH connectivity to your configured hosts using:
+
+```bash
+uv run dmtri doctor
+```
+
+This uses the Ansible `ping` module to test access to all servers in your inventory.
+
+#### Options
+
+- `--verbose` — Show full output from each host
+
+#### Example
+
+```bash
+uv run dmtri doctor --verbose
+```
+
+If no response is received, ensure:
+
+- You're connected to the VPN (if applicable)
+- SSH access is configured correctly in your inventory file
+- Host IPs and credentials are reachable and valid
+
+###  Operation Details
+
+`dmtri` supports two main operations for both metadata and data: `refresh` and `clean`. These affect the following components:
+
+#### 🔄 Refresh
+
+- **Metadata**
+  - `seedpsd`: Scans for new metadata.
+  - `availability`: Refreshes the view in MongoDB.
+
+- **Data**
+  - Identifies data files in the archive for the specified epoch (by NSLC and time range).
+    - `seedpsd`: Recalculates seedPSD for new files.
+    - `wfcatalog`: Runs the collector on new data files.
+
+#### 🧹 Clean
+
+- **Metadata**
+  - `seedpsd`: Removes the metadata entries.
+  - `availability`: Refreshes the view in MongoDB.
+
+- **Data**
+  - Identifies data files in the archive for the specified epoch.
+    - `seedpsd`: Removes associated seedPSD data (via CLI).
+    - `wfcatalog`: Removes files using the `--delete` option in the collector.

dmtri-0.1.0.dist-info/RECORD

@@ -0,0 +1,13 @@
+dmtri/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+dmtri/cli.py,sha256=W8prY3t-1S2nBfgdrU3UkqFE321GtMwWQjh9E-Cudxc,4977
+dmtri/doctor.py,sha256=jZKJPKQEciPCvwAQu2WWpa92jcGq7QN7JE1VxSB-5SQ,1605
+dmtri/execution_plan.py,sha256=hVHwQ_nPUAicoAVIvl1sQBE56cTBYo7dBYnNwzqAtBs,1711
+dmtri/paths.py,sha256=UxR_TaFsdqfEdzD3uy1ox45gKf0ADhOhvzCcDJ5OM38,1477
+dmtri/utils.py,sha256=Z31wh0fOngyrDoqandkj1nwOVflSPUqMrWXF6-kO0nY,2750
+dmtri/hooks/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+dmtri/hooks/hooks.py,sha256=513anKlKihLaFen56qRGqj17pUfc_j6mEAJ_SQGkNgc,1197
+dmtri-0.1.0.dist-info/METADATA,sha256=dRdOjkNR0ML1_to5i5SwQCBdZsoSen_fceJhP1rtcjk,4206
+dmtri-0.1.0.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+dmtri-0.1.0.dist-info/entry_points.txt,sha256=GkiQuWx49WWaF2Lk838r-PYy9xib3YHTnU0avJWW80Y,41
+dmtri-0.1.0.dist-info/top_level.txt,sha256=qsH9klgTnBBHWZSzKqzLd8uIPNno_X-Uq34cBkqVyuE,6
+dmtri-0.1.0.dist-info/RECORD,,

dmtri-0.1.0.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (80.9.0)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

dmtri-0.1.0.dist-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+dmtri = dmtri.cli:main

dmtri-0.1.0.dist-info/top_level.txt

@@ -0,0 +1 @@
+dmtri