dmtri 0.1.0__tar.gz

dmtri-0.1.0/PKG-INFO

@@ -0,0 +1,174 @@
+Metadata-Version: 2.4
+Name: dmtri
+Version: 0.1.0
+Summary: CLI tool for triggering datacenter metadata updates
+Author-email: Nikos Sokos <nsokos@noa.com>
+Requires-Python: >=3.8
+Description-Content-Type: text/markdown
+Requires-Dist: ansible>=6.7.0
+Requires-Dist: ansible-runner>=2.3.6
+Requires-Dist: appdirs>=1.4.4
+Requires-Dist: pytest>=8.3.5
+Requires-Dist: pytest-cov>=5.0.0
+Requires-Dist: requests
+Requires-Dist: tomli>=2.2.1
+
+# dmtri
+
+[![Run Tests](https://github.com/EIDA/dmtri/actions/workflows/pytest.yml/badge.svg)](https://github.com/EIDA/dmtri/actions/workflows/pytest.yml)
+![Coverage](badges/coverage.svg)
+
+`dmtri` is a command-line tool designed to trigger metadata and data refresh simulations across EIDA nodes using Ansible automation.
+
+---
+- `seedpsd` 
+- `wfcatalog` 
+- `availability` 
+
+It uses `.dmtri_jobs.json` per host to track async job state and integrates with `uv` and `ansible` for CLI automation.
+
+---
+
+## Quick Start
+
+You can use `dmtri` in two ways, depending on your preference:
+
+### Option 1: Using `uvx`
+
+You can run commands using:
+
+```bash
+uvx dmtri <command> [options]
+
+```
+
+---
+
+### Option 2: Global (Tool-based) — Ideal for global CLI use
+
+```bash
+uv tool install dmtri
+```
+
+This makes `dmtri` available globally:
+
+```bash
+dmtri <command> [options]
+
+```
+
+You can update it any time using:
+
+```bash
+uv tool upgrade dmtri
+```
+
+---
+
+### 2. Configure inventory
+
+Edit the file: `src/inventory/hosts.ini`
+
+Example:
+
+```ini
+[seedpsd_nodes]
+<SEEDPSD_HOST> ansible_user=<USER> ansible_ssh_pass="<PASSWORD>" ansible_connection=ssh ansible_become=false ansible_ssh_common_args='-o StrictHostKeyChecking=no'
+
+[seedpsd_nodes:vars]
+seedpsd_dir=/home/<USER>/seedpsd
+
+[wf_catalogue]
+<WFCATALOG_HOST> ansible_user=<USER> ansible_ssh_pass="<PASSWORD>" ansible_connection=ssh ansible_become=false ansible_ssh_common_args='-o StrictHostKeyChecking=no'
+
+[wf_catalogue:vars]
+collector_dir=/home/<USER>/Programs/wfcatalogue<VERSION>/wfcatalog/collector2
+
+[ws_availability]
+<AVAILABILITY_HOST> ansible_user=<USER> ansible_ssh_pass="<PASSWORD>" ansible_connection=ssh ansible_become=false ansible_ssh_common_args='-o StrictHostKeyChecking=no'
+
+[ws_availability:vars]
+availability_dir=/home/<USER>/Programs/ws-availability<VERSION>/views
+mongo_user=<MONGO_USERNAME>
+mongo_pass=<MONGO_PASSWORD>
+mongo_authdb=<MONGO_DATABASE>
+
+```
+
+---
+
+### 3. Run example jobs
+
+By default, this command runs **all refresh playbooks** (seedpsd, wfcatalog, and availability):
+
+```bash
+uv run dmtri refresh -e "network=HL station=ATH starttime=2024-01-01T00:00:00 endtime=2024-01-02T00:00:00"
+```
+
+---
+
+### 4. Track jobs across hosts
+
+```bash
+uv run dmtri track
+```
+
+Shows per-host job status:
+```
+Job ID: j123... Type: seedpsd  Station: ATH  Status: ✅ Finished
+Job ID: manual-... Type: availability  Status: ✅ Manual (not tracked)
+```
+
+---
+
+### 5. Troubleshooting with `dmtri doctor`
+
+Check SSH connectivity to your configured hosts using:
+
+```bash
+uv run dmtri doctor
+```
+
+This uses the Ansible `ping` module to test access to all servers in your inventory.
+
+#### Options
+
+- `--verbose` — Show full output from each host
+
+#### Example
+
+```bash
+uv run dmtri doctor --verbose
+```
+
+If no response is received, ensure:
+
+- You're connected to the VPN (if applicable)
+- SSH access is configured correctly in your inventory file
+- Host IPs and credentials are reachable and valid
+
+###  Operation Details
+
+`dmtri` supports two main operations for both metadata and data: `refresh` and `clean`. These affect the following components:
+
+#### 🔄 Refresh
+
+- **Metadata**
+  - `seedpsd`: Scans for new metadata.
+  - `availability`: Refreshes the view in MongoDB.
+
+- **Data**
+  - Identifies data files in the archive for the specified epoch (by NSLC and time range).
+    - `seedpsd`: Recalculates seedPSD for new files.
+    - `wfcatalog`: Runs the collector on new data files.
+
+#### 🧹 Clean
+
+- **Metadata**
+  - `seedpsd`: Removes the metadata entries.
+  - `availability`: Refreshes the view in MongoDB.
+
+- **Data**
+  - Identifies data files in the archive for the specified epoch.
+    - `seedpsd`: Removes associated seedPSD data (via CLI).
+    - `wfcatalog`: Removes files using the `--delete` option in the collector.

dmtri-0.1.0/README.md

@@ -0,0 +1,159 @@
+# dmtri
+
+[![Run Tests](https://github.com/EIDA/dmtri/actions/workflows/pytest.yml/badge.svg)](https://github.com/EIDA/dmtri/actions/workflows/pytest.yml)
+![Coverage](badges/coverage.svg)
+
+`dmtri` is a command-line tool designed to trigger metadata and data refresh simulations across EIDA nodes using Ansible automation.
+
+---
+- `seedpsd` 
+- `wfcatalog` 
+- `availability` 
+
+It uses `.dmtri_jobs.json` per host to track async job state and integrates with `uv` and `ansible` for CLI automation.
+
+---
+
+## Quick Start
+
+You can use `dmtri` in two ways, depending on your preference:
+
+### Option 1: Using `uvx`
+
+You can run commands using:
+
+```bash
+uvx dmtri <command> [options]
+
+```
+
+---
+
+### Option 2: Global (Tool-based) — Ideal for global CLI use
+
+```bash
+uv tool install dmtri
+```
+
+This makes `dmtri` available globally:
+
+```bash
+dmtri <command> [options]
+
+```
+
+You can update it any time using:
+
+```bash
+uv tool upgrade dmtri
+```
+
+---
+
+### 2. Configure inventory
+
+Edit the file: `src/inventory/hosts.ini`
+
+Example:
+
+```ini
+[seedpsd_nodes]
+<SEEDPSD_HOST> ansible_user=<USER> ansible_ssh_pass="<PASSWORD>" ansible_connection=ssh ansible_become=false ansible_ssh_common_args='-o StrictHostKeyChecking=no'
+
+[seedpsd_nodes:vars]
+seedpsd_dir=/home/<USER>/seedpsd
+
+[wf_catalogue]
+<WFCATALOG_HOST> ansible_user=<USER> ansible_ssh_pass="<PASSWORD>" ansible_connection=ssh ansible_become=false ansible_ssh_common_args='-o StrictHostKeyChecking=no'
+
+[wf_catalogue:vars]
+collector_dir=/home/<USER>/Programs/wfcatalogue<VERSION>/wfcatalog/collector2
+
+[ws_availability]
+<AVAILABILITY_HOST> ansible_user=<USER> ansible_ssh_pass="<PASSWORD>" ansible_connection=ssh ansible_become=false ansible_ssh_common_args='-o StrictHostKeyChecking=no'
+
+[ws_availability:vars]
+availability_dir=/home/<USER>/Programs/ws-availability<VERSION>/views
+mongo_user=<MONGO_USERNAME>
+mongo_pass=<MONGO_PASSWORD>
+mongo_authdb=<MONGO_DATABASE>
+
+```
+
+---
+
+### 3. Run example jobs
+
+By default, this command runs **all refresh playbooks** (seedpsd, wfcatalog, and availability):
+
+```bash
+uv run dmtri refresh -e "network=HL station=ATH starttime=2024-01-01T00:00:00 endtime=2024-01-02T00:00:00"
+```
+
+---
+
+### 4. Track jobs across hosts
+
+```bash
+uv run dmtri track
+```
+
+Shows per-host job status:
+```
+Job ID: j123... Type: seedpsd  Station: ATH  Status: ✅ Finished
+Job ID: manual-... Type: availability  Status: ✅ Manual (not tracked)
+```
+
+---
+
+### 5. Troubleshooting with `dmtri doctor`
+
+Check SSH connectivity to your configured hosts using:
+
+```bash
+uv run dmtri doctor
+```
+
+This uses the Ansible `ping` module to test access to all servers in your inventory.
+
+#### Options
+
+- `--verbose` — Show full output from each host
+
+#### Example
+
+```bash
+uv run dmtri doctor --verbose
+```
+
+If no response is received, ensure:
+
+- You're connected to the VPN (if applicable)
+- SSH access is configured correctly in your inventory file
+- Host IPs and credentials are reachable and valid
+
+###  Operation Details
+
+`dmtri` supports two main operations for both metadata and data: `refresh` and `clean`. These affect the following components:
+
+#### 🔄 Refresh
+
+- **Metadata**
+  - `seedpsd`: Scans for new metadata.
+  - `availability`: Refreshes the view in MongoDB.
+
+- **Data**
+  - Identifies data files in the archive for the specified epoch (by NSLC and time range).
+    - `seedpsd`: Recalculates seedPSD for new files.
+    - `wfcatalog`: Runs the collector on new data files.
+
+#### 🧹 Clean
+
+- **Metadata**
+  - `seedpsd`: Removes the metadata entries.
+  - `availability`: Refreshes the view in MongoDB.
+
+- **Data**
+  - Identifies data files in the archive for the specified epoch.
+    - `seedpsd`: Removes associated seedPSD data (via CLI).
+    - `wfcatalog`: Removes files using the `--delete` option in the collector.

dmtri-0.1.0/pyproject.toml

@@ -0,0 +1,28 @@
+[project]
+name = "dmtri"
+version = "0.1.0"
+description = "CLI tool for triggering datacenter metadata updates"
+authors = [{ name = "Nikos Sokos", email = "nsokos@noa.com" }]
+dependencies = [
+    "ansible>=6.7.0",
+    "ansible-runner>=2.3.6",
+    "appdirs>=1.4.4",
+    "pytest>=8.3.5",
+    "pytest-cov>=5.0.0",
+    "requests",
+    "tomli>=2.2.1",
+]
+readme = "README.md"
+requires-python = ">=3.8"
+[tool.uv]
+dev-dependencies = [
+  "pytest",
+  "pytest-cov"
+]
+
+
+[project.scripts]
+dmtri = "dmtri.cli:main"
+[build-system]
+requires = ["setuptools>=62.0", "wheel"]
+build-backend = "setuptools.build_meta"

dmtri-0.1.0/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+

dmtri-0.1.0/src/dmtri/cli.py

@@ -0,0 +1,118 @@
+import argparse
+import sys
+import os
+from datetime import datetime, timezone
+import logging
+
+from dmtri.execution_plan import print_execution_plan
+from dmtri.utils import get_version, validate_and_normalize_args
+from dmtri.hooks.hooks import run_hook
+from dmtri.paths import COMMAND_PLAYBOOKS
+
+logging.basicConfig(level=logging.INFO, format="%(message)s")
+logger = logging.getLogger(__name__)
+
+PROJECT_URL = "https://github.com/EIDA/dmtri/issues"
+
+def main():
+    global_options = argparse.ArgumentParser(add_help=False)
+
+    parser = argparse.ArgumentParser(
+        description="Trigger data/metadata refresh or cleaning for multiple EIDA endpoints.",
+        parents=[global_options]
+    )
+    parser.add_argument(
+        "--version",
+        action="version",
+        version=f"%(prog)s version {get_version()} — See {PROJECT_URL} for updates or to report issues."
+    )
+
+    subparsers = parser.add_subparsers(dest="command")
+
+    shared = argparse.ArgumentParser(add_help=False, parents=[global_options])
+    shared.add_argument("-n", "--network", nargs="+", default=["*"], help="FDSN network code(s), supports wildcards")
+    shared.add_argument("-s", "--station", nargs="+", default=["*"], help="FDSN station code(s), supports wildcards")
+    shared.add_argument("-l", "--location", nargs="+", default=["*"], help="FDSN location code(s), supports wildcards")
+    shared.add_argument("-c", "--channel", nargs="+", default=["*"], help="FDSN channel code(s), supports wildcards")
+    shared.add_argument("-S", "--starttime", required=True, help="Start time (ISO 8601 format)")
+
+    now_str = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%S")
+    shared.add_argument("-E", "--endtime", default=now_str, help="End time (default: now in UTC)")
+    shared.add_argument("--type", default="data,metadata", help="Comma-separated types to process (data, metadata)")
+    shared.add_argument("--no-confirm", action="store_true", help="Skip confirmation prompt before executing")
+    shared.add_argument("--debug", action="store_true", help="Show verbose output from Ansible")
+
+    subparsers.add_parser("refresh", parents=[shared], help="Refresh data/metadata using configured playbooks.")
+    subparsers.add_parser("clean", parents=[shared], help="Clean outdated data/metadata using configured playbooks.")
+    subparsers.add_parser("track", help="Track job status via tracking playbook.")
+    doctor_parser = subparsers.add_parser("doctor", help="Check SSH connectivity to all inventory hosts")
+    doctor_parser.add_argument("-v", "--verbose", action="store_true", help="Show full Ansible output per host")
+
+    args = parser.parse_args()
+    if args.command == "track":
+        cfg_file = "ansible_verbose.cfg"
+    else:
+        cfg_file = "ansible_verbose.cfg" if getattr(args, "debug", False) else "ansible_quiet.cfg"
+
+    cfg_path = os.path.abspath(cfg_file)
+    os.environ["ANSIBLE_CONFIG"] = cfg_path
+
+    if args.command == "doctor":
+        from dmtri.doctor import run_diagnostics
+        return run_diagnostics(verbose=args.verbose)
+
+    elif args.command in ("refresh", "clean"):
+        start_dt, end_dt, types = validate_and_normalize_args(args, parser)
+
+        vars_to_pass = {
+            "network": args.network,
+            "station": args.station,
+            "location": args.location,
+            "channel": args.channel,
+            "starttime": args.starttime,
+            "endtime": args.endtime,
+            "type": [t.strip() for t in args.type.split(",")],
+            "debug": args.debug,
+        }
+
+        if args.command == "refresh":
+            selected_types = vars_to_pass["type"]
+            valid_types = set(COMMAND_PLAYBOOKS["refresh"].keys())
+
+            invalid = set(selected_types) - valid_types
+            if invalid:
+                print(f" Invalid --type: {', '.join(invalid)}. Valid types are: {', '.join(valid_types)}.")
+                sys.exit(1)
+
+            playbooks = []
+            for t in selected_types:
+                playbooks.extend(COMMAND_PLAYBOOKS["refresh"][t])
+        else:
+            playbooks = COMMAND_PLAYBOOKS.get("clean", [])
+
+        if not playbooks:
+            print(f"No playbooks configured for command '{args.command}'.")
+            sys.exit(1)
+
+        if not args.no_confirm:
+            for pb in playbooks:
+                print_execution_plan(args.command, vars_to_pass, pb)
+            proceed = input("\nDo you want to proceed with executing all playbooks? (y/N): ").strip().lower()
+            if proceed != "y":
+                print("Aborted by user.")
+                sys.exit(0)
+
+        for pb in playbooks:
+            run_hook(pb, vars_to_pass)
+
+    elif args.command == "track":
+        playbooks = COMMAND_PLAYBOOKS.get("track", [])
+        if not playbooks:
+            print("No playbooks configured for command 'track'.")
+            sys.exit(1)
+        for pb in playbooks:
+            run_hook(pb, {})
+
+    else:
+        parser.print_help()
+        sys.exit(1)

dmtri-0.1.0/src/dmtri/doctor.py

@@ -0,0 +1,52 @@
+import logging
+import ansible_runner
+from dmtri.paths import INVENTORY_FILE
+
+logger = logging.getLogger(__name__)
+
+def run_diagnostics(verbose=False, retries=1):
+    logger.info(f"Running connectivity check using inventory: {INVENTORY_FILE}")
+
+    ok_hosts = set()
+    unreachable_hosts = set()
+
+    for attempt in range(retries):
+        logger.info(f"Attempt {attempt + 1} of {retries}")
+
+        r = ansible_runner.run(
+            private_data_dir=".",
+            inventory=str(INVENTORY_FILE.resolve()),
+            module="ping",
+            host_pattern="all",
+            quiet=not verbose,
+            envvars={
+                "ANSIBLE_CACHE_PLUGIN": "memory"  # Disable caching to avoid plugin warnings
+            }
+        )
+
+        for event in r.events:
+            if not isinstance(event, dict):
+                continue
+            data = event.get("event_data", {})
+            host = data.get("host")
+            if not host:
+                continue
+            if event["event"] == "runner_on_ok":
+                ok_hosts.add(host)
+            elif event["event"] == "runner_on_unreachable":
+                unreachable_hosts.add(host)
+
+        # Break early if all hosts responded
+        if ok_hosts and not unreachable_hosts:
+            break
+
+    logger.info("\nResults:")
+
+    if not ok_hosts and not unreachable_hosts:
+        logger.warning("No responses received. Are you connected to the VPN?")
+        return
+
+    for host in sorted(ok_hosts):
+        logger.info(f"[OK] {host}")
+    for host in sorted(unreachable_hosts):
+        logger.error(f"[UNREACHABLE] {host}")

dmtri-0.1.0/src/dmtri/execution_plan.py

@@ -0,0 +1,59 @@
+import sys
+import yaml
+from pathlib import Path
+
+
+def get_playbook_metadata(playbook_path: str):
+    try:
+        with open(playbook_path) as f:
+            playbook = yaml.safe_load(f)
+
+        if not playbook or not isinstance(playbook, list) or not playbook[0]:
+            return {"error": "Empty playbook"}
+
+        metadata = {
+            "hosts": playbook[0].get("hosts", "unknown"),
+            "description": playbook[0].get("vars", {}).get("description", ""),
+            "tasks": []
+        }
+
+        for task in playbook[0].get("tasks", []):
+            desc = task.get("vars", {}).get("description", task.get("name", "Unnamed Task"))
+            metadata["tasks"].append(desc)
+
+        return metadata
+
+    except Exception as e:
+        return {"error": str(e)}
+
+
+
+    
+
+def print_execution_plan(command: str, variables: dict, playbook_path: str, metadata=None):
+    if metadata is None:
+        metadata = get_playbook_metadata(playbook_path)
+
+    print("\nExecution Plan")
+    print("-" * 40)
+    print(f"Command     : {command}")
+    print(f"Playbook    : {playbook_path}")
+
+    if "error" in metadata:
+        print(f"Warning     : {metadata['error']}")
+    else:
+        print(f"Hosts       : {metadata['hosts']}")
+        print(f"Description : {metadata['description']}")
+        print("Steps       :")
+        for idx, desc in enumerate(metadata['tasks'], start=1):
+            print(f"  {idx}. {desc}")
+
+    if variables:
+        print("\nVariables:")
+        for k, v in variables.items():
+            print(f"  {k:10}: {v}")
+
+    confirm = input("\nProceed with this execution plan? [y/N]: ").strip().lower()
+    if confirm != "y":
+        print("Aborted by user.")
+        sys.exit(0)

dmtri-0.1.0/src/dmtri/hooks/hooks.py

@@ -0,0 +1,45 @@
+import ansible_runner
+import sys
+from pathlib import Path
+import logging
+from dmtri.paths import INVENTORY_FILE
+
+
+logger = logging.getLogger(__name__)
+def run_hook(playbook_name: Path, vars: dict,inventory: Path = INVENTORY_FILE) -> None:
+    """
+    Run an Ansible playbook using ansible_runner.run_command, bypassing the project layout.
+    """
+
+    # Build the extra vars string manually
+    extra_vars = []
+    for key, value in vars.items():
+        if isinstance(value, list):
+            value = ",".join(value)
+        extra_vars.extend(["-e", f"{key}='{value}'"])
+
+    cmd = [
+        "ansible-playbook",
+        str(playbook_name.resolve()),
+        "-i",
+        str(inventory.resolve()),
+        
+    ] + extra_vars
+
+
+    logger.info(f"Running: {' '.join(str(x) for x in cmd)}")
+
+
+    out, err, rc = ansible_runner.run_command(
+        executable_cmd="ansible-playbook",
+        cmdline_args=cmd[1:], 
+        input_fd=sys.stdin,
+        output_fd=sys.stdout,
+        error_fd=sys.stderr,
+    )
+
+    if rc != 0:
+        logger.error(f"Playbook {playbook_name} failed with return code {rc}")
+        sys.exit(rc)
+    
+    logger.info(f"Playbook {playbook_name} completed successfully")

dmtri-0.1.0/src/dmtri/paths.py

@@ -0,0 +1,49 @@
+from pathlib import Path
+
+MODULE_DIR = Path(__file__).resolve().parent
+SRC_DIR = MODULE_DIR.parent
+
+# Directories
+PLAYBOOKS_DIR = SRC_DIR / "playbooks"
+INVENTORY_DIR = SRC_DIR / "inventory"
+
+# Inventory file
+INVENTORY_FILE = INVENTORY_DIR / "hosts.ini"
+
+# Specific playbooks
+PLAYBOOK_TRACK = PLAYBOOKS_DIR / "track.yml"
+PLAYBOOK_LIST_SDS = PLAYBOOKS_DIR / "list_sds_files.yml"
+
+# Refresh Data playbooks
+PLAYBOOK_SEEDPSD_REFRESH = PLAYBOOKS_DIR / "refresh" / "seedpsd_refresh.yml"
+PLAYBOOK_WFCATALOG_REFRESH = PLAYBOOKS_DIR / "refresh" / "wfcatalog_refresh.yml"
+PLAYBOOK_AVAILABILITY_REFRESH = PLAYBOOKS_DIR / "refresh" / "availability_refresh.yml"
+# Refresh metadata playbooks
+PLAYBOOK_METADATA_REFRESH = PLAYBOOKS_DIR / "refresh" / "seedpsd_metadata_refresh.yml"
+
+# Clean playbooks
+PLAYBOOK_SEEDPSD_CLEAN = PLAYBOOKS_DIR / "clean" /"seedpsd_clean.yml"
+PLAYBOOK_WFCATALOG_CLEAN = PLAYBOOKS_DIR / "clean" / "wfcatalog_clean.yml"
+PLAYBOOK_AVAILABILITY_CLEAN = PLAYBOOKS_DIR / "clean" / "availability_clean.yml"
+
+# Group playbooks by command
+COMMAND_PLAYBOOKS = {
+    "refresh": {
+        "data": [
+            PLAYBOOK_SEEDPSD_REFRESH,
+            PLAYBOOK_WFCATALOG_REFRESH,
+            PLAYBOOK_AVAILABILITY_REFRESH,
+        ],
+        "metadata": [
+            PLAYBOOK_METADATA_REFRESH,
+        ]
+    },
+    "clean": [
+        PLAYBOOK_SEEDPSD_CLEAN,
+        PLAYBOOK_WFCATALOG_CLEAN,
+        PLAYBOOK_AVAILABILITY_CLEAN,
+    ],
+    "track": [
+        PLAYBOOK_TRACK
+    ]
+}