runledger-cli 0.1.0__py3-none-any.whl

cli/tests/test_data_features.py

@@ -0,0 +1,254 @@
+"""
+Feature showcase tests using test-data/sample-runs.
+Each test is labelled with a version number (v1, v2, v3) to reflect
+the order in which features were added.
+
+Run with:
+    python -m pytest cli/tests/test_data_features.py -v -s
+
+Assumes the RunLedger backend is already running (docker compose up -d).
+"""
+
+import json
+import subprocess
+import time
+import uuid
+from pathlib import Path
+import tempfile
+
+import pytest
+import requests
+
+# ------------------------------------------------------------
+# Configuration
+# ------------------------------------------------------------
+PROJECT_ROOT = Path(__file__).parent.parent.parent   # run-ledger/
+HEALTH_URL = "http://localhost:8080/actuator/health"
+API_BASE = "http://localhost:8080/api/runs"
+SAMPLE_FOLDER = PROJECT_ROOT / "test-data" / "sample-runs"
+# Unique batch per session to avoid cross‑test interference
+BATCH_NAME = f"feature-test-{uuid.uuid4().hex[:8]}"
+
+# ------------------------------------------------------------
+# Helper: run the CLI script
+# ------------------------------------------------------------
+def run_cli(*args):
+    cmd = ["python", str(PROJECT_ROOT / "cli" / "ledger.py"), *args]
+    return subprocess.run(cmd, capture_output=True, encoding='utf-8', cwd=str(PROJECT_ROOT))
+
+# ------------------------------------------------------------
+# Ensure backend is available, clean batch, and ingest sample data
+# ------------------------------------------------------------
+@pytest.fixture(scope="session")
+def setup_data():
+    """Wait for backend, clear any old mapping for this batch, then scan."""
+    # Wait for backend health
+    for _ in range(30):
+        try:
+            resp = requests.get(HEALTH_URL, timeout=2)
+            if resp.status_code == 200:
+                break
+        except Exception:
+            pass
+        time.sleep(2)
+    else:
+        pytest.fail("Backend is not reachable. Start it with 'docker compose up -d'")
+
+    # Ingest sample runs into the fresh batch
+    result = run_cli("scan", str(SAMPLE_FOLDER), "--batch", BATCH_NAME)
+    assert result.returncode == 0, result.stderr
+    assert "Successfully ingested 9 run(s)" in result.stdout
+
+# ------------------------------------------------------------
+# v1 – Basic metric block search
+# ------------------------------------------------------------
+def test_v1_metric_discovery(setup_data):
+    """Metrics endpoint returns keys from the batch (via mapping)."""
+    result = run_cli("metrics", "--batch", BATCH_NAME)
+    assert result.returncode == 0
+    keys = json.loads(result.stdout)
+    # Keys present in the sample files
+    assert "accuracy" in keys
+    assert "loss" in keys
+    assert "f1_score" in keys
+    assert "experiment" in keys
+    print("✅ Metric discovery (batch) --- SUCCESS")
+
+def test_v1_numeric_block_search_dot_path(setup_data):
+    """Numeric greater-than search using explicit dot-path metrics.accuracy."""
+    # Only run4 and run5 have a metrics.accuracy field (0.91 and 0.93 > 0.8)
+    result = run_cli("search", "--metric", "metrics.accuracy", "--op", "gt",
+                     "--value", "0.8", "--batch", BATCH_NAME, "--json")
+    assert result.returncode == 0
+    data = json.loads(result.stdout)
+    assert data["totalElements"] == 2   # run4, run5
+    print(f"✅ metrics.accuracy > 0.8 --- {data['totalElements']} runs found")
+
+def test_v1_text_equality_search_root_key(setup_data):
+    """Text equality search on the top-level 'status' field."""
+    # run5.json has "status": "completed" at the root
+    result = run_cli("search", "--metric", "status", "--op", "eq",
+                     "--value", "completed", "--batch", BATCH_NAME, "--json")
+    assert result.returncode == 0
+    data = json.loads(result.stdout)
+    assert data["totalElements"] == 1
+    assert data["content"][0]["payload"]["experiment"] == "deep_config"
+    print("✅ status = completed (root key) --- 1 run found")
+
+# ------------------------------------------------------------
+# v2 – Generalised path, shorthand keys, batch isolation
+# ------------------------------------------------------------
+def test_v2_dot_path_search(setup_data):
+    """Explicit dot‑path works for deeply nested fields."""
+    result = run_cli("search", "--metric", "config.optimizer.settings.learning_rate",
+                     "--op", "eq", "--value", "0.0001", "--batch", BATCH_NAME, "--json")
+    assert result.returncode == 0
+    data = json.loads(result.stdout)
+    # run5 has learning_rate = 0.0001
+    assert data["totalElements"] == 1
+    print("✅ config.optimizer.settings.learning_rate = 0.0001 --- 1 run found")
+
+def test_v2_shorthand_key_resolution_shallowest_wins(setup_data):
+    """Shorthand 'accuracy' resolves to the shallowest occurrence (root in run3.json)."""
+    result = run_cli("search", "--metric", "accuracy", "--op", "gt",
+                     "--value", "0.8", "--batch", BATCH_NAME, "--json")
+    assert result.returncode == 0
+    data = json.loads(result.stdout)
+    # Only run3 has a root-level accuracy (0.88)
+    assert data["totalElements"] == 1
+    assert data["content"][0]["payload"]["experiment"] == "simple_top_level"
+    print("✅ Shorthand 'accuracy' (shallowest wins) > 0.8 --- 1 run found")
+
+def test_v2_batch_isolation_empty(setup_data):
+    """Searching a different batch returns no results."""
+    result = run_cli("search", "--metric", "accuracy", "--op", "gt",
+                     "--value", "0.5", "--batch", "nonexistent-batch", "--json")
+    assert result.returncode == 0
+    data = json.loads(result.stdout)
+    assert data["totalElements"] == 0
+    print("✅ accuracy > 0.5 in non‑existent batch --- 0 runs (correct)")
+
+# ------------------------------------------------------------
+# v3 – Full‑text phrase search, fuzzy search, summary output
+# ------------------------------------------------------------
+def test_v3_fulltext_phrase_search(setup_data):
+    """Full‑text search finds a phrase inside notes (using stemming)."""
+    # run1 notes contain "calibration samples"
+    result = run_cli("search", "--q", "calibration", "--batch", BATCH_NAME, "--json")
+    assert result.returncode == 0
+    data = json.loads(result.stdout)
+    # Both run1 and run2 mention "calibration"
+    assert data["totalElements"] >= 1, "Expected at least one run with 'calibration'"
+    print(f"✅ Full‑text: 'calibration' --- {data['totalElements']} runs found")
+
+def test_v3_fuzzy_search(setup_data):
+    """Fuzzy search with a typo still finds the original word."""
+    # "calibraation" is a misspelling of "calibration"
+    result = run_cli("search", "--q", "calibraation", "--fuzzy",
+                     "--batch", BATCH_NAME, "--json")
+    assert result.returncode == 0
+    data = json.loads(result.stdout)
+    assert data["totalElements"] >= 1, "Fuzzy search should find runs with 'calibration'"
+    print(f"✅ Fuzzy: 'calibraation' --- {data['totalElements']} runs found")
+
+def test_v3_summary_output(setup_data):
+    """Summary flag returns only IDs and source files."""
+    # Use dot-path to metrics.accuracy to get results
+    result = run_cli("search", "--metric", "metrics.accuracy", "--op", "gt",
+                     "--value", "0.9", "--batch", BATCH_NAME, "--summary")
+    assert result.returncode == 0
+    stdout = result.stdout
+    # Should find run4 and run5
+    assert "duplicate_keys" in stdout or "deep_config" in stdout, "Summary should list at least one source file"
+    print("✅ metrics.accuracy > 0.9 (summary) --- table displayed")
+
+
+# ------------------------------------------------------------
+# v4 – Array‑aware search (new)
+# ------------------------------------------------------------
+def test_array_basic_search(setup_data):
+    """Search inside an array: client.results[].threshold > 0.96"""
+    result = run_cli("search", "--metric", "client.results[].threshold",
+                     "--op", "gt", "--value", "0.96",
+                     "--batch", BATCH_NAME, "--json")
+    assert result.returncode == 0, result.stderr
+    data = json.loads(result.stdout)
+    # run7 has 0.99, run8 has 0.99 (plus non‑numeric values safely ignored)
+    assert data["totalElements"] == 2
+    print("✅ Array basic search (threshold > 0.96) --- 2 runs found")
+
+def test_array_text_search(setup_data):
+    """Search inside an array: client.results[].status = 'error'"""
+    result = run_cli("search", "--metric", "client.results[].status",
+                     "--op", "eq", "--value", "error",
+                     "--batch", BATCH_NAME, "--json")
+    assert result.returncode == 0, result.stderr
+    data = json.loads(result.stdout)
+    # Only run8 has status = "error"
+    assert data["totalElements"] == 1
+    print("✅ Array text search (status = 'error') --- 1 run found")
+
+def test_array_stress_nested_search(setup_data):
+    """Search inside nested arrays: phases[].calibration.points[].threshold > 0.97"""
+    result = run_cli("search", "--metric", "phases[].calibration.points[].threshold",
+                     "--op", "gt", "--value", "0.97",
+                     "--batch", BATCH_NAME, "--json")
+    assert result.returncode == 0, result.stderr
+    data = json.loads(result.stdout)
+    # run9 has 0.99 (phase1) and 0.98 (phase2) – both > 0.97
+    assert data["totalElements"] == 1
+    print("✅ Array stress nested search (threshold > 0.97) --- 1 run found")
+
+def test_array_mixed_types_safe(setup_data):
+    """Search inside an array with mixed types: client.results[].threshold > 0.7"""
+    result = run_cli("search", "--metric", "client.results[].threshold",
+                     "--op", "gt", "--value", "0.7",
+                     "--batch", BATCH_NAME, "--json")
+    assert result.returncode == 0, result.stderr
+    data = json.loads(result.stdout)
+    # run7 (0.99,0.95) and run8 (0.99,'none',0.5) – non‑numeric safely ignored
+    assert data["totalElements"] == 2
+    print("✅ Array mixed‑type safety (threshold > 0.7) --- 2 runs found")
+
+# ------------------------------------------------------------
+# JSONL ingestion
+# ------------------------------------------------------------
+def test_jsonl_ingestion(setup_data):
+    """Ingest a JSONL file line by line and verify correct _source metadata."""
+    batch = f"jsonl-test-{uuid.uuid4().hex[:8]}"
+
+    with tempfile.TemporaryDirectory() as tmpdir:
+        # Create a JSONL file with 3 records
+        jl_file = Path(tmpdir) / "steps.jsonl"
+        lines = [
+            '{"step": 0, "loss": 0.95}',
+            '{"step": 1, "loss": 0.82}',
+            '{"step": 2, "loss": 0.67}',
+        ]
+        jl_file.write_text("\n".join(lines), encoding="utf-8")
+
+        # Scan with --jsonl
+        result = run_cli("scan", str(tmpdir), "--batch", batch, "--jsonl")
+        assert result.returncode == 0, f"scan failed: {result.stderr}"
+
+        # Verify via API
+        resp = requests.get(API_BASE, params={"batch": batch, "size": 50})
+        assert resp.status_code == 200, f"API error: {resp.text}"
+        data = resp.json()
+        content = data.get("content", [])
+        assert len(content) == 3, f"Expected 3 runs, got {len(content)}"
+
+        # Check _source metadata
+        for run in content:
+            payload = run.get("payload", {})
+            src = payload.get("_source", {})
+            assert src.get("file") == "steps.jsonl", f"Unexpected file: {src}"
+            idx = src.get("index")
+            assert idx is not None and 0 <= idx <= 2, f"Index out of range: {idx}"
+            # Verify the actual data lines up
+            expected = json.loads(lines[idx])
+            for k, v in expected.items():
+                assert payload.get(k) == v, f"Mismatch at line {idx}: {k} {v} != {payload.get(k)}"
+
+        # Clean up the temporary batch
+        requests.delete(f"{API_BASE}/batch/{batch}")  # if this endpoint exists, otherwise manual cleanup

cli/tests/test_integration.py

@@ -0,0 +1,177 @@
+import json
+import subprocess
+import tempfile
+import time
+import uuid
+import os
+from pathlib import Path
+
+import pytest
+import requests
+
+PROJECT_ROOT = Path(__file__).parent.parent.parent
+COMPOSE_FILE = PROJECT_ROOT / "docker-compose.yml"
+HEALTH_URL = "http://localhost:8080/actuator/health"
+API_BASE = "http://localhost:8080/api/runs"
+
+def wait_for_health(timeout=120):
+    start = time.time()
+    while time.time() - start < timeout:
+        try:
+            resp = requests.get(HEALTH_URL, timeout=2)
+            if resp.status_code == 200:
+                return True
+        except Exception:
+            pass
+        time.sleep(2)
+    return False
+
+@pytest.fixture(scope="session")
+def docker_stack():
+    """Start the Docker Compose stack and wait for it to be healthy."""
+    subprocess.run(
+        ["docker", "compose", "-f", str(COMPOSE_FILE), "up", "-d", "--build"],
+        cwd=str(PROJECT_ROOT),
+        check=True,
+        capture_output=True,
+    )
+    if not wait_for_health():
+        subprocess.run(
+            ["docker", "compose", "-f", str(COMPOSE_FILE), "down"],
+            cwd=str(PROJECT_ROOT),
+            capture_output=True,
+        )
+        pytest.fail("Backend did not become healthy in time")
+    yield
+    subprocess.run(
+        ["docker", "compose", "-f", str(COMPOSE_FILE), "down"],
+        cwd=str(PROJECT_ROOT),
+        capture_output=True,
+    )
+
+def run_cli(*args):
+    cmd = ["python", str(PROJECT_ROOT / "cli" / "ledger.py"), *args]
+    return subprocess.run(cmd, capture_output=True, encoding='utf-8', cwd=str(PROJECT_ROOT))
+
+@pytest.fixture
+def sample_folder():
+    """Create a temp folder with valid and corrupt JSON files."""
+    with tempfile.TemporaryDirectory() as tmpdir:
+        folder = Path(tmpdir)
+        (folder / "run1.json").write_text(json.dumps({
+            "experiment": "exp1", "metrics": {"accuracy": 0.95, "loss": 0.1}
+        }))
+        (folder / "run2.json").write_text(json.dumps({
+            "experiment": "exp2", "metrics": {"accuracy": 0.80, "loss": 0.2}
+        }))
+        (folder / "batch.json").write_text(json.dumps([
+            {"experiment": "exp3", "metrics": {"f1": 0.88}},
+            {"experiment": "exp4", "metrics": {"f1": 0.92}}
+        ]))
+        (folder / "bad.json").write_text("this is not json")
+        yield folder
+
+pytestmark = pytest.mark.integration
+
+def test_full_golden_path(docker_stack, sample_folder):
+    batch = f"golden-{uuid.uuid4()}"
+
+    # 1. scan folder (non‑interactive)
+    result = run_cli("scan", str(sample_folder), "--batch", batch)
+    assert result.returncode == 0, result.stderr
+    assert "Successfully ingested 4 run(s)" in result.stdout
+    assert f"'{batch}'" in result.stdout
+
+    # 2. metrics
+    result = run_cli("metrics", "--batch", batch)
+    assert result.returncode == 0, result.stderr
+    metrics = json.loads(result.stdout)
+    assert set(metrics).issuperset({"accuracy", "loss", "f1"})
+
+    # 3. search: accuracy > 0.9
+    result = run_cli("search", "--metric", "accuracy", "--op", "gt", "--value", "0.9", "--batch", batch, "--json")
+    assert result.returncode == 0, result.stderr
+    data = json.loads(result.stdout)
+    assert data["totalElements"] == 1
+    run = data["content"][0]
+    assert run["payload"]["experiment"] == "exp1"
+    assert run["payload"]["metrics"]["accuracy"] == 0.95
+    assert "_source" in run["payload"], "source metadata missing"
+    assert run["payload"]["_source"]["file"] == "run1.json"
+
+    # 4. search: loss < 0.15
+    result = run_cli("search", "--metric", "loss", "--op", "lt", "--value", "0.15", "--batch", batch, "--json")
+    assert result.returncode == 0, result.stderr
+    data = json.loads(result.stdout)
+    assert data["totalElements"] == 1
+    assert data["content"][0]["payload"]["experiment"] == "exp1"
+
+    # 5. status
+    result = run_cli("status")
+    assert result.returncode == 0, result.stderr
+    assert "running" in result.stdout.lower()
+
+    # 6. deep API verification
+    resp = requests.get(API_BASE, params={"batch": batch})
+    assert resp.status_code == 200
+    runs = resp.json()["content"]
+    assert len(runs) == 4
+    ids = [r["id"] for r in runs]
+    assert len(set(ids)) == 4, "IDs must be unique"
+    exp_names = {r["payload"]["experiment"] for r in runs}
+    assert exp_names == {"exp1", "exp2", "exp3", "exp4"}
+    for r in runs:
+        assert r["payload"]["_source"]["file"] in ["run1.json", "run2.json", "batch.json"]
+        assert r["createdAt"] is not None
+    # Verify ordering (descending by createdAt)
+    timestamps = [r["createdAt"] for r in runs]
+    assert timestamps == sorted(timestamps, reverse=True), "runs must be ordered by createdAt desc"
+
+def test_scan_nonexistent_folder(docker_stack):
+    result = run_cli("scan", "/no/such/path")
+    assert result.returncode != 0 or "Folder not found" in result.stdout
+
+def test_scan_corrupt_file_skipped(docker_stack, sample_folder):
+    batch = f"corrupt-{uuid.uuid4()}"
+    result = run_cli("scan", str(sample_folder), "--batch", batch)
+    assert result.returncode == 0, result.stderr
+    assert "Successfully ingested 4 run(s)" in result.stdout
+
+def test_metrics_empty_batch(docker_stack):
+    result = run_cli("metrics", "--batch", f"nonexistent-{uuid.uuid4()}")
+    assert result.returncode == 0, result.stderr
+    assert json.loads(result.stdout) == []
+
+def test_search_no_results(docker_stack, sample_folder):
+    batch = f"noresults-{uuid.uuid4()}"
+    run_cli("scan", str(sample_folder), "--batch", batch)
+    result = run_cli("search", "--metric", "accuracy", "--op", "gt", "--value", "0.99", "--batch", batch, "--json")
+    assert result.returncode == 0, result.stderr
+    data = json.loads(result.stdout)
+    assert data["totalElements"] == 0
+    assert data["content"] == []
+
+def test_multi_filter_and_returns_results():
+    # Ensure the compound test data folder exists
+    compound_dir = "test-data/sample-compound-runs"
+    if not os.path.isdir(compound_dir):
+        pytest.skip(f"Folder {compound_dir} not found")
+
+    # Scan the folder with explicit UTF-8 encoding to avoid UnicodeDecodeError
+    scan_result = subprocess.run(
+        ["python", "cli/ledger.py", "scan", compound_dir, "--batch", "compound-int"],
+        capture_output=True, text=True, encoding="utf-8", errors="replace"
+    )
+    assert scan_result.returncode == 0, f"Scan failed: {scan_result.stderr}"
+
+    # Search with AND
+    result = subprocess.run(
+        ["python", "cli/ledger.py", "search", "--metric", "accuracy", "--op", "gt", "--value", "0.9",
+         "--metric", "loss", "--op", "lt", "--value", "0.2", "--combine", "and", "--batch", "compound-int"],
+        capture_output=True, text=True, encoding="utf-8", errors="replace"
+    )
+    assert result.returncode == 0, f"Search failed: {result.stderr}"
+    assert result.stdout is not None, "stdout is None (decoding issue)"
+
+    # compound_run1.json has accuracy=0.95, loss=0.15 → should match
+    assert "compound_run1" in result.stdout, f"Output: {result.stdout}"