biller-cli 0.1.0__py3-none-any.whl

biller_cli/ai/ingest.py

@@ -0,0 +1,394 @@
+"""
+biller_cli/ai/ingest.py
+
+Chunks bbps_biller_integrator_codebase.md and writes embeddings to ChromaDB.
+
+Split strategy:
+  Pass 1 — split on file boundary markers (^## src/)
+  Pass 2 — split oversized chunks (>400 tokens) at method/blank boundaries
+            with 50-token overlap
+
+Exclusions — dead scaffolding deleted in Phase 0:
+  BillerController, UserService, UserServiceImpl,
+  UserDao, UserDaoImpl, User.java
+
+Run:
+  biller-cli ingest --source ~/Desktop/bbps/AIGateway/docs/bbps_biller_integrator_codebase.md
+"""
+
+from __future__ import annotations
+
+import re
+import subprocess
+import sys
+from pathlib import Path
+from typing import Generator
+
+import chromadb
+import tiktoken
+
+# ---------------------------------------------------------------------------
+# Constants
+# ---------------------------------------------------------------------------
+
+CHROMA_DIR = Path.home() / ".biller-cli" / "chroma"
+COLLECTION_NAME = "biller_codebase"
+EMBED_MODEL = "nomic-embed-text"
+MAX_TOKENS = 256
+OVERLAP_TOKENS = 32
+
+# Regex that matches file-level headers only.
+# Matches: ## src/main/java/... and ## src/main/resources/...
+# Does NOT match: ## Java Source Files, ## Resource Files, ## AGENT_REBUILD_GUIDE, etc.
+FILE_HEADER_RE = re.compile(r"^## (src/\S+)", re.MULTILINE)
+
+# Dead scaffolding — explicitly deleted in Phase 0.
+# These paths exist in the dump (generated from biller-audit, not the clean tree).
+# They must never be indexed.
+EXCLUDE_PATHS: frozenset[str] = frozenset(
+    {
+        "src/main/java/bharat/connect/biller/controller/BillerController.java",
+        "src/main/java/bharat/connect/biller/service/UserService.java",
+        "src/main/java/bharat/connect/biller/service/impl/UserServiceImpl.java",
+        "src/main/java/bharat/connect/biller/dao/UserDao.java",
+        "src/main/java/bharat/connect/biller/dao/impl/UserDaoImpl.java",
+        "src/main/java/bharat/connect/biller/model/User.java",
+    }
+)
+
+# tiktoken encoder — cl100k_base is accurate enough for token budgeting on Java.
+# It is NOT the nomic-embed-text tokeniser, but the counts are close enough for
+# a 400-token ceiling. Do not use len(text.split()) — that undercounts by ~30%.
+_ENC = tiktoken.get_encoding("cl100k_base")
+
+
+# ---------------------------------------------------------------------------
+# Pre-flight checks
+# ---------------------------------------------------------------------------
+
+
+def _check_ollama_model(model: str) -> None:
+    """Exit with an actionable error if the required Ollama model is not pulled."""
+    try:
+        result = subprocess.run(
+            ["ollama", "list"], capture_output=True, text=True, timeout=10
+        )
+        if model not in result.stdout:
+            print(
+                f"\nError: Ollama model '{model}' is not available.\n"
+                f"Pull it with:  ollama pull {model}\n"
+            )
+            sys.exit(1)
+    except FileNotFoundError:
+        print(
+            "\nError: Ollama is not installed or not on PATH.\n"
+            "Install from: https://ollama.com\n"
+            f"Then run:     ollama pull {model}\n"
+        )
+        sys.exit(1)
+    except subprocess.TimeoutExpired:
+        print(
+            "\nError: Ollama did not respond within 10 seconds.\n"
+            "Start it with: ollama serve\n"
+        )
+        sys.exit(1)
+
+
+# ---------------------------------------------------------------------------
+# Parsing — Pass 1
+# ---------------------------------------------------------------------------
+
+
+def _doc_type(file_path: str) -> str:
+    """Return 'source' for Java files, 'resource' for everything else."""
+    return "source" if file_path.endswith(".java") else "resource"
+
+
+def _parse_file_sections(text: str) -> Generator[tuple[str, str], None, None]:
+    """
+    Yield (file_path, content) pairs for every ## src/ section in the dump.
+
+    Skips sections whose file_path is in EXCLUDE_PATHS.
+    Content includes everything between the file header and the next ## src/ header
+    (or end of document), minus the header line itself.
+    """
+    matches = list(FILE_HEADER_RE.finditer(text))
+    for i, match in enumerate(matches):
+        file_path = match.group(1)
+
+        if file_path in EXCLUDE_PATHS:
+            continue
+
+        # Content runs from end of this header line to start of next ## src/ header.
+        content_start = match.end()
+        content_end = matches[i + 1].start() if i + 1 < len(matches) else len(text)
+        content = text[content_start:content_end].strip()
+
+        if content:
+            yield file_path, content
+
+
+# ---------------------------------------------------------------------------
+# Chunking — Pass 2
+# ---------------------------------------------------------------------------
+
+
+def _token_count(text: str) -> int:
+    return len(_ENC.encode(text))
+
+
+def _split_oversized(content: str, file_path: str) -> list[str]:
+    sub_chunks = []
+    for delimiter in (r"\}\n\n", r"\n\n"):
+        parts = re.split(delimiter, content)
+        if len(parts) > 1:
+            sub_chunks = _merge_parts(parts, file_path)
+            break
+    else:
+        sub_chunks = [content]
+
+    # Safety pass — force hard split on any chunk still over ceiling.
+    # Handles JAXB-generated classes with no natural delimiters.
+    final = []
+    for chunk in sub_chunks:
+        if _token_count(chunk) > MAX_TOKENS:
+            final.extend(_hard_split(chunk))
+        else:
+            final.append(chunk)
+    return final
+
+
+def _merge_parts(parts: list[str], file_path: str) -> list[str]:
+    """
+    Greedily merge split parts into chunks that stay under MAX_TOKENS.
+    When a chunk would exceed MAX_TOKENS, close it and start a new one
+    seeded with OVERLAP_TOKENS of the previous chunk's tail for context.
+    """
+    chunks: list[str] = []
+    current = ""
+
+    for part in parts:
+        candidate = (current + "\n\n" + part).strip() if current else part.strip()
+        if _token_count(candidate) <= MAX_TOKENS:
+            current = candidate
+        else:
+            if current:
+                chunks.append(current)
+                # Seed next chunk with overlap from tail of current.
+                tail_tokens = _ENC.encode(current)[-OVERLAP_TOKENS:]
+                overlap_text = _ENC.decode(tail_tokens)
+                current = (overlap_text + "\n\n" + part.strip()).strip()
+            else:
+                # Single part already exceeds MAX_TOKENS — keep it intact.
+                # Splitting a single method mid-line is worse than an oversized chunk.
+                chunks.append(part.strip())
+                current = ""
+
+    if current:
+        chunks.append(current)
+
+    return [c for c in chunks if c]
+
+
+def _hard_split(content: str) -> list[str]:
+    """Token-boundary split for content with no natural delimiters."""
+    tokens = _ENC.encode(content)
+    chunks: list[str] = []
+    step = MAX_TOKENS - OVERLAP_TOKENS
+    for start in range(0, len(tokens), step):
+        chunk_tokens = tokens[start : start + MAX_TOKENS]
+        chunks.append(_ENC.decode(chunk_tokens))
+    return chunks
+
+
+def _chunk_file(file_path: str, content: str) -> list[dict]:
+    """
+    Return a list of chunk dicts ready for ChromaDB insertion.
+
+    Each dict has keys: text, file_path, chunk_index, language, token_count, doc_type
+    """
+    token_count = _token_count(content)
+
+    if token_count <= MAX_TOKENS:
+        sub_chunks = [content]
+    else:
+        sub_chunks = _split_oversized(content, file_path)
+
+    language = "java" if file_path.endswith(".java") else (
+        "xml" if file_path.endswith(".xsd") else (
+            "sql" if file_path.endswith(".sql") else "properties"
+        )
+    )
+
+    result = []
+    for idx, chunk_text in enumerate(sub_chunks):
+        result.append(
+            {
+                "text": chunk_text,
+                "file_path": file_path,
+                "chunk_index": idx,
+                "language": language,
+                "token_count": _token_count(chunk_text),
+                "doc_type": _doc_type(file_path),
+            }
+        )
+    return result
+
+
+# ---------------------------------------------------------------------------
+# Embedding — via Ollama HTTP API
+# ---------------------------------------------------------------------------
+
+
+def _embed_batch(texts: list[str]) -> list[list[float]]:
+    """
+    Embed a batch of texts using nomic-embed-text via the Ollama Python library.
+    Returns a list of embedding vectors in the same order as input texts.
+    """
+    import ollama  # imported here so the rest of the module is importable without ollama
+
+    embeddings = []
+    for text in texts:
+        response = ollama.embeddings(model=EMBED_MODEL, prompt=text)
+        embeddings.append(response["embedding"])
+    return embeddings
+
+
+# ---------------------------------------------------------------------------
+# ChromaDB write
+# ---------------------------------------------------------------------------
+
+
+def _get_collection(client: chromadb.PersistentClient) -> chromadb.Collection:
+    """
+    Delete and recreate the collection on every ingest run.
+    This prevents duplicate chunk accumulation when re-ingesting
+    after a codebase update. No partial-update strategy — full rebuild only.
+    """
+    try:
+        client.delete_collection(COLLECTION_NAME)
+    except Exception:
+        pass  # Collection did not exist — first run.
+    return client.create_collection(
+        name=COLLECTION_NAME,
+        metadata={"hnsw:space": "cosine"},
+    )
+
+
+BATCH_SIZE = 50  # ChromaDB add() performance degrades with very large single batches.
+
+
+def _write_to_chroma(
+    collection: chromadb.Collection, chunks: list[dict]
+) -> None:
+    """Write all chunks to ChromaDB in batches."""
+    total = len(chunks)
+    for batch_start in range(0, total, BATCH_SIZE):
+        batch = chunks[batch_start : batch_start + BATCH_SIZE]
+        texts = [f"{c['file_path']}\n\n{c['text']}" for c in batch]
+        embeddings = _embed_batch(texts)
+        ids = [
+            f"{c['file_path']}::chunk_{c['chunk_index']}" for c in batch
+        ]
+        metadatas = [
+            {
+                "file_path": c["file_path"],
+                "chunk_index": c["chunk_index"],
+                "language": c["language"],
+                "token_count": c["token_count"],
+                "doc_type": c["doc_type"],
+            }
+            for c in batch
+        ]
+        collection.add(
+            ids=ids,
+            embeddings=embeddings,
+            documents=texts,
+            metadatas=metadatas,
+        )
+        done = min(batch_start + BATCH_SIZE, total)
+        print(f"  Embedded and stored {done}/{total} chunks...", end="\r")
+
+    print()  # newline after the progress line
+
+
+# ---------------------------------------------------------------------------
+# Public entry point
+# ---------------------------------------------------------------------------
+
+
+def run_ingest(source_path: Path) -> None:
+    """
+    Full ingest pipeline. Called by `biller-cli ingest --source <path>`.
+
+    Steps:
+      1. Pre-flight: verify Ollama + nomic-embed-text available
+      2. Read source file
+      3. Parse file sections (Pass 1)
+      4. Chunk oversized sections (Pass 2)
+      5. Write to ChromaDB (delete + recreate collection)
+    """
+    # --- 1. Pre-flight ---
+    print("Checking Ollama availability...")
+    _check_ollama_model(EMBED_MODEL)
+    print(f"  OK: '{EMBED_MODEL}' is available.\n")
+
+    # --- 2. Read source ---
+    if not source_path.exists():
+        print(f"Error: Source file not found: {source_path}")
+        sys.exit(1)
+
+    print(f"Reading: {source_path}")
+    text = source_path.read_text(encoding="utf-8")
+    print(f"  {len(text):,} characters loaded.\n")
+
+    # --- 3. Parse ---
+    print("Parsing file sections...")
+    all_chunks: list[dict] = []
+    excluded_count = 0
+    section_count = 0
+
+    for file_path, content in _parse_file_sections(text):
+        section_count += 1
+        file_chunks = _chunk_file(file_path, content)
+        all_chunks.extend(file_chunks)
+
+    # Count excluded sections separately for the summary.
+    for match in FILE_HEADER_RE.finditer(text):
+        if match.group(1) in EXCLUDE_PATHS:
+            excluded_count += 1
+
+    print(f"  {section_count} sections ingested, {excluded_count} excluded (dead scaffolding).")
+    print(f"  {len(all_chunks)} total chunks after Pass 2 splitting.\n")
+
+    if not all_chunks:
+        print("Error: No chunks produced. Verify the source file format.")
+        print("Expected headers matching: ## src/<path>")
+        sys.exit(1)
+
+    # --- 4 & 5. Embed + write ---
+    print(f"Initialising ChromaDB at: {CHROMA_DIR}")
+    CHROMA_DIR.mkdir(parents=True, exist_ok=True)
+    client = chromadb.PersistentClient(path=str(CHROMA_DIR))
+    collection = _get_collection(client)
+    print(f"  Collection '{COLLECTION_NAME}' ready (previous data cleared).\n")
+
+    print(f"Embedding and storing {len(all_chunks)} chunks...")
+    print("  This will take several minutes on first run.\n")
+    _write_to_chroma(collection, all_chunks)
+
+    # --- Summary ---
+    final_count = collection.count()
+    print(f"\nIngest complete.")
+    print(f"  Collection : {COLLECTION_NAME}")
+    print(f"  Location   : {CHROMA_DIR}")
+    print(f"  Chunks     : {final_count}")
+    print(f"  Excluded   : {excluded_count} dead scaffolding files\n")
+    print("Smoke test:")
+    print('  python -c "')
+    print('  import chromadb')
+    print(f'  c = chromadb.PersistentClient(path=\\"{CHROMA_DIR}\\")')
+    print(f'  col = c.get_collection(\\"{COLLECTION_NAME}\\")')
+    print('  r = col.query(query_texts=[\\\"BillFetchService implementation\\\"], n_results=3)')
+    print('  [print(m[\\\"file_path\\\"]) for m in r[\\\"metadatas\\\"][0]]')
+    print('  "')

biller_cli/commands/down.py

@@ -0,0 +1,96 @@
+import os
+import signal
+import time
+import yaml
+import psutil
+import typer
+import subprocess
+from pathlib import Path
+from rich.console import Console
+
+app = typer.Typer(help="Stop the Biller middleware stack.")
+console = Console()
+
+PID_FILE_NAME = ".biller/biller.pid"
+
+def _down_docker(project_dir: Path):
+    compose_file = project_dir / "docker-compose.yml"
+    if not compose_file.exists():
+        console.print("[yellow]docker-compose.yml not found. Nothing to stop.[/yellow]")
+        return
+        
+    console.print("[blue]Stopping Docker stack...[/blue]")
+    result = subprocess.run(["docker", "compose", "-f", str(compose_file), "down"])
+    if result.returncode == 0:
+        console.print("[green]✓ Stack stopped successfully.[/green]")
+    else:
+        console.print("[red]Failed to stop Docker compose stack.[/red]")
+        raise typer.Exit(result.returncode)
+
+def _down_native(project_dir: Path):
+    pid_file = project_dir / PID_FILE_NAME
+
+    if not pid_file.exists():
+        console.print("[yellow]No running native stack found (no PID file).[/yellow]")
+        return
+
+    try:
+        pid = int(pid_file.read_text().strip())
+    except ValueError:
+        console.print("[yellow]Corrupted PID file. Removing it.[/yellow]")
+        pid_file.unlink()
+        return
+
+    if not psutil.pid_exists(pid):
+        console.print("[yellow]Process is already dead. Cleaning up stale PID file.[/yellow]")
+        pid_file.unlink()
+        return
+
+    try:
+        console.print(f"[blue]Sending SIGTERM to PID {pid}...[/blue]")
+        os.kill(pid, signal.SIGTERM)
+
+        # Wait gracefully for up to 10 seconds
+        deadline = time.time() + 10
+        while time.time() < deadline:
+            if not psutil.pid_exists(pid):
+                break
+            time.sleep(0.5)
+
+        # Escalate to SIGKILL if it refuses to die
+        if psutil.pid_exists(pid):
+            console.print("[yellow]Process did not terminate cleanly. Force killing (SIGKILL)...[/yellow]")
+            os.kill(pid, signal.SIGKILL)
+
+        pid_file.unlink()
+        console.print("[green]✓ Native stack stopped successfully.[/green]")
+
+    except ProcessLookupError:
+        pid_file.unlink()
+        console.print("[green]✓ Already stopped.[/green]")
+    except PermissionError:
+        console.print(f"[red]Permission denied. Try running: kill -9 {pid}[/red]")
+
+@app.callback(invoke_without_command=True)
+def down(
+    project_dir: Path = typer.Option(Path("."), "--dir", "-d", help="Project directory"),
+):
+    """Stop the Biller middleware stack."""
+    biller_yaml_path = project_dir / "biller.yaml"
+    if not biller_yaml_path.exists():
+        console.print("[red]No biller.yaml found. Are you in the project directory?[/red]")
+        raise typer.Exit(1)
+
+    try:
+        config = yaml.safe_load(biller_yaml_path.read_text())
+    except Exception:
+        config = {}
+
+    runtime = config.get("runtime", "docker")
+
+    if runtime == "docker":
+        _down_docker(project_dir)
+    elif runtime == "native":
+        _down_native(project_dir)
+    else:
+        console.print(f"[red]Unknown runtime '{runtime}'.[/red]")

biller_cli/commands/downgrade.py

@@ -0,0 +1,142 @@
+"""
+biller-cli downgrade
+
+Rolls back the biller-core version pin in biller.yaml and pom.xml.
+
+DATABASE WARNING:
+  Downgrade does NOT touch the database. If the upgrade applied schema
+  migrations, a downgrade will leave the DB in a state incompatible
+  with the older code. You must roll back the database manually.
+  This warning is printed before the confirmation prompt — always.
+"""
+
+import subprocess
+import sys
+import typer
+import yaml
+from pathlib import Path
+from rich.console import Console
+from packaging.version import Version
+
+from biller_cli.utils.version_pin import get_version_pin, set_version_pin
+
+app = typer.Typer(help="Downgrade biller-core to an older version.")
+console = Console()
+
+REGISTRY_URL_DEFAULT = "https://biller-registry-production.up.railway.app"
+
+
+def _get_registry_url(project_dir: Path) -> str:
+    """Read BILLER_REGISTRY_URL from environment, then .env.biller, fallback to default."""
+    # Environment variable takes highest priority
+    import os
+    env_var = os.environ.get("BILLER_REGISTRY_URL")
+    if env_var:
+        return env_var
+    # Then .env.biller file
+    env_file = project_dir / ".env.biller"
+    if env_file.exists():
+        for line in env_file.read_text().splitlines():
+            if line.startswith("BILLER_REGISTRY_URL="):
+                return line.split("=", 1)[1].strip()
+    return REGISTRY_URL_DEFAULT
+
+
+def _version_exists_in_registry(registry_url: str, version: str) -> bool:
+    """Return True if the version exists in the registry manifest."""
+    import requests
+    try:
+        resp = requests.get(f"{registry_url}/api/v1/manifest", timeout=10)
+        if resp.status_code != 200:
+            return True  # Fail open — don't block downgrade on registry unavailability
+        manifest = resp.json()
+        return version in manifest.get("versions", {})
+    except Exception:
+        return True  # Fail open
+
+
+def _restart_stack(project_dir: Path) -> None:
+    """
+    Restart the stack using the biller-cli executable found via sys.executable.
+    Avoids PATH lookup failures when installed in a venv (e.g. via pipx).
+    """
+    biller_cli_exe = Path(sys.executable).parent / "biller-cli"
+    if not biller_cli_exe.exists():
+        console.print("[red]Cannot find biller-cli executable for auto-restart.[/red]")
+        console.print("Run manually: [bold]biller-cli down && biller-cli up[/bold]")
+        return
+
+    dir_str = str(project_dir)
+
+    result_down = subprocess.run([str(biller_cli_exe), "down", "--dir", dir_str])
+    if result_down.returncode != 0:
+        console.print("[yellow]⚠ biller-cli down exited with an error. Proceeding to up anyway.[/yellow]")
+
+    result_up = subprocess.run([str(biller_cli_exe), "up", "--dir", dir_str])
+    if result_up.returncode != 0:
+        console.print("[red]biller-cli up failed after downgrade.[/red]")
+        console.print("Check logs and run [bold]biller-cli up[/bold] manually.")
+
+
+@app.callback(invoke_without_command=True)
+def downgrade(
+    project_dir: Path = typer.Option(Path("."), "--dir", "-d", help="Project directory"),
+    to: str = typer.Option(..., "--to", help="Target version to downgrade to"),
+    restart: bool = typer.Option(False, "--restart", help="Restart the stack after downgrade"),
+):
+    """Downgrade biller-core to an older version."""
+    # Resolve current version
+    try:
+        current_version = get_version_pin(project_dir)
+    except (FileNotFoundError, KeyError) as e:
+        console.print(f"[red]{e}[/red]")
+        console.print("Run [bold]biller-cli init[/bold] first.")
+        raise typer.Exit(1)
+
+    # Guard: same version
+    if Version(to) == Version(current_version):
+        console.print(f"Already on [bold]{current_version}[/bold]. Nothing to do.")
+        raise typer.Exit(0)
+
+    # Guard: target is newer
+    if Version(to) > Version(current_version):
+        console.print(f"[red]{to} is newer than {current_version}.[/red]")
+        console.print("Use [bold]biller-cli upgrade[/bold] to upgrade.")
+        raise typer.Exit(1)
+
+    # Validate target version exists in registry
+    registry_url = _get_registry_url(project_dir)
+    if not _version_exists_in_registry(registry_url, to):
+        console.print(f"[red]Version {to} is not in the registry manifest.[/red]")
+        console.print(f"Check available versions: {registry_url}/api/v1/manifest")
+        raise typer.Exit(1)
+
+    # DB warning — always shown before confirmation
+    console.print("\n[yellow bold]⚠ DATABASE WARNING[/yellow bold]")
+    console.print(
+        "Downgrade does [bold]NOT[/bold] roll back database schema changes.\n"
+        "If the upgrade applied migrations, the older code may fail to boot\n"
+        "or behave incorrectly against the current schema.\n"
+        "Roll back your database manually before proceeding."
+    )
+
+    console.print(
+        f"\nDowngrading biller-core: [bold]{current_version}[/bold] → [bold]{to}[/bold]"
+    )
+    typer.confirm("Continue with downgrade?", abort=True)
+
+    # Update version pin
+    try:
+        set_version_pin(project_dir, to)
+    except (FileNotFoundError, RuntimeError) as e:
+        console.print(f"[red]Failed to update version pin: {e}[/red]")
+        raise typer.Exit(1)
+
+    console.print(f"\n[green bold]✓ Downgrade complete.[/green bold]")
+    console.print(f"biller-core pinned to [bold]{to}[/bold] in biller.yaml and pom.xml.")
+
+    if restart:
+        console.print("\nRestarting stack...")
+        _restart_stack(project_dir)
+    else:
+        console.print("\nRun [bold]biller-cli down && biller-cli up[/bold] to apply the downgrade.")