journeyman-cli 1.1.0__py3-none-any.whl

journeyman/__init__.py

@@ -0,0 +1,2 @@
+"""journeyman — route implementation tasks to a local or economical model."""
+__version__ = "1.1.0"

journeyman/backend_util.py

@@ -0,0 +1,120 @@
+"""Shared config and HTTP helpers for journeyman scripts.
+
+Single source for load/save config, model listing, and local model picking.
+Import from here — do not duplicate these helpers in setup/detect/worker.
+"""
+from __future__ import annotations
+
+import json
+import os
+import pathlib
+import re
+import stat
+import urllib.error
+import urllib.request
+
+from .constants import CONFIG_PATH  # re-exported for callers/tests
+
+# Patterns that usually are not usable as chat/code-generation workers.
+_NON_CHAT_RE = re.compile(
+    r"(embed|embedding|rerank|whisper|tts|clip|text-embedding)",
+    re.IGNORECASE,
+)
+_SIZE_RE = re.compile(r"(\d+(?:\.\d+)?)\s*[bB]\b")
+_INSTRUCT_RE = re.compile(r"(instruct|chat|-it\b|:it\b)", re.IGNORECASE)
+
+
+def get_json(url: str, timeout: int = 3):
+    """Fetch JSON from *url*; return parsed object or None on any error."""
+    try:
+        with urllib.request.urlopen(url, timeout=timeout) as resp:
+            return json.loads(resp.read())
+    except (OSError, json.JSONDecodeError, ValueError):
+        return None
+
+
+def list_models_openai(base_url: str) -> list[str] | None:
+    """Return model IDs from an OpenAI-compatible /v1/models endpoint, or None."""
+    data = get_json(base_url.rstrip("/") + "/v1/models")
+    if data is None:
+        return None
+    return [m["id"] for m in data.get("data", [])]
+
+
+def list_models_ollama(base_url: str) -> list[str] | None:
+    """Return model names from Ollama's /api/tags endpoint, or None."""
+    data = get_json(base_url.rstrip("/") + "/api/tags")
+    if data is None:
+        return None
+    return [m["name"] for m in data.get("models", [])]
+
+
+def load_config() -> dict:
+    """Load saved journeyman config, or return empty dict."""
+    if CONFIG_PATH.exists():
+        try:
+            return json.loads(CONFIG_PATH.read_text(encoding="utf-8"))
+        except (OSError, json.JSONDecodeError):
+            pass
+    return {}
+
+
+def save_config(cfg: dict) -> None:
+    """Write config as UTF-8 JSON with owner-only permissions (0600)."""
+    CONFIG_PATH.parent.mkdir(parents=True, exist_ok=True)
+    payload = json.dumps(cfg, indent=2) + "\n"
+    # Write via temp + replace so we can set mode before the file is readable.
+    tmp = CONFIG_PATH.with_suffix(CONFIG_PATH.suffix + ".tmp")
+    fd = os.open(
+        str(tmp),
+        os.O_WRONLY | os.O_CREAT | os.O_TRUNC,
+        stat.S_IRUSR | stat.S_IWUSR,  # 0600
+    )
+    try:
+        with os.fdopen(fd, "w", encoding="utf-8") as f:
+            f.write(payload)
+        os.replace(str(tmp), str(CONFIG_PATH))
+        os.chmod(CONFIG_PATH, stat.S_IRUSR | stat.S_IWUSR)
+    except Exception:
+        try:
+            os.unlink(tmp)
+        except OSError:
+            pass
+        raise
+    print(f"Config saved → {CONFIG_PATH}")
+
+
+def is_non_chat_model(name: str) -> bool:
+    """Return True if *name* looks like an embedding/ASR/TTS/etc. model."""
+    return bool(_NON_CHAT_RE.search(name))
+
+
+def _model_sort_key(name: str) -> tuple:
+    """Prefer instruct/chat tags, then smaller parameter counts (more economical)."""
+    instruct_penalty = 0 if _INSTRUCT_RE.search(name) else 1
+    match = _SIZE_RE.search(name)
+    size = float(match.group(1)) if match else 50.0  # unknown mid-tier
+    return (instruct_penalty, size, name.lower())
+
+
+def pick_local_model(models: list[str], preferred: str | None = None) -> str:
+    """Choose a local chat/code model from *models*.
+
+    Order of preference:
+      1. Explicit *preferred* if it is installed
+      2. Chat-like models (exclude embeddings / ASR / TTS / CLIP)
+      3. Instruct/chat-tagged names, then smaller sizes
+      4. Fall back to the full list if every model was filtered out
+    """
+    if not models:
+        raise ValueError("pick_local_model requires a non-empty models list")
+    if preferred and preferred in models:
+        return preferred
+    chat = [m for m in models if not is_non_chat_model(m)]
+    pool = chat or list(models)
+    return min(pool, key=_model_sort_key)
+
+
+def read_text_utf8(path: pathlib.Path) -> str:
+    """Read a text file as UTF-8; raise UnicodeDecodeError on binary/invalid data."""
+    return path.read_text(encoding="utf-8")

journeyman/constants.py

@@ -0,0 +1,95 @@
+"""Shared backend configuration for all journeyman scripts.
+
+This is the single source of truth for provider/backend data.
+Import from here — do not duplicate these lists.
+"""
+from __future__ import annotations
+
+import os
+import pathlib
+
+# ---------------------------------------------------------------------------
+# Config path
+# ---------------------------------------------------------------------------
+
+CONFIG_PATH = pathlib.Path(os.environ.get(
+    "JOURNEYMAN_CONFIG",
+    str(pathlib.Path.home() / ".config" / "journeyman" / "config.json"),
+))
+
+# Append-only usage log for `journeyman stats`.  Override with JOURNEYMAN_USAGE_LOG.
+USAGE_LOG_PATH = pathlib.Path(os.environ.get(
+    "JOURNEYMAN_USAGE_LOG",
+    str(pathlib.Path.home() / ".local" / "share" / "journeyman" / "usage.jsonl"),
+))
+
+# ---------------------------------------------------------------------------
+# Backend definitions
+# ---------------------------------------------------------------------------
+
+# (friendly_name, base_url, list_api_type)
+# list_api_type controls which endpoint to probe for model enumeration.
+# All local backends use OpenAI-compat /v1/chat/completions for inference.
+LOCAL_BACKENDS: list[tuple[str, str, str]] = [
+    ("ollama",        "http://localhost:11434", "ollama"),
+    ("lmstudio",      "http://localhost:1234",  "openai"),
+    ("llama.cpp/mlx", "http://localhost:8080",  "openai"),
+]
+
+# (friendly_name, env_var, base_url, api_type, default_model)
+REMOTE_BACKENDS: list[tuple[str, str, str, str, str]] = [
+    (
+        "groq",
+        "GROQ_API_KEY",
+        "https://api.groq.com/openai/v1",
+        "openai",
+        "llama-3.1-8b-instant",
+    ),
+    (
+        "together",
+        "TOGETHER_API_KEY",
+        "https://api.together.xyz/v1",
+        "openai",
+        "meta-llama/Meta-Llama-3.1-8B-Instruct-Turbo",
+    ),
+    (
+        "gemini",
+        "GEMINI_API_KEY",
+        "https://generativelanguage.googleapis.com/v1beta/openai",
+        "openai",
+        "gemini-2.0-flash",
+    ),
+    (
+        "openai",
+        "OPENAI_API_KEY",
+        "https://api.openai.com/v1",
+        "openai",
+        "gpt-4o-mini",
+    ),
+]
+
+# Convenience lookup: provider name → env var name (used by sb_worker auth).
+REMOTE_ENV_VAR: dict[str, str] = {
+    name: env_var for name, env_var, *_ in REMOTE_BACKENDS
+}
+
+# All valid remote provider names (used for validation in setup).
+VALID_REMOTE_PROVIDERS: list[str] = [name for name, *_ in REMOTE_BACKENDS]
+
+# ---------------------------------------------------------------------------
+# Pricing table  (approximate mid-2025 list prices, USD per 1 M tokens)
+# ---------------------------------------------------------------------------
+# Key: (provider_name, model_name)  →  (input_usd_per_1M, output_usd_per_1M)
+# Use (provider, None) as the fallback for any model from that provider.
+# Local providers always return (0.0, 0.0).
+MODEL_PRICING: dict[tuple[str, str | None], tuple[float, float]] = {
+    # Remote — default models
+    ("groq",          "llama-3.1-8b-instant"):                          (0.05,  0.08),
+    ("together",      "meta-llama/Meta-Llama-3.1-8B-Instruct-Turbo"):  (0.18,  0.18),
+    ("gemini",        "gemini-2.0-flash"):                             (0.10,  0.40),
+    ("openai",        "gpt-4o-mini"):                                  (0.15,  0.60),
+    # Local — always zero
+    ("ollama",        None):                                           (0.0,   0.0),
+    ("lmstudio",      None):                                           (0.0,   0.0),
+    ("llama.cpp/mlx", None):                                           (0.0,   0.0),
+}

journeyman/detect_worker.py

@@ -0,0 +1,244 @@
+#!/usr/bin/env python3
+"""Detect the preferred local-first model backend for journeyman worker tasks.
+
+Priority:
+  1. Local servers (no data leaves the machine):
+       Ollama         → http://localhost:11434
+       LM Studio      → http://localhost:1234
+       llama.cpp / mlx_lm → http://localhost:8080
+
+  2. Low-cost remote APIs (checked only if no local server is found; requires env var):
+       GROQ_API_KEY      → api.groq.com           llama-3.1-8b-instant
+       TOGETHER_API_KEY  → api.together.xyz        meta-llama/Meta-Llama-3.1-8B-Instruct-Turbo
+       GEMINI_API_KEY    → generativelanguage...   gemini-2.0-flash
+       OPENAI_API_KEY    → api.openai.com          gpt-4o-mini
+
+Usage (standalone):
+    python3 detect_worker.py           # print JSON, exit 0 if found / 1 if not
+    python3 detect_worker.py --list    # human-readable list of all backends
+    python3 detect_worker.py --require-local  # fail with exit 1 if only remote found
+
+Output JSON (exit 0):
+    {
+      "name":      "ollama",
+      "endpoint":  "http://localhost:11434",
+      "model":     "llama3:8b",
+      "api_type":  "openai",
+      "is_local":  true
+    }
+
+    When a local server is running but has no models installed, the JSON
+    will include "model": null and "error": "<message>". Callers must check
+    for this before attempting an API call.
+
+Exit codes:
+    0  backend found; JSON written to stdout
+    1  no backend found (or --require-local and only remote available);
+       also when a local server is up but has no models installed
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import os
+import sys
+
+from .backend_util import (
+    list_models_ollama,
+    list_models_openai,
+    load_config,
+    pick_local_model,
+)
+from .constants import (
+    LOCAL_BACKENDS,
+    REMOTE_BACKENDS,
+)
+
+# ---------------------------------------------------------------------------
+# Detection
+# ---------------------------------------------------------------------------
+
+def detect_local() -> dict | None:
+    """Return the first live local backend that has at least one model, or None.
+
+    If a server is running but has no models installed, returns a result dict
+    with ``"model": None`` and ``"error": "<message>"`` instead of falling
+    through to remote. Callers must check for the error key before use.
+    """
+    for name, url, api_type in LOCAL_BACKENDS:
+        if api_type == "ollama":
+            models = list_models_ollama(url)
+        else:
+            models = list_models_openai(url)
+
+        if models is None:
+            continue  # server not running
+
+        if not models:
+            # Server is running but no models installed.
+            # Return an error result rather than silently falling through to
+            # remote — the user almost certainly does not want their data sent
+            # to a remote API just because they forgot to pull a model.
+            return {
+                "name": name,
+                "endpoint": url,
+                "model": None,
+                "api_type": "openai",
+                "is_local": True,
+                "available_models": [],
+                "error": (
+                    f"No models installed on {name}. "
+                    f"Install one (e.g. ollama pull llama3.2) and retry."
+                ),
+            }
+
+        config = load_config()
+        preferred = config.get("preferred_local_model")
+        model = pick_local_model(models, preferred=preferred)
+
+        return {
+            "name": name,
+            "endpoint": url,
+            "model": model,
+            "api_type": "openai",  # normalised: all local use /v1/chat/completions
+            "is_local": True,
+            "available_models": models,
+        }
+
+    return None
+
+
+def detect_remote(preferred_provider: str | None = None) -> dict | None:
+    """Return the first available remote backend, or None.
+
+    If *preferred_provider* is given (e.g. "groq"), it is tried first
+    regardless of the default list order.
+    """
+    config = load_config()
+    remote_model_overrides: dict = config.get("remote_models", {})
+
+    ordered = list(REMOTE_BACKENDS)
+    if preferred_provider:
+        ordered.sort(key=lambda b: 0 if b[0] == preferred_provider else 1)
+
+    for name, env_var, endpoint, api_type, default_model in ordered:
+        api_key = os.environ.get(env_var) or config.get("api_keys", {}).get(name)
+        if not api_key:
+            continue
+
+        model = remote_model_overrides.get(name, default_model)
+        return {
+            "name": name,
+            "endpoint": endpoint,
+            "model": model,
+            "api_type": api_type,
+            "api_key": api_key,
+            "is_local": False,
+        }
+
+    return None
+
+
+def detect(require_local: bool = False, preferred_remote: str | None = None) -> dict | None:
+    """Full detection: local first, remote fallback."""
+    local = detect_local()
+    if local:
+        return local
+    if require_local:
+        return None
+    return detect_remote(preferred_provider=preferred_remote)
+
+
+# ---------------------------------------------------------------------------
+# CLI
+# ---------------------------------------------------------------------------
+
+def _cmd_list() -> int:
+    print("Local backends:")
+    for name, url, api_type in LOCAL_BACKENDS:
+        if api_type == "ollama":
+            models = list_models_ollama(url)
+        else:
+            models = list_models_openai(url)
+
+        if models is None:
+            print(f"  ✗  {name:<16} {url}  (not running)")
+        else:
+            model_str = ", ".join(models) if models else "(no models installed)"
+            print(f"  ✓  {name:<16} {url}  →  {model_str}")
+
+    print("\nRemote backends:")
+    config = load_config()
+    for name, env_var, endpoint, _, default_model in REMOTE_BACKENDS:
+        api_key = os.environ.get(env_var) or config.get("api_keys", {}).get(name)
+        status = "✓ configured" if api_key else "✗ not set    "
+        override = config.get("remote_models", {}).get(name)
+        model_info = f"{override} (override)" if override else default_model
+        print(f"  {status}  {name:<10} [{env_var}]  →  {model_info}")
+
+    pref = config.get("preferred_remote")
+    if pref:
+        print(f"\nPreferred remote: {pref}")
+    return 0
+
+
+def _cmd_detect(require_local: bool) -> int:
+    config = load_config()
+    preferred_remote = config.get("preferred_remote")
+
+    result = detect(require_local=require_local, preferred_remote=preferred_remote)
+    if result is None:
+        if require_local:
+            print(
+                "No local model server found (--require-local set). "
+                "Run: journeyman setup --list",
+                file=sys.stderr,
+            )
+        else:
+            print(
+                "No model backend found. Run: journeyman setup --list",
+                file=sys.stderr,
+            )
+        return 1
+
+    # Warn about remote use before printing JSON so callers can capture JSON cleanly.
+    if not result["is_local"]:
+        print(
+            f"[journeyman] WARNING: Using remote backend '{result['name']}' — "
+            f"spec and context files will be sent to {result['endpoint']}",
+            file=sys.stderr,
+        )
+
+    # Omit api_key from stdout output (security).
+    safe = {k: v for k, v in result.items() if k != "api_key"}
+    print(json.dumps(safe))
+
+    # Server running but no usable model is not a successful detection.
+    if result.get("error") or not result.get("model"):
+        err = result.get("error") or f"Backend '{result.get('name')}' returned no model."
+        print(err, file=sys.stderr)
+        return 1
+
+    return 0
+
+
+def main(argv: list[str] | None = None) -> int:
+    ap = argparse.ArgumentParser(
+        description="Detect preferred local-first model backend for journeyman.",
+    )
+    ap.add_argument("--list", action="store_true", help="Show all backends (human-readable)")
+    ap.add_argument(
+        "--require-local",
+        action="store_true",
+        help="Exit 1 if only a remote backend is available",
+    )
+    args = ap.parse_args(argv)
+
+    if args.list:
+        return _cmd_list()
+    return _cmd_detect(require_local=args.require_local)
+
+
+if __name__ == "__main__":
+    sys.exit(main())

journeyman/main.py

@@ -0,0 +1,48 @@
+import sys
+
+
+def print_help():
+    print("""journeyman — route implementation tasks to a local or economical model
+
+Usage:
+  journeyman worker  --task SPEC --out FILE [options]   Run the worker
+  journeyman setup   [options]                          Configure backends
+  journeyman detect  [--list]                           Show detected backend
+  journeyman stats   [options]                          Show usage statistics
+
+Run 'journeyman <command> --help' for details on a specific command.
+""")
+
+def main() -> int:
+    args = sys.argv[1:]
+    if not args:
+        print_help()
+        return 2
+
+    command = args[0]
+    sub_args = args[1:]
+
+    if command in ("-h", "--help", "help"):
+        print_help()
+        return 0
+    elif command in ("-v", "--version", "version"):
+        print("journeyman 1.1.0")
+        return 0
+    elif command == "worker":
+        from journeyman import sb_worker
+        return sb_worker.main(sub_args)
+    elif command == "setup":
+        from journeyman import setup
+        return setup.main(sub_args)
+    elif command == "detect":
+        from journeyman import detect_worker
+        return detect_worker.main(sub_args)
+    elif command == "stats":
+        from journeyman import stats
+        return stats.main(sub_args)
+    else:
+        print(f"[journeyman] Unknown command: {command}. Run 'journeyman help' for usage.", file=sys.stderr)
+        return 2
+
+if __name__ == "__main__":
+    sys.exit(main())