codrspot-processor-mcp 0.1.0__py3-none-any.whl

codepreproc_client/layer1_business/cli/init.py

@@ -0,0 +1,578 @@
+"""codepreproc init: register a project into PostgreSQL."""
+
+from __future__ import annotations
+
+import asyncio
+import json
+import os
+import re
+import sys
+from collections import Counter
+from pathlib import Path
+from typing import Any
+
+# ---------------------------------------------------------------------------
+# Language detection
+# ---------------------------------------------------------------------------
+
+EXTENSION_TO_LANGUAGE: dict[str, str] = {
+    ".py": "python",
+    ".ts": "typescript",
+    ".tsx": "typescript",
+    ".js": "javascript",
+    ".mjs": "javascript",
+    ".jsx": "javascript",
+    ".go": "go",
+    ".rs": "rust",
+    ".dart": "dart",
+    ".php": "php",
+    ".rb": "ruby",
+    ".java": "java",
+    ".kt": "kotlin",
+    ".cs": "csharp",
+    ".cpp": "cpp",
+    ".c": "c",
+    ".swift": "swift",
+}
+
+# Directories always skipped during scanning
+_ALWAYS_SKIP_DIRS = frozenset({
+    ".git", "node_modules", "__pycache__", ".venv", "venv", "env",
+    "dist", "build", ".next", ".nuxt", "target", "vendor",
+    "coverage", ".cache", ".pytest_cache", ".mypy_cache",
+    ".dart_tool", ".flutter-plugins", "storage",
+    ".idea", ".vscode",
+})
+
+# File suffixes that are binary / not source
+_SKIP_SUFFIXES = frozenset({".pyc", ".pyo", ".class", ".o", ".a", ".so", ".dll", ".exe"})
+
+# Default ignore list written into every new project entry
+_DEFAULT_IGNORE = [
+    ".git",
+    "node_modules",
+    "vendor",
+    "__pycache__",
+    "dist",
+    "build",
+    ".venv",
+    "venv",
+    "*.pyc",
+]
+
+
+# ---------------------------------------------------------------------------
+# Helpers
+# ---------------------------------------------------------------------------
+
+def _slugify(name: str) -> str:
+    """Convert a directory name to a safe project_id slug."""
+    slug = re.sub(r"[^a-z0-9_-]", "_", name.lower())
+    slug = re.sub(r"_+", "_", slug).strip("_")
+    return slug or "project"
+
+
+def _read_safe(path: Path, max_bytes: int = 8192) -> str:
+    try:
+        return path.read_text(encoding="utf-8", errors="replace")[:max_bytes]
+    except Exception:
+        return ""
+
+
+def _walk_extensions(root: Path) -> Counter:
+    """Count file extensions across the project tree, skipping ignored dirs."""
+    counts: Counter = Counter()
+    for dirpath, dirnames, filenames in os.walk(root):
+        # Prune directories in-place so os.walk skips them entirely
+        dirnames[:] = [d for d in dirnames if d not in _ALWAYS_SKIP_DIRS]
+        for fname in filenames:
+            fpath = Path(fname)
+            if fpath.suffix in _SKIP_SUFFIXES:
+                continue
+            if fpath.suffix:
+                counts[fpath.suffix.lower()] += 1
+    return counts
+
+
+def _detect_languages(ext_counts: Counter) -> list[str]:
+    """Return the primary languages detected from extension counts.
+
+    A language is included if its files account for ≥5% of all matched
+    source files.  At least one language is always returned.
+    """
+    lang_counts: Counter = Counter()
+    for ext, count in ext_counts.items():
+        lang = EXTENSION_TO_LANGUAGE.get(ext)
+        if lang:
+            lang_counts[lang] += count
+
+    if not lang_counts:
+        return ["unknown"]
+
+    total = sum(lang_counts.values())
+    threshold = max(1, int(total * 0.05))
+    langs = [lang for lang, cnt in lang_counts.most_common() if cnt >= threshold]
+    return langs or [lang_counts.most_common(1)[0][0]]
+
+
+# ---------------------------------------------------------------------------
+# Framework / tech-stack detection
+# ---------------------------------------------------------------------------
+
+def _detect_framework(root: Path) -> dict[str, Any]:
+    """Probe well-known marker files and return framework metadata."""
+    info: dict[str, Any] = {
+        "framework": None,
+        "tech_stack": [],
+        "extra_ignore": [],
+    }
+
+    # ── Node.js / TypeScript ──────────────────────────────────────────────
+    pkg_json = root / "package.json"
+    if pkg_json.exists():
+        try:
+            pkg = json.loads(pkg_json.read_text(encoding="utf-8"))
+            all_deps: dict = {**pkg.get("dependencies", {}), **pkg.get("devDependencies", {})}
+            if "@nestjs/core" in all_deps:
+                info["framework"] = "nestjs"
+                info["tech_stack"].append("NestJS")
+            elif "next" in all_deps:
+                info["framework"] = "nextjs"
+                info["tech_stack"].append("Next.js")
+                info["extra_ignore"].append(".next")
+            elif "react" in all_deps:
+                info["framework"] = "react"
+                info["tech_stack"].append("React")
+            elif "express" in all_deps:
+                info["framework"] = "express"
+                info["tech_stack"].append("Express")
+            else:
+                info["tech_stack"].append("Node.js")
+        except Exception:
+            pass
+
+    # ── Flutter / Dart ────────────────────────────────────────────────────
+    if (root / "pubspec.yaml").exists():
+        info["framework"] = "flutter"
+        info["tech_stack"].append("Flutter")
+        info["extra_ignore"].extend([".dart_tool", ".flutter-plugins"])
+
+    # ── PHP / Laravel ─────────────────────────────────────────────────────
+    composer_json = root / "composer.json"
+    if composer_json.exists():
+        try:
+            comp = json.loads(composer_json.read_text(encoding="utf-8"))
+            reqs: dict = {**comp.get("require", {}), **comp.get("require-dev", {})}
+            if "laravel/framework" in reqs:
+                info["framework"] = "laravel"
+                info["tech_stack"].append("Laravel")
+            info["extra_ignore"].extend(["vendor", "storage/logs", "storage/framework"])
+        except Exception:
+            pass
+
+    # ── Python web frameworks ─────────────────────────────────────────────
+    if not info["framework"]:
+        req_files = [
+            root / "requirements.txt",
+            root / "requirements-dev.txt",
+            root / "requirements-prod.txt",
+        ]
+        for req_file in req_files:
+            if req_file.exists():
+                content = _read_safe(req_file).lower()
+                if "fastapi" in content:
+                    info["framework"] = "fastapi"
+                    info["tech_stack"].append("FastAPI")
+                    break
+                elif "django" in content:
+                    info["framework"] = "django"
+                    info["tech_stack"].append("Django")
+                    break
+                elif "flask" in content:
+                    info["framework"] = "flask"
+                    info["tech_stack"].append("Flask")
+                    break
+
+        pyproject = root / "pyproject.toml"
+        if not info["framework"] and pyproject.exists():
+            content = _read_safe(pyproject).lower()
+            for fw, label in [("fastapi", "FastAPI"), ("django", "Django"), ("flask", "Flask")]:
+                if fw in content:
+                    info["framework"] = fw
+                    info["tech_stack"].append(label)
+                    break
+
+    # ── Rust ──────────────────────────────────────────────────────────────
+    if (root / "Cargo.toml").exists():
+        if not info["framework"]:
+            info["framework"] = "rust"
+        info["tech_stack"].append("Rust/Cargo")
+        if "target" not in info["extra_ignore"]:
+            info["extra_ignore"].append("target")
+
+    # ── Go ────────────────────────────────────────────────────────────────
+    if (root / "go.mod").exists():
+        if not info["framework"]:
+            info["framework"] = "go"
+        info["tech_stack"].append("Go modules")
+
+    # ── Java / Spring Boot ────────────────────────────────────────────────
+    pom = root / "pom.xml"
+    if pom.exists():
+        pom_text = pom.read_text(encoding="utf-8", errors="ignore").lower()
+        if "spring-boot" in pom_text:
+            if not info["framework"]:
+                info["framework"] = "spring"
+            info["tech_stack"].append("Spring Boot")
+        elif not info["framework"]:
+            info["framework"] = "java"
+            info["tech_stack"].append("Java/Maven")
+        if "target" not in info["extra_ignore"]:
+            info["extra_ignore"].append("target")
+    else:
+        for gf in ("build.gradle", "build.gradle.kts"):
+            gradle = root / gf
+            if gradle.exists():
+                gradle_text = gradle.read_text(encoding="utf-8", errors="ignore").lower()
+                if "spring-boot" in gradle_text:
+                    if not info["framework"]:
+                        info["framework"] = "spring"
+                    info["tech_stack"].append("Spring Boot")
+                elif not info["framework"]:
+                    info["framework"] = "java"
+                    info["tech_stack"].append("Java/Gradle")
+                if "build" not in info["extra_ignore"]:
+                    info["extra_ignore"].append("build")
+                break
+
+    return info
+
+
+def _build_ignore_list(extra: list[str]) -> list[str]:
+    """Combine default ignores with framework-specific extras (deduped, order-stable)."""
+    seen: set[str] = set()
+    result: list[str] = []
+    for item in _DEFAULT_IGNORE + extra:
+        if item not in seen:
+            seen.add(item)
+            result.append(item)
+    return result
+
+
+# ---------------------------------------------------------------------------
+# Project context generation
+# ---------------------------------------------------------------------------
+
+def _gather_summary(root: Path, framework_info: dict[str, Any]) -> str:
+    """Build a short textual description of the project for LLM input."""
+    parts: list[str] = []
+
+    # README (first ~1500 chars)
+    for readme_name in ("README.md", "README.rst", "README.txt", "readme.md"):
+        readme = root / readme_name
+        if readme.exists():
+            content = _read_safe(readme, max_bytes=2000).strip()
+            if content:
+                parts.append(f"README:\n{content[:1500]}")
+            break
+
+    # Top-level directory / file listing
+    try:
+        top_dirs = sorted(p.name for p in root.iterdir() if p.is_dir() and p.name not in _ALWAYS_SKIP_DIRS)
+        top_files = sorted(p.name for p in root.iterdir() if p.is_file())
+        if top_dirs:
+            parts.append(f"Top-level directories: {', '.join(top_dirs[:20])}")
+        if top_files:
+            parts.append(f"Top-level files: {', '.join(top_files[:20])}")
+    except Exception:
+        pass
+
+    if framework_info.get("tech_stack"):
+        parts.append(f"Detected tech stack: {', '.join(framework_info['tech_stack'])}")
+
+    return "\n\n".join(parts)
+
+
+def _call_llm(endpoint: str, model: str, api_key: str, prompt_system: str, prompt_user: str) -> str | None:
+    """Make a single chat-completion call. Returns None on any failure."""
+    try:
+        import httpx
+
+        headers: dict[str, str] = {"Content-Type": "application/json"}
+        if api_key:
+            headers["Authorization"] = f"Bearer {api_key}"
+
+        payload = {
+            "model": model,
+            "messages": [
+                {"role": "system", "content": prompt_system},
+                {"role": "user", "content": prompt_user},
+            ],
+            "max_tokens": 200,
+            "temperature": 0.3,
+        }
+
+        with httpx.Client(timeout=15.0) as client:
+            resp = client.post(
+                f"{endpoint.rstrip('/')}/chat/completions",
+                headers=headers,
+                json=payload,
+            )
+            resp.raise_for_status()
+            return resp.json()["choices"][0]["message"]["content"].strip()
+    except Exception:
+        return None
+
+
+def _generate_context(
+    project_name: str,
+    summary: str,
+    defaults: dict[str, Any],
+) -> str | None:
+    """Try configured LLM providers in order of preference, return None on failure."""
+    system_prompt = (
+        "You are a technical assistant. Based on the project summary, write a concise "
+        "2–3 sentence project_context. Use the same language as the README (Spanish if "
+        "the README is in Spanish). Describe what the project does, its main technologies, "
+        "and the architectural pattern if visible. Output ONLY the description — no quotes, "
+        "no markdown formatting."
+    )
+    user_prompt = f"Project name: {project_name}\n\n{summary[:3000]}"
+
+    llm_providers: dict = defaults.get("llm_providers", {})
+
+    # 1. Prefer OpenAI provider when a valid API key is available
+    for _pname, pdata in llm_providers.items():
+        if pdata.get("type") == "openai":
+            api_key = os.getenv(pdata.get("api_key_env", ""), "")
+            if api_key:
+                result = _call_llm(
+                    endpoint="https://api.openai.com/v1",
+                    model=pdata.get("model", "gpt-4o-mini"),
+                    api_key=api_key,
+                    prompt_system=system_prompt,
+                    prompt_user=user_prompt,
+                )
+                if result:
+                    return result
+
+    # 2. Fall back to the local LLM endpoint
+    local_endpoint = defaults.get("llm_endpoint", "http://127.0.0.1:8080/v1")
+    local_model = defaults.get("llm_model", "qwen2.5-coder-7b")
+    return _call_llm(
+        endpoint=local_endpoint,
+        model=local_model,
+        api_key="",
+        prompt_system=system_prompt,
+        prompt_user=user_prompt,
+    )
+
+
+def _fallback_context(project_name: str, languages: list[str], framework_info: dict[str, Any]) -> str:
+    """Template-based project context when no LLM is available."""
+    tech = framework_info.get("tech_stack")
+    tech_str = f" using {', '.join(tech)}" if tech else ""
+    lang_str = ", ".join(languages)
+    return f"Project {project_name}{tech_str}. Primary languages: {lang_str}."
+
+
+# ---------------------------------------------------------------------------
+# Registry helpers
+# ---------------------------------------------------------------------------
+
+def _is_already_registered(stubs: list[tuple[str, str]], root: Path) -> tuple[bool, str | None]:
+    """Return (True, project_id) if root is already registered, else (False, None)."""
+    root_resolved = str(root.resolve())
+    for pid, existing in stubs:
+        try:
+            existing_resolved = str(Path(existing).expanduser().resolve())
+        except Exception:
+            existing_resolved = existing
+        if existing_resolved == root_resolved:
+            return True, pid
+    return False, None
+
+
+def _unique_project_id(base: str, existing_ids: set[str]) -> str:
+    """Return base or base_N (first N≥2 not already taken)."""
+    if base not in existing_ids:
+        return base
+    counter = 2
+    while f"{base}_{counter}" in existing_ids:
+        counter += 1
+    return f"{base}_{counter}"
+
+
+
+
+def _mint_jwt_for_project(project_id: str, repo_root: Path) -> tuple[str, Any]:
+    """Derive this project's fingerprint and mint a JWT naming it in claims."""
+    from codepreproc_client.layer1_business.config import get_credentials_path, get_server_url
+    from codepreproc_client.layer1_business.license.fingerprint import derive_fingerprint
+    from codepreproc_client.layer1_business.license.jwt_client import JWTClient
+    from codepreproc_client.layer1_business.manifest.credentials import load_manifest
+
+    manifest = load_manifest(get_credentials_path())
+    attestation = derive_fingerprint(project_id, repo_root)
+    jwt_client = JWTClient(base_url=get_server_url(), manifest=manifest, fingerprint_attestation=attestation)
+    jwt = asyncio.run(jwt_client.get_bearer_token())
+    return jwt, attestation
+
+
+def _persist_project(project: Any, jwt: str, fingerprint_attestation: Any) -> None:
+    """Register/update a ProjectConfig via the server API, then mirror it locally."""
+    from codepreproc_client.layer1_business.allowed_scope_cache import get_allowed_projects
+    from codepreproc_client.layer1_business.api_client.registry_client import register_project
+    from codepreproc_client.layer1_business.config import get_home_dir, get_server_url
+    from codepreproc_client.layer1_business.local_registry import save_local_project
+
+    stored = asyncio.run(
+        register_project(
+            base_url=get_server_url(),
+            jwt=jwt,
+            project=project,
+            fingerprint_attestation=fingerprint_attestation,
+        )
+    )
+    save_local_project(get_home_dir(), stored)
+    # Refresh the tenant's allowed-project cache now that the tenant/project set may
+    # have changed - subsequent calls read this cache instead of hitting the server.
+    asyncio.run(
+        get_allowed_projects(home_dir=get_home_dir(), base_url=get_server_url(), jwt=jwt, force_refresh=True)
+    )
+
+
+def _set_superkg_id(project_id: str, system_id: str, jwt: str) -> None:
+    from codepreproc_client.layer1_business.api_client.registry_client import set_superkg_id
+    from codepreproc_client.layer1_business.config import get_home_dir, get_server_url
+    from codepreproc_client.layer1_business.local_registry import load_local_projects, save_local_project
+
+    asyncio.run(set_superkg_id(base_url=get_server_url(), jwt=jwt, project_id=project_id, system_id=system_id))
+    local = load_local_projects(get_home_dir())
+    project = local.get(project_id)
+    if project is not None:
+        save_local_project(get_home_dir(), project.model_copy(update={"superkg_system_id": system_id}))
+
+
+# ---------------------------------------------------------------------------
+# Public entry point
+# ---------------------------------------------------------------------------
+
+def init_project(path: Path | None = None, verbose: bool = True) -> int:
+    """Register a project in the codepreproc PostgreSQL registry.
+
+    Returns an exit code (0 = success / already registered, 1 = error).
+    """
+    from codepreproc_client.layer1_business.config import get_home_dir
+    from codepreproc_client.layer1_business.local_registry import load_local_projects
+
+    target = (path or Path.cwd()).resolve()
+
+    if not target.exists() or not target.is_dir():
+        print(f"Error: '{target}' is not a directory.", file=sys.stderr)
+        return 1
+
+    # ── Load existing project stubs from the local registry mirror ────────
+    local_projects = load_local_projects(get_home_dir())
+    stubs = [(pid, str(p.root)) for pid, p in local_projects.items()]
+
+    # ── Idempotency check ─────────────────────────────────────────────────
+    already, existing_id = _is_already_registered(stubs, target)
+    if already:
+        if verbose:
+            print(f"Project already registered as '{existing_id}' — nothing to do.")
+        return 0
+
+    # ── Derive project_id ─────────────────────────────────────────────────
+    existing_ids = {pid for pid, _ in stubs}
+    project_id = _unique_project_id(_slugify(target.name), existing_ids)
+
+    if verbose:
+        print(f"Initializing '{project_id}' from {target} …")
+
+    # ── Detect languages ──────────────────────────────────────────────────
+    if verbose:
+        print("  Scanning source files …")
+    ext_counts = _walk_extensions(target)
+    languages = _detect_languages(ext_counts)
+    if verbose:
+        print(f"  Languages detected : {', '.join(languages)}")
+
+    # ── Detect framework ──────────────────────────────────────────────────
+    framework_info = _detect_framework(target)
+    if verbose and framework_info.get("tech_stack"):
+        print(f"  Tech stack detected: {', '.join(framework_info['tech_stack'])}")
+
+    # ── Build ignore list ─────────────────────────────────────────────────
+    ignore = _build_ignore_list(framework_info.get("extra_ignore", []))
+
+    # ── Mint a JWT naming this project, then fetch shared defaults ────────
+    jwt, fingerprint_attestation = _mint_jwt_for_project(project_id, target)
+
+    async def _load_defaults() -> dict:
+        from codepreproc_client.layer1_business.api_client.registry_client import get_defaults
+        from codepreproc_client.layer1_business.config import get_server_url
+
+        try:
+            config = await get_defaults(base_url=get_server_url(), jwt=jwt)
+            return config.model_dump(mode="python")
+        except Exception:
+            return {}
+
+    defaults: dict = asyncio.run(_load_defaults())
+
+    # ── Generate project_context via LLM (optional) ───────────────────────
+    if verbose:
+        print("  Generating project_context …")
+    summary = _gather_summary(target, framework_info)
+    context = _generate_context(target.name, summary, defaults)
+    if context is None:
+        if verbose:
+            print("  LLM unavailable — using template fallback.")
+        context = _fallback_context(target.name, languages, framework_info)
+
+    # ── Persist project to PostgreSQL ─────────────────────────────────────
+    qdrant_collection = f"{project_id}_code"
+    from contracts.dtos import ProjectConfig, ValidationConfig
+
+    project_obj = ProjectConfig(
+        project_id=project_id,
+        root=target,
+        languages=languages,
+        qdrant_collection=qdrant_collection,
+        branch_aware=False,
+        project_context=context,
+        ignore=ignore,
+        llm_policy=defaults.get("default_llm_policy", "local_only"),
+        validation=ValidationConfig(),
+    )
+    _persist_project(project_obj, jwt, fingerprint_attestation)
+
+    if verbose:
+        print(f"\nProject '{project_id}' registered")
+        print(f"  root              : {target}")
+        print(f"  languages         : {', '.join(languages)}")
+        print(f"  qdrant_collection : {qdrant_collection}")
+
+    # ── Bootstrap in SuperKG via the codepreproc-server proxy (optional, fire-and-forget) ─
+    superkg_enabled = os.getenv("SUPERKG_ENABLED", "1").strip().lower() not in {"0", "false", "no", "off"}
+    if superkg_enabled:
+        from codepreproc_client.layer1_business.api_client.superkg_client import SuperKGClient
+        from codepreproc_client.layer1_business.config import get_server_url
+
+        async def _jwt_provider(_project_id: str) -> str:
+            return jwt
+
+        superkg_client = SuperKGClient(base_url=get_server_url(), jwt_provider=_jwt_provider, enabled=True)
+        system_id = asyncio.run(superkg_client.bootstrap(project_id, target.name, context))
+        if system_id:
+            _set_superkg_id(project_id, system_id, jwt)
+            if verbose:
+                print(f"  superkg_system_id : {system_id}")
+        elif verbose:
+            print("  superkg           : unavailable (skipped)")
+
+    if verbose:
+        print("\nRun 'codepreproc-mcp' to start the server, then use the 'reindex' tool to index the project.")
+
+    return 0

codepreproc_client/layer1_business/cli/main.py

@@ -0,0 +1,78 @@
+"""CLI entrypoint for codepreproc.
+
+Usage
+-----
+  codepreproc init [PATH]
+      Register a project in the codepreproc registry.
+      PATH defaults to the current working directory.
+      Exits immediately with no changes if the project is already registered.
+"""
+
+from __future__ import annotations
+
+import sys
+from pathlib import Path
+
+from dotenv import load_dotenv
+
+load_dotenv(Path(__file__).parents[4] / ".env", override=False)
+
+
+_USAGE = """\
+Usage: codepreproc <command> [options]
+
+Commands:
+  init [PATH]   Scan PATH (default: CWD), mint a JWT for it, and register it
+                with codepreproc-server over the API.
+                Requires credentials.yaml and CODEPREPROC_SERVER_URL to be set.
+
+Options:
+  -h, --help    Show this message and exit.
+
+Note: db-init, db-seed, and license-add are codepreproc-server-admin operator
+commands now (they touch the shared Postgres instance directly) - run
+`codepreproc-server-admin` on the server install, not this client.
+"""
+
+
+def run() -> None:
+    """Console script entry point: codepreproc."""
+    args = sys.argv[1:]
+
+    if not args or args[0] in ("-h", "--help"):
+        print(_USAGE, end="")
+        sys.exit(0)
+
+    command = args[0]
+
+    if command == "init":
+        from codepreproc_client.layer1_business.cli.init import init_project
+
+        path: Path | None = None
+        remaining = [a for a in args[1:] if not a.startswith("-")]
+        flags = [a for a in args[1:] if a.startswith("-")]
+
+        if "-h" in flags or "--help" in flags:
+            print("Usage: codepreproc init [PATH]\n")
+            print("  Scan PATH (default: CWD) and register it in PostgreSQL.")
+            print("  Does nothing if the project root is already registered.")
+            sys.exit(0)
+
+        if remaining:
+            path = Path(remaining[0])
+
+        sys.exit(init_project(path=path))
+
+    elif command in ("db-init", "db-seed", "license-add"):
+        print(
+            f"codepreproc: '{command}' is now a codepreproc-server-admin operator command "
+            "(it touches the shared Postgres instance directly). Run "
+            f"`codepreproc-server-admin` on the server install, not this client.",
+            file=sys.stderr,
+        )
+        sys.exit(1)
+
+    else:
+        print(f"codepreproc: unknown command '{command}'", file=sys.stderr)
+        print("Run 'codepreproc --help' for usage.", file=sys.stderr)
+        sys.exit(1)