superlocalmemory 2.8.5 → 3.0.0

package/src/superlocalmemory/learning/project_context.py

@@ -0,0 +1,366 @@
+# Copyright (c) 2026 Varun Pratap Bhardwaj / Qualixar
+# Licensed under the MIT License - see LICENSE file
+# Part of SuperLocalMemory V3 | https://qualixar.com | https://varunpratap.com
+
+"""Project context detection via filesystem analysis.
+
+Scans a directory for project manifest files (package.json, pyproject.toml,
+Cargo.toml, etc.) and extracts project metadata: name, languages, frameworks,
+build tools.  No database access -- pure filesystem analysis.
+
+Ported from V2 ProjectContextManager.  V2 memory.db multi-signal detection
+replaced with direct filesystem scanning (the DB-based signals are handled
+by the retrieval layer in V3).
+
+Part of Qualixar | Author: Varun Pratap Bhardwaj
+"""
+
+from __future__ import annotations
+
+import json
+import logging
+import re
+from pathlib import Path
+from typing import Any, Optional
+
+logger = logging.getLogger(__name__)
+
+
+# ---------------------------------------------------------------------------
+# Manifest definitions
+# ---------------------------------------------------------------------------
+
+# Each manifest maps a filename to a parser function name.
+# The parser extracts (project_name, languages, frameworks) from the file.
+
+_MANIFEST_FILES: list[dict[str, Any]] = [
+    {
+        "filename": "package.json",
+        "languages": ["javascript", "typescript"],
+        "parser": "_parse_package_json",
+    },
+    {
+        "filename": "pyproject.toml",
+        "languages": ["python"],
+        "parser": "_parse_pyproject_toml",
+    },
+    {
+        "filename": "Cargo.toml",
+        "languages": ["rust"],
+        "parser": "_parse_cargo_toml",
+    },
+    {
+        "filename": "go.mod",
+        "languages": ["go"],
+        "parser": "_parse_go_mod",
+    },
+    {
+        "filename": "pom.xml",
+        "languages": ["java"],
+        "parser": "_parse_pom_xml",
+    },
+    {
+        "filename": "build.gradle",
+        "languages": ["java", "kotlin"],
+        "parser": "_parse_gradle",
+    },
+    {
+        "filename": "build.gradle.kts",
+        "languages": ["kotlin"],
+        "parser": "_parse_gradle",
+    },
+    {
+        "filename": "Gemfile",
+        "languages": ["ruby"],
+        "parser": "_parse_gemfile",
+    },
+    {
+        "filename": "composer.json",
+        "languages": ["php"],
+        "parser": "_parse_composer_json",
+    },
+    {
+        "filename": "pubspec.yaml",
+        "languages": ["dart"],
+        "parser": "_parse_pubspec_yaml",
+    },
+]
+
+# Framework detection via dependency names
+_FRAMEWORK_HINTS: dict[str, str] = {
+    "react": "React",
+    "react-dom": "React",
+    "vue": "Vue",
+    "next": "Next.js",
+    "nuxt": "Nuxt",
+    "angular": "Angular",
+    "svelte": "Svelte",
+    "@sveltejs/kit": "SvelteKit",
+    "express": "Express",
+    "fastapi": "FastAPI",
+    "django": "Django",
+    "flask": "Flask",
+    "fastify": "Fastify",
+    "spring-boot": "Spring Boot",
+    "rails": "Rails",
+    "nestjs": "NestJS",
+    "@nestjs/core": "NestJS",
+    "tailwindcss": "Tailwind CSS",
+    "pytest": "pytest",
+    "jest": "Jest",
+    "vitest": "Vitest",
+    "torch": "PyTorch",
+    "tensorflow": "TensorFlow",
+    "transformers": "Hugging Face",
+}
+
+
+class ProjectContextManager:
+    """Detect project context from the filesystem.
+
+    No database needed -- scans the target directory for manifest files
+    and extracts project metadata.
+    """
+
+    def detect(self, path: Path | str) -> dict[str, Any]:
+        """Detect project context from a directory.
+
+        Args:
+            path: Directory to scan.
+
+        Returns:
+            Dict with keys: project_name, languages, frameworks,
+            build_tool, manifest_file. All values may be None/empty
+            when no manifest is found.
+        """
+        directory = Path(path)
+        if not directory.is_dir():
+            return _empty_context()
+
+        # Try each manifest file
+        for manifest_def in _MANIFEST_FILES:
+            manifest_path = directory / manifest_def["filename"]
+            if not manifest_path.exists():
+                continue
+
+            parser_name = manifest_def["parser"]
+            parser = _PARSERS.get(parser_name)
+            if parser is None:
+                continue
+
+            try:
+                info = parser(manifest_path)
+            except Exception as exc:
+                logger.debug(
+                    "Failed to parse %s: %s", manifest_path, exc
+                )
+                continue
+
+            # Merge manifest-defined languages
+            languages = list(
+                set(info.get("languages", []) + manifest_def["languages"])
+            )
+            return {
+                "project_name": info.get("project_name") or directory.name,
+                "languages": languages,
+                "frameworks": info.get("frameworks", []),
+                "build_tool": info.get("build_tool"),
+                "manifest_file": manifest_def["filename"],
+            }
+
+        # Fallback: infer from file extensions
+        return self._infer_from_extensions(directory)
+
+    # ------------------------------------------------------------------
+    # Fallback extension scanning
+    # ------------------------------------------------------------------
+
+    @staticmethod
+    def _infer_from_extensions(directory: Path) -> dict[str, Any]:
+        """Scan for source files and infer language from extensions."""
+        ext_map: dict[str, str] = {
+            ".py": "python",
+            ".js": "javascript",
+            ".ts": "typescript",
+            ".tsx": "typescript",
+            ".jsx": "javascript",
+            ".rs": "rust",
+            ".go": "go",
+            ".java": "java",
+            ".kt": "kotlin",
+            ".rb": "ruby",
+            ".php": "php",
+            ".dart": "dart",
+            ".swift": "swift",
+            ".cs": "csharp",
+            ".cpp": "cpp",
+            ".c": "c",
+        }
+        detected: set[str] = set()
+        try:
+            for child in directory.iterdir():
+                if child.is_file() and child.suffix in ext_map:
+                    detected.add(ext_map[child.suffix])
+        except PermissionError:
+            pass
+
+        return {
+            "project_name": directory.name,
+            "languages": sorted(detected),
+            "frameworks": [],
+            "build_tool": None,
+            "manifest_file": None,
+        }
+
+
+# ----------------------------------------------------------------------
+# Parsers (one per manifest type)
+# ----------------------------------------------------------------------
+
+
+def _parse_package_json(path: Path) -> dict[str, Any]:
+    with open(path, "r", encoding="utf-8") as f:
+        data = json.load(f)
+    deps = set(data.get("dependencies", {}).keys())
+    deps |= set(data.get("devDependencies", {}).keys())
+    frameworks = [
+        _FRAMEWORK_HINTS[d] for d in deps if d in _FRAMEWORK_HINTS
+    ]
+    lang = ["typescript"] if "typescript" in deps else ["javascript"]
+    return {
+        "project_name": data.get("name", ""),
+        "languages": lang,
+        "frameworks": sorted(set(frameworks)),
+        "build_tool": "npm",
+    }
+
+
+def _parse_pyproject_toml(path: Path) -> dict[str, Any]:
+    text = path.read_text(encoding="utf-8")
+    name = ""
+    # Simple TOML name extraction (no toml library required)
+    m = re.search(r'name\s*=\s*["\']([^"\']+)["\']', text)
+    if m:
+        name = m.group(1)
+
+    # Detect frameworks from dependencies section
+    frameworks: list[str] = []
+    for hint_key, hint_name in _FRAMEWORK_HINTS.items():
+        if hint_key in text.lower():
+            frameworks.append(hint_name)
+
+    build_tool = "poetry" if "[tool.poetry]" in text else "setuptools"
+    if "[tool.hatch]" in text:
+        build_tool = "hatch"
+
+    return {
+        "project_name": name,
+        "languages": ["python"],
+        "frameworks": sorted(set(frameworks)),
+        "build_tool": build_tool,
+    }
+
+
+def _parse_cargo_toml(path: Path) -> dict[str, Any]:
+    text = path.read_text(encoding="utf-8")
+    m = re.search(r'name\s*=\s*"([^"]+)"', text)
+    return {
+        "project_name": m.group(1) if m else "",
+        "languages": ["rust"],
+        "frameworks": [],
+        "build_tool": "cargo",
+    }
+
+
+def _parse_go_mod(path: Path) -> dict[str, Any]:
+    text = path.read_text(encoding="utf-8")
+    m = re.search(r"^module\s+(\S+)", text, re.MULTILINE)
+    module_name = m.group(1) if m else ""
+    project_name = module_name.split("/")[-1] if module_name else ""
+    return {
+        "project_name": project_name,
+        "languages": ["go"],
+        "frameworks": [],
+        "build_tool": "go",
+    }
+
+
+def _parse_pom_xml(path: Path) -> dict[str, Any]:
+    text = path.read_text(encoding="utf-8")
+    m = re.search(r"<artifactId>([^<]+)</artifactId>", text)
+    frameworks = []
+    if "spring-boot" in text.lower():
+        frameworks.append("Spring Boot")
+    return {
+        "project_name": m.group(1) if m else "",
+        "languages": ["java"],
+        "frameworks": frameworks,
+        "build_tool": "maven",
+    }
+
+
+def _parse_gradle(path: Path) -> dict[str, Any]:
+    return {
+        "project_name": path.parent.name,
+        "languages": ["java", "kotlin"],
+        "frameworks": [],
+        "build_tool": "gradle",
+    }
+
+
+def _parse_gemfile(path: Path) -> dict[str, Any]:
+    text = path.read_text(encoding="utf-8")
+    frameworks = ["Rails"] if "rails" in text.lower() else []
+    return {
+        "project_name": path.parent.name,
+        "languages": ["ruby"],
+        "frameworks": frameworks,
+        "build_tool": "bundler",
+    }
+
+
+def _parse_composer_json(path: Path) -> dict[str, Any]:
+    with open(path, "r", encoding="utf-8") as f:
+        data = json.load(f)
+    return {
+        "project_name": data.get("name", "").split("/")[-1],
+        "languages": ["php"],
+        "frameworks": [],
+        "build_tool": "composer",
+    }
+
+
+def _parse_pubspec_yaml(path: Path) -> dict[str, Any]:
+    text = path.read_text(encoding="utf-8")
+    m = re.search(r"^name:\s*(\S+)", text, re.MULTILINE)
+    frameworks = ["Flutter"] if "flutter" in text.lower() else []
+    return {
+        "project_name": m.group(1) if m else "",
+        "languages": ["dart"],
+        "frameworks": frameworks,
+        "build_tool": "pub",
+    }
+
+
+# Parser dispatch table
+_PARSERS: dict[str, Any] = {
+    "_parse_package_json": _parse_package_json,
+    "_parse_pyproject_toml": _parse_pyproject_toml,
+    "_parse_cargo_toml": _parse_cargo_toml,
+    "_parse_go_mod": _parse_go_mod,
+    "_parse_pom_xml": _parse_pom_xml,
+    "_parse_gradle": _parse_gradle,
+    "_parse_gemfile": _parse_gemfile,
+    "_parse_composer_json": _parse_composer_json,
+    "_parse_pubspec_yaml": _parse_pubspec_yaml,
+}
+
+
+def _empty_context() -> dict[str, Any]:
+    return {
+        "project_name": None,
+        "languages": [],
+        "frameworks": [],
+        "build_tool": None,
+        "manifest_file": None,
+    }

package/src/superlocalmemory/learning/ranker.py

@@ -0,0 +1,155 @@
+# Copyright (c) 2026 Varun Pratap Bhardwaj / Qualixar
+# Licensed under the MIT License - see LICENSE file
+# Part of SuperLocalMemory V3 | https://qualixar.com | https://varunpratap.com
+
+"""3-phase adaptive ranker — from heuristic to ML.
+
+Phase 1: cross-encoder score only (cold start)
+Phase 2: heuristic boosts (some data)
+Phase 3: LightGBM model (enough training data)
+
+Transitions are automatic based on accumulated training data.
+"""
+
+from __future__ import annotations
+
+import logging
+import math
+from typing import Any
+
+from superlocalmemory.learning.features import FeatureExtractor, FeatureVector, FEATURE_DIM
+
+logger = logging.getLogger(__name__)
+
+# Phase thresholds
+PHASE_2_THRESHOLD = 50   # signals needed to enter Phase 2
+PHASE_3_THRESHOLD = 200  # signals needed to enter Phase 3
+
+
+class AdaptiveRanker:
+    """3-phase adaptive re-ranker for V3 retrieval results."""
+
+    def __init__(self, signal_count: int = 0, model_state: bytes | None = None) -> None:
+        self._signal_count = signal_count
+        self._model = None
+        if model_state:
+            self._load_model(model_state)
+
+    @property
+    def phase(self) -> int:
+        if self._signal_count >= PHASE_3_THRESHOLD and self._model is not None:
+            return 3
+        if self._signal_count >= PHASE_2_THRESHOLD:
+            return 2
+        return 1
+
+    @property
+    def signal_count(self) -> int:
+        return self._signal_count
+
+    @signal_count.setter
+    def signal_count(self, value: int) -> None:
+        self._signal_count = value
+
+    def rerank(self, results: list[dict], query_context: dict) -> list[dict]:
+        """Re-rank retrieval results based on current phase."""
+        if not results:
+            return results
+
+        if self.phase == 3:
+            return self._rerank_ml(results, query_context)
+        elif self.phase == 2:
+            return self._rerank_heuristic(results, query_context)
+        else:
+            return self._rerank_baseline(results)
+
+    def train(self, training_data: list[dict]) -> bool:
+        """Train LightGBM model on labeled data. Returns True if model was trained."""
+        if len(training_data) < PHASE_3_THRESHOLD:
+            return False
+
+        try:
+            import lightgbm as lgb
+        except ImportError:
+            logger.warning("LightGBM not installed. Phase 3 ranking unavailable.")
+            return False
+
+        features_list = []
+        labels = []
+        for item in training_data:
+            fv = item.get("features", {})
+            label = item.get("label", 0.0)
+            # Convert feature dict to ordered list
+            vec = [fv.get(name, 0.0) for name in FeatureExtractor.extract(
+                {"channel_scores": {}, "fact": {}}, {"query_type": ""}
+            ).features.keys()]
+            # Simpler: just use the feature values in order
+            from superlocalmemory.learning.features import FEATURE_NAMES
+            vec = [float(fv.get(name, 0.0)) for name in FEATURE_NAMES]
+            features_list.append(vec)
+            labels.append(float(label))
+
+        if not features_list:
+            return False
+
+        dataset = lgb.Dataset(features_list, label=labels)
+        params = {
+            "objective": "binary",
+            "metric": "binary_logloss",
+            "num_leaves": 15,
+            "learning_rate": 0.1,
+            "verbose": -1,
+        }
+        self._model = lgb.train(params, dataset, num_boost_round=50)
+        logger.info("LightGBM model trained with %d examples", len(features_list))
+        return True
+
+    def get_model_state(self) -> bytes | None:
+        """Serialize model for persistence."""
+        if self._model is None:
+            return None
+        return self._model.model_to_string().encode("utf-8")
+
+    # -- Phase implementations --
+
+    def _rerank_baseline(self, results: list[dict]) -> list[dict]:
+        """Phase 1: rank by cross-encoder score."""
+        return sorted(results, key=lambda r: r.get("cross_encoder_score", r.get("score", 0)), reverse=True)
+
+    def _rerank_heuristic(self, results: list[dict], query_context: dict) -> list[dict]:
+        """Phase 2: heuristic boosts on top of cross-encoder."""
+        scored = []
+        for r in results:
+            base = r.get("cross_encoder_score", r.get("score", 0))
+            # Boosts
+            recency_boost = 0.1 * math.exp(-r.get("fact", {}).get("age_days", 30) / 30)
+            access_boost = 0.05 * min(r.get("fact", {}).get("access_count", 0) / 10, 1.0)
+            trust_boost = 0.1 * (r.get("trust_score", 0.5) - 0.5)
+            final = base + recency_boost + access_boost + trust_boost
+            scored.append({**r, "_adaptive_score": final})
+        return sorted(scored, key=lambda r: r["_adaptive_score"], reverse=True)
+
+    def _rerank_ml(self, results: list[dict], query_context: dict) -> list[dict]:
+        """Phase 3: LightGBM prediction."""
+        if self._model is None:
+            return self._rerank_heuristic(results, query_context)
+
+        feature_vectors = FeatureExtractor.extract_batch(results, query_context)
+        predictions = []
+        for fv in feature_vectors:
+            vec = [fv.to_list()]
+            pred = self._model.predict(vec)[0]
+            predictions.append(pred)
+
+        paired = list(zip(results, predictions))
+        paired.sort(key=lambda x: x[1], reverse=True)
+        return [r for r, _ in paired]
+
+    def _load_model(self, state: bytes) -> None:
+        """Load model from serialized state."""
+        try:
+            import lightgbm as lgb
+            self._model = lgb.Booster(model_str=state.decode("utf-8"))
+        except (ImportError, Exception) as exc:
+            logger.warning("Could not load LightGBM model: %s", exc)
+            self._model = None