flurryx-code-memory 0.4.0__py3-none-any.whl

code_memory/extractor/gitignore.py

@@ -0,0 +1,162 @@
+"""Lightweight .gitignore matcher with no external deps.
+
+Supports the subset of gitignore syntax that matters in practice:
+
+- blank lines + ``#`` comments
+- ``!`` negation
+- ``/`` anchored (root-relative) vs unanchored patterns
+- trailing ``/`` directory-only patterns
+- ``*`` (no slash), ``?``, ``**`` globs
+- collects ``.gitignore`` files recursively from the root downward
+
+This is deliberately not a full re-implementation of git's matcher; the goal
+is to avoid indexing build artifacts (``.angular/``, ``tmp/``, ``dist/``…)
+that repos already ignore.
+"""
+
+from __future__ import annotations
+
+import fnmatch
+import re
+from dataclasses import dataclass
+from pathlib import Path
+
+
+@dataclass(frozen=True)
+class _Rule:
+    base: Path  # directory containing the .gitignore that owns the rule
+    pattern: str
+    negate: bool
+    dir_only: bool
+    anchored: bool
+    regex: re.Pattern[str]
+
+
+def _compile(pattern: str) -> re.Pattern[str]:
+    parts = pattern.split("/")
+    out: list[str] = []
+    for i, seg in enumerate(parts):
+        if seg == "**":
+            out.append(".*")
+        else:
+            # fnmatch.translate adds anchors we don't want — strip them
+            tr = fnmatch.translate(seg)
+            # python's translate yields "(?s:...)\\Z"; extract inner
+            m = re.match(r"\(\?s:(.*)\)\\Z", tr)
+            inner = m.group(1) if m else tr
+            out.append(inner)
+        if i != len(parts) - 1:
+            out.append("/")
+    body = "".join(out)
+    # collapse leading "**/" -> "(?:.*/)?" to allow zero-segment match
+    body = re.sub(r"^\.\*/", "(?:.*/)?", body)
+    return re.compile(rf"^{body}$")
+
+
+def _parse_line(line: str, base: Path) -> _Rule | None:
+    raw = line.rstrip("\n").rstrip("\r")
+    if not raw or raw.lstrip().startswith("#"):
+        return None
+    negate = raw.startswith("!")
+    if negate:
+        raw = raw[1:]
+    raw = raw.strip()
+    if not raw:
+        return None
+    dir_only = raw.endswith("/")
+    if dir_only:
+        raw = raw[:-1]
+    anchored = "/" in raw and not raw.startswith("**/")
+    if raw.startswith("/"):
+        raw = raw[1:]
+    return _Rule(
+        base=base,
+        pattern=raw,
+        negate=negate,
+        dir_only=dir_only,
+        anchored=anchored,
+        regex=_compile(raw),
+    )
+
+
+class GitignoreMatcher:
+    """Walk-aware gitignore matcher.
+
+    Load all ``.gitignore`` files under ``root`` upfront, then call
+    :meth:`match` per candidate path during the walk.
+    """
+
+    def __init__(self, root: Path, rules: list[_Rule]) -> None:
+        self._root = root.resolve()
+        self._rules = rules
+
+    @classmethod
+    def from_root(cls, root: str | Path) -> GitignoreMatcher:
+        root_path = Path(root).resolve()
+        rules: list[_Rule] = []
+        if not root_path.is_dir():
+            return cls(root_path, rules)
+        # always seed with .git/ so we never index the git directory
+        rules.append(
+            _Rule(
+                base=root_path,
+                pattern=".git",
+                negate=False,
+                dir_only=True,
+                anchored=False,
+                regex=_compile(".git"),
+            )
+        )
+        for gi in root_path.rglob(".gitignore"):
+            try:
+                lines = gi.read_text(encoding="utf-8", errors="replace").splitlines()
+            except OSError:
+                continue
+            base = gi.parent.resolve()
+            for line in lines:
+                rule = _parse_line(line, base)
+                if rule is not None:
+                    rules.append(rule)
+        return cls(root_path, rules)
+
+    def match(self, path: Path, *, is_dir: bool) -> bool:
+        """Return True if ``path`` is ignored. Last matching rule wins.
+
+        Per gitignore semantics, a file is ignored if any of its ancestor
+        directories is ignored, so we evaluate the chain root -> path and
+        return the final state.
+        """
+        try:
+            abs_path = path.resolve()
+        except OSError:
+            return False
+        # Build list of (candidate_path, is_dir) from root toward the leaf.
+        chain: list[tuple[Path, bool]] = []
+        for parent in reversed(abs_path.parents):
+            if self._root in (parent, *parent.parents) or parent == self._root:
+                chain.append((parent, True))
+        chain.append((abs_path, is_dir))
+
+        ignored = False
+        for candidate, candidate_is_dir in chain:
+            if candidate == self._root:
+                continue
+            for rule in self._rules:
+                if rule.dir_only and not candidate_is_dir:
+                    continue
+                try:
+                    rel_from_base = candidate.relative_to(rule.base)
+                except ValueError:
+                    continue
+                rel_str = rel_from_base.as_posix()
+                if rel_str in ("", "."):
+                    continue
+                if rule.anchored:
+                    hit = bool(rule.regex.match(rel_str))
+                else:
+                    hit = bool(rule.regex.match(rel_str)) or any(
+                        rule.regex.match(part) for part in rel_from_base.parts
+                    )
+                if hit:
+                    ignored = not rule.negate
+        return ignored

code_memory/extractor/nuget.py

@@ -0,0 +1,275 @@
+"""Resolve `.csproj` references to physical DLL paths.
+
+The graph layer wants Assembly nodes keyed by `Name, Version=…`.
+Csproj parsing gives us PackageReference + ProjectReference logical
+identities; this module turns those into concrete `.dll` files on
+disk so the metadata reader has something to open.
+
+Two sources, in priority order:
+
+1. **NuGet global cache** at ``$NUGET_PACKAGES`` or ``~/.nuget/packages``.
+   Standard layout:
+   ``{cache}/{pkg_lower}/{version}/lib/{tfm}/{Foo}.dll``.
+2. **Build output** of project references:
+   ``{ref_csproj_dir}/bin/{config}/{tfm}/{AssemblyName}.dll``.
+   We pick Debug if both exist; the contents differ only in PDBs /
+   optimisation, not in the public type surface that we index.
+
+Either source can be missing (offline machine, build never run);
+unresolved references are skipped — we never emit fictional paths.
+"""
+
+from __future__ import annotations
+
+import os
+from collections.abc import Iterable
+from dataclasses import dataclass
+from pathlib import Path
+
+from .csproj import CsprojInfo, PackageRef
+
+
+def nuget_cache_dirs() -> list[Path]:
+    """Return candidate roots in priority order, deduped.
+
+    ``NUGET_PACKAGES`` overrides the default. We also probe both the
+    POSIX (``~/.nuget``) and Windows-style (``%USERPROFILE%\\.nuget``)
+    locations because cross-platform repos sometimes drag both.
+    """
+    candidates: list[Path] = []
+    env = os.environ.get("NUGET_PACKAGES")
+    if env:
+        candidates.append(Path(env).expanduser())
+    candidates.append(Path.home() / ".nuget" / "packages")
+    # Dedupe while preserving order.
+    seen: set[Path] = set()
+    out: list[Path] = []
+    for c in candidates:
+        r = c.resolve() if c.exists() else c
+        if r in seen:
+            continue
+        seen.add(r)
+        if r.exists():
+            out.append(r)
+    return out
+
+
+# TFM compatibility table — only the broad strokes. Real NuGet
+# compatibility is a directed graph of moniker rules; we cover the
+# common parent-fallback so resolution doesn't silently miss anything
+# obvious. Order matters: earlier = preferred.
+_TFM_FALLBACKS: dict[str, tuple[str, ...]] = {
+    "net8.0": (
+        "net8.0",
+        "net7.0",
+        "net6.0",
+        "net5.0",
+        "netstandard2.1",
+        "netstandard2.0",
+        "netstandard1.6",
+    ),
+    "net7.0": (
+        "net7.0",
+        "net6.0",
+        "net5.0",
+        "netstandard2.1",
+        "netstandard2.0",
+    ),
+    "net6.0": (
+        "net6.0",
+        "net5.0",
+        "netstandard2.1",
+        "netstandard2.0",
+    ),
+    "net5.0": ("net5.0", "netstandard2.1", "netstandard2.0"),
+    "netstandard2.1": ("netstandard2.1", "netstandard2.0"),
+    "netstandard2.0": ("netstandard2.0", "netstandard1.6", "netstandard1.4"),
+    "net48": ("net48", "net472", "net471", "net47", "net462", "net461", "net46", "net45", "netstandard2.0"),
+    "net472": ("net472", "net471", "net47", "net462", "net461", "net46", "net45", "netstandard2.0"),
+    "net471": ("net471", "net47", "net462", "net461", "net46", "net45", "netstandard2.0"),
+    "net47": ("net47", "net462", "net461", "net46", "net45", "netstandard2.0"),
+    "net46": ("net46", "net45", "netstandard1.4", "netstandard1.3"),
+    "net45": ("net45", "netstandard1.3", "netstandard1.2"),
+}
+
+
+def _candidate_tfms(target: str | None) -> list[str]:
+    """Return TFMs to try, in priority order, for a project's <TargetFramework>.
+
+    Multi-target projects come in as a semicolon-joined string
+    (``net6.0;net8.0``); we expand each side and concatenate fallback
+    chains, then dedupe.
+    """
+    if not target:
+        return ["netstandard2.0", "netstandard2.1", "net8.0", "net6.0"]
+    chunks = [t.strip() for t in target.split(";") if t.strip()]
+    out: list[str] = []
+    seen: set[str] = set()
+    for c in chunks:
+        for tfm in _TFM_FALLBACKS.get(c, (c,)):
+            if tfm not in seen:
+                seen.add(tfm)
+                out.append(tfm)
+    return out
+
+
+def resolve_package_dlls(
+    pkg: PackageRef, target_framework: str | None
+) -> list[Path]:
+    """Find DLLs for a single ``PackageReference`` in any local NuGet cache.
+
+    Returns all DLLs under the best-matching TFM directory — most
+    packages ship one or two assemblies per TFM but some (Roslyn,
+    BouncyCastle) ship a bag.
+    """
+    caches = nuget_cache_dirs()
+    if not caches:
+        return []
+
+    name_lower = pkg.name.lower()
+    versions = _versions_for(caches, name_lower, pkg.version)
+    tfms = _candidate_tfms(target_framework)
+
+    results: list[Path] = []
+    for cache in caches:
+        for ver in versions:
+            lib_root = cache / name_lower / ver / "lib"
+            if not lib_root.is_dir():
+                continue
+            for tfm in tfms:
+                tfm_dir = lib_root / tfm
+                if tfm_dir.is_dir():
+                    results.extend(sorted(tfm_dir.glob("*.dll")))
+                    if results:
+                        return results
+            # No exact/fallback TFM match — let any flat DLL stand in
+            # for older packages that ship `lib/*.dll` without a TFM
+            # subdir. Better than nothing for net2.0-era libs.
+            for dll in sorted(lib_root.glob("*.dll")):
+                if dll.is_file():
+                    results.append(dll)
+            if results:
+                return results
+    return results
+
+
+def _versions_for(
+    caches: list[Path], name_lower: str, requested: str | None
+) -> list[str]:
+    """Pick which on-disk version directories to consider, best first.
+
+    Requested version present on disk → use it directly. Otherwise
+    return all versions sorted descending so we try newer first
+    (lexicographic on padded chunks is good enough for the
+    Major.Minor.Patch[-suffix] layout NuGet emits).
+    """
+    if requested:
+        # Direct hit if the requested version directory exists in any cache.
+        for cache in caches:
+            if (cache / name_lower / requested).is_dir():
+                return [requested]
+    # Fall back to anything we have on disk.
+    seen: set[str] = set()
+    for cache in caches:
+        d = cache / name_lower
+        if not d.is_dir():
+            continue
+        for child in d.iterdir():
+            if child.is_dir():
+                seen.add(child.name)
+    return sorted(seen, key=_version_sort_key, reverse=True)
+
+
+def _version_sort_key(version: str) -> tuple[int, ...]:
+    """Coarse semver-ish sort: split on dots, zero-pad missing chunks.
+
+    Pre-release suffixes (``1.2.3-beta``) compare lower than their
+    release equivalents because the int parse strips at the first
+    non-digit. Good enough for "newest first" without dragging in a
+    real packaging dep.
+    """
+    parts = version.split(".")
+    out: list[int] = []
+    for p in parts[:4]:  # cap to four segments
+        digits = ""
+        for ch in p:
+            if ch.isdigit():
+                digits += ch
+            else:
+                break
+        out.append(int(digits) if digits else 0)
+    while len(out) < 4:
+        out.append(0)
+    return tuple(out)
+
+
+def resolve_project_reference_dlls(
+    ref_csproj: Path, target_framework: str | None
+) -> list[Path]:
+    """Find the built DLL for a sibling project, if any.
+
+    Looks under ``{proj_dir}/bin/{config}/{tfm}/`` for each candidate
+    tfm in priority order, preferring Debug over Release because dev
+    workstations build Debug by default and that's the file most
+    likely to exist.
+    """
+    base = ref_csproj.parent
+    tfms = _candidate_tfms(target_framework)
+    for config in ("Debug", "Release"):
+        for tfm in tfms:
+            out = base / "bin" / config / tfm
+            if out.is_dir():
+                dlls = sorted(out.glob("*.dll"))
+                if dlls:
+                    return dlls
+    # Older non-SDK projects sometimes drop output straight in bin/.
+    flat = base / "bin"
+    if flat.is_dir():
+        return sorted(flat.glob("*.dll"))
+    return []
+
+
+@dataclass
+class ResolvedRefs:
+    """All DLL paths resolved for one csproj."""
+
+    package_dlls: dict[str, list[Path]]  # PackageRef.name -> dlls
+    project_dlls: dict[str, list[Path]]  # referenced csproj path -> dlls
+
+    def all_paths(self) -> list[Path]:
+        out: list[Path] = []
+        for v in self.package_dlls.values():
+            out.extend(v)
+        for v in self.project_dlls.values():
+            out.extend(v)
+        return out
+
+
+def resolve_refs(info: CsprojInfo) -> ResolvedRefs:
+    """Resolve every PackageReference + ProjectReference of one csproj.
+
+    Returns an empty ``ResolvedRefs`` rather than raising on missing
+    NuGet cache / absent build outputs — callers care about "what did
+    we manage to find" more than about failures.
+    """
+    package_dlls: dict[str, list[Path]] = {}
+    for pkg in info.package_references:
+        dlls = resolve_package_dlls(pkg, info.target_framework)
+        if dlls:
+            package_dlls[pkg.name] = dlls
+
+    project_dlls: dict[str, list[Path]] = {}
+    for ref_path in info.project_references:
+        dlls = resolve_project_reference_dlls(Path(ref_path), info.target_framework)
+        if dlls:
+            project_dlls[ref_path] = dlls
+    return ResolvedRefs(package_dlls=package_dlls, project_dlls=project_dlls)
+
+
+def all_referenced_dlls(infos: Iterable[CsprojInfo]) -> set[Path]:
+    """Deduplicated set of every DLL referenced by any project."""
+    out: set[Path] = set()
+    for info in infos:
+        refs = resolve_refs(info)
+        out.update(refs.all_paths())
+    return out

code_memory/extractor/sanity.py

@@ -0,0 +1,124 @@
+"""Extraction sanity checks — catch UTF-8 / parser drift at ingest time.
+
+The historical UTF-8 byte-vs-str slicing bug silently chopped every
+identifier in non-ASCII files, and nobody noticed until a user got
+empty callers/callees a year later. This module exists so that class
+of regression fails loudly at ingest, not at user-report.
+
+The check is intentionally narrow: for each extracted Symbol whose
+name is a plain identifier, the snippet must contain that identifier
+as a substring. Anything more sophisticated would have to mirror the
+extractor's logic, which is exactly what we're trying to validate —
+so we keep this check independent and dumb on purpose.
+"""
+
+from __future__ import annotations
+
+import re
+from dataclasses import dataclass
+
+from .treesitter import ExtractedFile, Symbol
+
+# Plain identifier — letters / digits / underscore. Skips generics
+# (``Foo<T>``), operator overloads (``operator +``), F# parameterised
+# names, and anything else the extractor reasonably emits but where a
+# literal substring check would false-positive.
+_PLAIN_IDENT = re.compile(r"^[A-Za-z_][A-Za-z0-9_]*$")
+
+
+@dataclass(frozen=True)
+class SanityViolation:
+    """One symbol whose snippet didn't contain its name verbatim."""
+
+    path: str
+    name: str
+    kind: str
+    start_line: int
+
+
+def is_checkable(symbol: Symbol) -> bool:
+    """Whether this symbol's name is safe to round-trip against the snippet."""
+    return bool(_PLAIN_IDENT.match(symbol.name))
+
+
+def _contains_as_word(haystack: str, needle: str) -> bool:
+    """``needle`` appears in ``haystack`` as a complete word.
+
+    A plain substring check would miss the historical UTF-8 chop bug:
+    when ``CommandeRules`` got truncated to ``mmandeRules``, the
+    truncated name is still a substring of the snippet containing the
+    real ``CommandeRules`` declaration. Word boundaries close that
+    hole — ``mmandeRules`` is not a whole-word occurrence inside
+    ``CommandeRules``.
+    """
+    pattern = r"(?<![A-Za-z0-9_])" + re.escape(needle) + r"(?![A-Za-z0-9_])"
+    return re.search(pattern, haystack) is not None
+
+
+def violations_in(ex: ExtractedFile) -> list[SanityViolation]:
+    """Return symbols whose snippet doesn't contain their plain-identifier name.
+
+    Non-plain names (generics, operators, F# parameterised) are skipped
+    rather than flagged — they aren't reliably substring-checkable.
+    Returns ``[]`` on a clean file.
+    """
+    out: list[SanityViolation] = []
+    for sym in ex.symbols:
+        if not is_checkable(sym):
+            continue
+        if _contains_as_word(sym.snippet, sym.name):
+            continue
+        out.append(
+            SanityViolation(
+                path=ex.path,
+                name=sym.name,
+                kind=sym.kind,
+                start_line=sym.start_line,
+            )
+        )
+    return out
+
+
+@dataclass
+class SanitySummary:
+    """Aggregate counts across one ingest run."""
+
+    symbols_checked: int = 0
+    symbols_failed: int = 0
+    sample_violations: list[SanityViolation] = None  # type: ignore[assignment]
+
+    def __post_init__(self) -> None:
+        if self.sample_violations is None:
+            self.sample_violations = []
+
+    @property
+    def failure_rate(self) -> float:
+        if self.symbols_checked == 0:
+            return 0.0
+        return self.symbols_failed / self.symbols_checked
+
+    def record(self, ex: ExtractedFile, *, keep_samples: int = 10) -> None:
+        for sym in ex.symbols:
+            if not is_checkable(sym):
+                continue
+            self.symbols_checked += 1
+            if _contains_as_word(sym.snippet, sym.name):
+                continue
+            self.symbols_failed += 1
+            if len(self.sample_violations) < keep_samples:
+                self.sample_violations.append(
+                    SanityViolation(
+                        path=ex.path,
+                        name=sym.name,
+                        kind=sym.kind,
+                        start_line=sym.start_line,
+                    )
+                )
+
+
+# Threshold above which the ingest run flags itself as suspect. Tuned
+# from real corpora: a healthy ingest sits at 0% (every plain
+# identifier round-trips). The historical UTF-8 bug pushed the
+# failure rate close to 100% on French C# repos. Anything above ~2%
+# almost certainly means a real bug, not edge-case syntax.
+SUSPECT_THRESHOLD = 0.02

code_memory/extractor/sln.py

@@ -0,0 +1,108 @@
+"""Minimal Visual Studio `.sln` parser.
+
+A solution file is a project group. The parser pulls out just the
+project list — name, csproj path, and a stable GUID — which is enough
+to wire ``Solution`` graph nodes and ``MEMBER_OF`` edges from each
+indexed Project to its containing Solution.
+
+Folders ("solution items") are skipped; we want code projects, not
+the IDE's tree organization. Unparseable lines are dropped silently
+so a single corrupted entry never aborts the ingest.
+
+Format reference (informal — Microsoft never published a real
+grammar): https://learn.microsoft.com/en-us/visualstudio/extensibility/internals/solution-dot-sln-file
+"""
+
+from __future__ import annotations
+
+import logging
+import re
+from dataclasses import dataclass, field
+from pathlib import Path
+
+log = logging.getLogger(__name__)
+
+# ``Project("{type-guid}") = "Name", "RelativePath.csproj", "{proj-guid}"``
+_PROJECT_LINE = re.compile(
+    r'^Project\("\{(?P<type>[0-9A-Fa-f-]+)\}"\)\s*=\s*'
+    r'"(?P<name>[^"]+)",\s*'
+    r'"(?P<path>[^"]+)",\s*'
+    r'"\{(?P<guid>[0-9A-Fa-f-]+)\}"'
+)
+
+# "Solution folder" type GUID per MS docs — these have no buildable
+# output and shouldn't be indexed as projects.
+_FOLDER_TYPE_GUID = "2150E333-8FDC-42A3-9474-1A3956D46DE8"
+
+
+@dataclass(frozen=True)
+class SolutionProject:
+    """One project entry inside a .sln file."""
+
+    name: str
+    csproj_path: str  # absolute, resolved
+    guid: str
+    type_guid: str
+
+
+@dataclass
+class SolutionInfo:
+    """Parsed view of one .sln file."""
+
+    path: str
+    name: str
+    projects: list[SolutionProject] = field(default_factory=list)
+
+
+def parse_sln(sln_path: str | Path) -> SolutionInfo | None:
+    """Parse one `.sln`. Returns ``None`` on read failure."""
+    p = Path(sln_path).resolve()
+    try:
+        # .sln files are Windows-encoded MBCS-or-UTF8 with BOM in
+        # practice; ``utf-8-sig`` strips the BOM if present without
+        # caring otherwise.
+        text = p.read_text(encoding="utf-8-sig", errors="replace")
+    except OSError as e:
+        log.warning("sln: failed to read %s — %s", p, e)
+        return None
+
+    info = SolutionInfo(path=str(p), name=p.stem)
+    base = p.parent
+    for line in text.splitlines():
+        m = _PROJECT_LINE.match(line.strip())
+        if not m:
+            continue
+        type_guid = m.group("type").lower()
+        if type_guid == _FOLDER_TYPE_GUID.lower():
+            continue
+        rel = m.group("path").replace("\\", "/")
+        candidate = (base / rel).resolve()
+        if not candidate.exists():
+            # Some solutions reference projects outside the cloned
+            # working tree (shared infra). Skip — we can't index what
+            # we don't have on disk.
+            continue
+        info.projects.append(
+            SolutionProject(
+                name=m.group("name"),
+                csproj_path=str(candidate),
+                guid=m.group("guid").lower(),
+                type_guid=type_guid,
+            )
+        )
+    return info
+
+
+def walk_solutions(root: str | Path) -> list[SolutionInfo]:
+    """Find every `.sln` under ``root`` and parse it."""
+    out: list[SolutionInfo] = []
+    root_path = Path(root).resolve()
+    for sln in root_path.rglob("*.sln"):
+        # Skip artifacts in obvious build outputs; .sln rarely lives
+        # there but the filter is cheap and matches the csproj rule.
+        if any(part in {"bin", "obj", "node_modules"} for part in sln.parts):
+            continue
+        info = parse_sln(sln)
+        if info is not None:
+            out.append(info)
+    return out