code-recycle 0.1.0__py3-none-any.whl

code_recycle/__init__.py

@@ -0,0 +1,3 @@
+"""code_recycle: search your own code and verified sources before searching the web."""
+
+__version__ = "0.1.0"  # kept in step with pyproject.toml

code_recycle/chunks.py

@@ -0,0 +1,168 @@
+"""Split indexed files into searchable pieces ("chunks").
+
+- Python: one chunk per function, method and class, with exact line numbers.
+  Source: Python docs, ast module (research plan Topic 3).
+- Markdown research entries (URL, where on the page, what it taught, code pattern): one
+  structured chunk each. The format is the author's own, so this parser is original applied work.
+- Other Markdown: one chunk per heading section.
+- Anything else, or Python that doesn't parse: fixed windows of lines.
+"""
+import ast  # Python's own parser, so no third-party install is needed
+import re  # finding Markdown headings and research-entry fields
+from dataclasses import dataclass, field  # a plain record for one chunk
+
+WINDOW = 60  # lines per chunk for plain text
+OVERLAP = 10  # lines shared between neighbouring windows, so a match on a boundary isn't lost
+MAX_SECTION_LINES = 200  # a longer Markdown section is cut into windows
+
+
+@dataclass
+class Chunk:
+    path: str  # file path relative to its indexed folder
+    kind: str  # function, method, class, module, research_entry, section or text
+    name: str  # function or class name, heading text, or "" for plain text
+    start: int  # first line, counting from 1
+    end: int  # last line, inclusive
+    text: str  # the exact source text
+    fields: dict = field(default_factory=dict)  # research entries only: url, where, taught, code
+
+
+def chunk_file(path, text):
+    """All chunks for one file. `path` is only used for its extension and for labelling."""
+    if path.endswith((".py", ".pyi")):
+        return python_chunks(path, text)
+    if path.endswith(".md"):
+        return markdown_chunks(path, text)
+    return text_chunks(path, text)
+
+
+# ---------- Python ----------
+
+def python_chunks(path, text):
+    try:
+        tree = ast.parse(text)  # the whole module as a tree of nodes
+    except (SyntaxError, ValueError):  # broken or non-UTF-8 source: fall back to plain windows
+        return text_chunks(path, text)
+    lines = text.splitlines()
+    out = []
+    doc = ast.get_docstring(tree)  # the module's own description, if it has one
+    if doc:
+        first = tree.body[0]  # a docstring is always the first statement
+        out.append(Chunk(path, "module", path.rsplit("/", 1)[-1], first.lineno, first.end_lineno,
+                         _lines(lines, first.lineno, first.end_lineno)))
+    _visit(tree, path, lines, out, inside_class=None)
+    return out or text_chunks(path, text)  # a file of plain statements still gets indexed
+
+
+def _visit(node, path, lines, out, inside_class):
+    for child in ast.iter_child_nodes(node):
+        if isinstance(child, (ast.FunctionDef, ast.AsyncFunctionDef)):
+            start = _start_with_decorators(child)
+            name = f"{inside_class}.{child.name}" if inside_class else child.name
+            kind = "method" if inside_class else "function"
+            out.append(Chunk(path, kind, name, start, child.end_lineno, _lines(lines, start, child.end_lineno)))
+            # Functions defined inside functions stay part of their parent's chunk
+        elif isinstance(child, ast.ClassDef):
+            start = _start_with_decorators(child)
+            # The class chunk is its header, docstring and method names, not every line again,
+            # so a search for the class doesn't just return its biggest method's text twice
+            header_end = child.body[0].lineno - 1 if child.body else child.end_lineno
+            doc = ast.get_docstring(child) or ""
+            methods = [c.name for c in child.body if isinstance(c, (ast.FunctionDef, ast.AsyncFunctionDef))]
+            summary = _lines(lines, start, max(start, header_end))
+            if doc:
+                summary += "\n" + doc
+            if methods:
+                summary += "\nmethods: " + ", ".join(methods)
+            out.append(Chunk(path, "class", child.name, start, child.end_lineno, summary))
+            _visit(child, path, lines, out, inside_class=child.name)
+
+
+def _start_with_decorators(node):
+    # lineno is the "def" or "class" line; decorators sit above it and belong to it
+    return min([node.lineno] + [d.lineno for d in node.decorator_list])
+
+
+def _lines(lines, start, end):
+    return "\n".join(lines[start - 1:end])  # line numbers count from 1, list indexes from 0
+
+
+# ---------- Markdown ----------
+
+HEADING = re.compile(r"^(#{1,6})\s+(.*)")
+# Bold or plain: "- **URL:** x", "- URL: x", "**What it taught (my reading):** x".
+FIELD = re.compile(r"^\s*(?:[-*]\s+)?(?:\*\*)?(URLs?|Where on the page|What it taught|Code pattern from source)\b[^:*\n]*?:(?:\*\*)?\s?(.*)")
+FIELD_KEYS = {"URL": "url", "URLs": "url", "Where on the page": "where",
+              "What it taught": "taught", "Code pattern from source": "code"}
+
+
+def markdown_chunks(path, text):
+    lines = text.splitlines()
+    sections = []  # (heading text, first line, last line)
+    current_name, current_start, in_fence = "", 1, False
+    for i, line in enumerate(lines, start=1):
+        if line.lstrip().startswith("```"):
+            in_fence = not in_fence  # a "#" inside a code block is not a heading
+        m = None if in_fence else HEADING.match(line)
+        if m:
+            if i > current_start:
+                sections.append((current_name, current_start, i - 1))
+            current_name, current_start = m.group(2).strip(), i
+    if lines:
+        sections.append((current_name, current_start, len(lines)))
+
+    out = []
+    for name, start, end in sections:
+        body = lines[start - 1:end]
+        if not any(l.strip() for l in body):
+            continue  # an empty section has nothing to find
+        fields = research_fields(body)
+        if fields.get("url"):
+            out.append(Chunk(path, "research_entry", name, start, end, "\n".join(body), fields))
+        elif end - start + 1 <= MAX_SECTION_LINES:
+            out.append(Chunk(path, "section", name, start, end, "\n".join(body)))
+        else:
+            for c in _windows(path, body, start):
+                c.kind, c.name = "section", name
+                out.append(c)
+    return out
+
+
+def research_fields(body):
+    """The four fields of a research entry, as plain text. Empty dict if the section has no URL field."""
+    fields, key, in_fence = {}, None, False
+    for line in body:
+        if line.lstrip().startswith("```"):
+            in_fence = not in_fence  # inside a code block, "#" lines are code, not headings
+        m = None if in_fence else FIELD.match(line)
+        if m:
+            key = FIELD_KEYS[m.group(1)]
+            fields[key] = m.group(2).strip()
+        elif key and (in_fence or not HEADING.match(line)):
+            fields[key] = fields[key] + "\n" + line if fields[key] else line  # a field runs on until the next label
+    out = {k: v.strip() for k, v in fields.items()}
+    if out.get("code"):
+        out["code"] = strip_fences(out["code"])
+    return out
+
+
+def strip_fences(text):
+    """Code blocks without their ``` marker lines, so the code reads as code. Blank lines inside are kept."""
+    return "\n".join(l for l in text.splitlines() if not l.lstrip().startswith("```")).strip("\n")
+
+
+# ---------- Plain text ----------
+
+def text_chunks(path, text):
+    return _windows(path, text.splitlines(), 1)
+
+
+def _windows(path, lines, first_line):
+    out, step = [], WINDOW - OVERLAP
+    for i in range(0, max(len(lines), 1), step):
+        part = lines[i:i + WINDOW]
+        if any(l.strip() for l in part):
+            out.append(Chunk(path, "text", "", first_line + i, first_line + i + len(part) - 1, "\n".join(part)))
+        if i + WINDOW >= len(lines):
+            break  # this window reached the end of the file
+    return out

code_recycle/cli.py

@@ -0,0 +1,69 @@
+"""The code-recycle command: init, index, search and serve.
+
+Installed as a command by [project.scripts] in pyproject.toml (research plan Topic 12).
+"""
+import argparse  # reads the command-line options
+import os  # the CODE_RECYCLE_SETTINGS environment variable
+import sys  # exit codes and error output
+from pathlib import Path  # paths that work on every platform
+
+from . import __version__, files, index, search, settings as settings_mod
+
+DEFAULT_NAME = "code_recycle.toml"
+
+
+def find_settings(given):
+    """Which settings file to use: --settings, then $CODE_RECYCLE_SETTINGS, then ./code_recycle.toml,
+    then ~/.config/code_recycle/code_recycle.toml."""
+    if given:
+        return Path(given)
+    if os.environ.get("CODE_RECYCLE_SETTINGS"):
+        return Path(os.environ["CODE_RECYCLE_SETTINGS"])
+    local = Path.cwd() / DEFAULT_NAME
+    if local.exists():
+        return local
+    return Path.home() / ".config" / "code_recycle" / DEFAULT_NAME
+
+
+def main(argv=None):
+    p = argparse.ArgumentParser(prog="code-recycle", description="Search your own code and verified sources before the web.")
+    p.add_argument("--version", action="version", version=f"code-recycle {__version__}")
+    p.add_argument("--settings", help="settings file (default: ./code_recycle.toml, then ~/.config/code_recycle/)")
+    sub = p.add_subparsers(dest="command", required=True)
+    sub.add_parser("init", help="write an example settings file in this folder")
+    sub.add_parser("index", help="build or rebuild the index and print the index report")
+    q = sub.add_parser("search", help="search from the command line")
+    q.add_argument("query", nargs="+", help="what you're looking for, in plain words")
+    q.add_argument("--from", dest="from_path", help="the file or folder you're working in, for reuse rules")
+    q.add_argument("-n", "--limit", type=int, default=5)
+    sub.add_parser("serve", help="run the MCP server over stdio (what an MCP client launches)")
+    a = p.parse_args(argv)
+
+    if a.command == "init":
+        target = Path(a.settings or DEFAULT_NAME)
+        if target.exists():
+            print(f"{target} already exists; not overwriting it.", file=sys.stderr)
+            return 1
+        target.write_text(settings_mod.EXAMPLE)
+        print(f"Wrote {target}. Edit the folders list, then run: code-recycle index")
+        return 0
+
+    path = find_settings(a.settings)
+    try:
+        s = settings_mod.load(path)
+        if a.command == "index":
+            walked = index.build(s)
+            print(files.report(walked))
+        elif a.command == "search":
+            print(search.format_answer(search.search(s, " ".join(a.query), from_path=a.from_path, limit=a.limit)))
+        elif a.command == "serve":
+            from .server import build_server  # imported here so the other commands don't need the MCP SDK loaded
+            build_server(path).run()  # blocks, speaking MCP over stdin and stdout
+    except (settings_mod.SettingsError, index.IndexBuildError) as e:
+        print(f"code-recycle: {e}", file=sys.stderr)
+        return 2
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

code_recycle/embed.py

@@ -0,0 +1,68 @@
+"""Search by meaning: local embeddings through fastembed, stored in the same SQLite index.
+
+Sources: fastembed README, "Quickstart" and "Dense text embeddings"; fastembed's own
+TextEmbedding.query_embed docstring for the query form (research plan Topic 4).
+Vectors live in a plain SQLite table and are compared with numpy, so no vector database is needed
+(Topic 5: about 25,000 pieces of 384 numbers is under 40 MB). Combining with keyword results uses
+Reciprocal Rank Fusion, k = 60 (Cormack et al. 2009, Topic 6).
+
+fastembed is an optional install: pip install "code-recycle[embed]". Without it, search is keyword only.
+"""
+MAX_CHARS = 400  # each piece is cut to this before embedding (results/prereg_embeddings.md, amendment 1)
+EMBEDDED_KINDS = {"function", "method", "class", "module", "research_entry"}  # sections and text windows stay keyword only
+RRF_K = 60  # from the RRF paper; not tuned
+
+
+class EmbeddingsUnavailable(RuntimeError):
+    """Embeddings are switched on in settings but fastembed isn't installed."""
+
+
+def load_model(name):
+    try:
+        from fastembed import TextEmbedding  # imported here, so keyword-only installs never need it
+    except ImportError:
+        raise EmbeddingsUnavailable(
+            'Embeddings are on in settings, but fastembed is not installed. Run: pip install "code-recycle[embed]"'
+        )
+    return TextEmbedding(model_name=name)  # downloads the model on first use, then works offline
+
+
+def piece_text(name, text, taught="", where="", code=""):
+    """What gets embedded for one piece (results/prereg_embeddings.md, amendment 1).
+
+    Code pieces: the name, then the piece's own text. Research entries: the heading, what it
+    taught, where on the page, then the copied code. Cut to MAX_CHARS.
+    """
+    parts = (name, taught, where, code) if (taught or where or code) else (name, text)
+    return "\n".join(p for p in parts if p)[:MAX_CHARS]
+
+
+def embed_pieces(model, texts):
+    """Document vectors, unit length, as one float32 numpy array (one row per text)."""
+    import numpy as np  # installed with fastembed
+    v = np.array(list(model.embed(texts)), dtype=np.float32)
+    return v / np.maximum(np.linalg.norm(v, axis=1, keepdims=True), 1e-12)  # unit length, so a dot product is cosine
+
+
+def embed_query(model, text):
+    import numpy as np
+    v = np.array(list(model.query_embed([text]))[0], dtype=np.float32)  # the model's query form
+    return v / max(float(np.linalg.norm(v)), 1e-12)
+
+
+def nearest(matrix, rowids, qvec, n):
+    """The n rowids whose vectors are closest in meaning to the query, best first."""
+    import numpy as np
+    scores = matrix @ qvec  # cosine similarity, because every vector is unit length
+    n = min(n, len(rowids))
+    top = np.argpartition(-scores, n - 1)[:n]  # the n best, unordered (fast)
+    return [rowids[i] for i in top[np.argsort(-scores[top])]]  # then ordered
+
+
+def fuse(*rankings, k=RRF_K):
+    """Reciprocal Rank Fusion: each item scores the sum of 1 / (k + rank) over the rankings it appears in."""
+    score = {}
+    for ranking in rankings:
+        for rank, item in enumerate(ranking, start=1):
+            score[item] = score.get(item, 0.0) + 1.0 / (k + rank)
+    return sorted(score, key=lambda item: -score[item])

code_recycle/files.py

@@ -0,0 +1,155 @@
+"""Walk the indexed folders, deciding which files go in and recording why the rest don't.
+
+Skip rules use pathspec's GitIgnoreSpec (research plan Topic 9). The order of checks and the
+index report are original applied work.
+"""
+import os  # os.walk, so whole skipped folders are never entered
+from dataclasses import dataclass, field  # plain records for the walk's results
+from concurrent.futures import ProcessPoolExecutor  # spreads the slow secret scan over every CPU core
+from pathlib import Path  # path handling that works on every platform
+from pathspec import GitIgnoreSpec  # Git's own .gitignore behaviour, edge cases included
+
+from . import secret_check as secrets  # the secrets policy (Topic 10); not named secrets.py, which is a standard-library module
+
+ALWAYS_SKIP = [  # never useful to search
+    ".git/", ".hg/", ".svn/", "__pycache__/", "*venv/", "node_modules/", "site-packages/",
+    ".pytest_cache/", ".mypy_cache/", ".ruff_cache/", ".tox/", ".code_recycle/",
+    "code_recycle.toml",  # the settings file itself
+]
+
+
+@dataclass
+class Skipped:
+    path: str  # path relative to its indexed folder
+    reason: str  # plain-words reason shown in the index report
+
+
+@dataclass
+class Walk:
+    included: list = field(default_factory=list)  # (folder, relative path) pairs to index
+    skipped: list = field(default_factory=list)  # Skipped records for the index report
+
+
+def _spec(lines):
+    return GitIgnoreSpec.from_lines(lines)  # compile once, match many times
+
+
+def _is_binary(path):
+    # Original rule: a zero byte in the first 8 KB means it isn't text worth searching
+    with open(path, "rb") as f:
+        return b"\0" in f.read(8192)
+
+
+def walk(settings):
+    """Every file under each folder, sorted into included and skipped, with a reason for each skip.
+
+    Two passes: cheap checks first, then the slow secret scan on what's left, spread over every CPU core.
+    """
+    out = Walk()
+    user_skip = _spec(ALWAYS_SKIP + settings.skip)  # built-in skips plus the user's own
+    allow = _spec(settings.secrets_allow)  # files the user checked and allowed despite a scanner flag
+    include = _spec(settings.include)  # file types worth indexing
+    candidates = []  # (folder, relative path, full path, needs a secret scan), in walk order
+    for folder in settings.folders:
+        gitignore = _spec([])  # empty unless the folder has its own .gitignore
+        gi = folder / ".gitignore"
+        if settings.use_gitignore and gi.is_file():
+            with open(gi, encoding="utf-8", errors="replace") as f:
+                gitignore = _spec(f)  # a file object works as the list of lines
+        for root, dirs, names in os.walk(folder):
+            rel_root = Path(root).relative_to(folder).as_posix()  # posix slashes, so patterns match on Windows too
+            prefix = "" if rel_root == "." else rel_root + "/"
+            # Prune skipped folders here so os.walk never goes inside them. Patterns ending in "/"
+            # only match folders, so each name is tested with a trailing slash.
+            kept = []
+            for d in sorted(dirs):
+                rel = prefix + d + "/"
+                if user_skip.match_file(rel):
+                    out.skipped.append(Skipped(rel, "skip rule"))
+                elif gitignore.match_file(rel):
+                    out.skipped.append(Skipped(rel, ".gitignore"))
+                else:
+                    kept.append(d)
+            dirs[:] = kept  # changing the list in place is how os.walk is told not to descend
+            for name in sorted(names):
+                rel = prefix + name
+                full = Path(root) / name
+                reason = _cheap_check(full, rel, settings, user_skip, gitignore, include)
+                if reason:
+                    out.skipped.append(Skipped(rel, reason))
+                else:
+                    candidates.append((folder, rel, full, not allow.match_file(rel)))
+    to_scan = [full for _, _, full, needs in candidates if needs]
+    found = dict(zip(to_scan, _scan_all(to_scan, settings.secrets_strict)))  # full path -> detector names
+    for folder, rel, full, needs in candidates:
+        if needs and found[full]:
+            out.skipped.append(Skipped(rel, "possible secret: " + ", ".join(found[full])))
+        else:
+            out.included.append((folder, rel))
+    return out
+
+
+def _cheap_check(full, rel, settings, user_skip, gitignore, include):
+    # Every check that doesn't need the secret scanner, cheapest first. Returns a reason or None.
+    if secrets.is_secret_name(rel):
+        return "secret file name (never read)"
+    if user_skip.match_file(rel):
+        return "skip rule"
+    if gitignore.match_file(rel):
+        return ".gitignore"
+    if not include.match_file(rel):
+        return "file type not included"
+    if full.is_symlink():
+        return "symbolic link"  # a link could point outside the folder the user chose
+    size = full.stat().st_size
+    if size > settings.max_file_kb * 1024:
+        return f"over {settings.max_file_kb} KB"
+    if size == 0:
+        return "empty"
+    if _is_binary(full):
+        return "binary"
+    return None
+
+
+PARALLEL_FROM = 50  # below this many files, starting worker processes costs more than it saves
+_worker_ctx = None  # each worker keeps its scanner's with-block open; if this were dropped, the block would close
+_worker_scan = None  # and the scanner would silently run with no detectors (caught by a test)
+
+
+def _init_worker(strict):
+    global _worker_ctx, _worker_scan
+    _worker_ctx = secrets.file_scanner(strict)  # the scanner's settings live inside a with-block
+    _worker_scan = _worker_ctx.__enter__()  # entered once and kept open for the worker's life
+
+
+def _scan_in_worker(path):
+    return _worker_scan(path)
+
+
+def _scan_all(paths, strict):
+    """Secret-scan every path, in order. Uses every CPU core when there are enough files."""
+    if len(paths) < PARALLEL_FROM:
+        with secrets.file_scanner(strict) as scan:
+            return [scan(p) for p in paths]
+    with ProcessPoolExecutor(initializer=_init_worker, initargs=(strict,)) as pool:  # one worker per core
+        return list(pool.map(_scan_in_worker, paths, chunksize=16))  # map keeps the input order
+
+
+def report(result):
+    """A plain-words index report: what went in, what didn't, and why."""
+    n = len(result.included)
+    lines = [f"Indexed {n} file{'' if n == 1 else 's'}. Skipped {len(result.skipped)}."]
+    flagged = [s for s in result.skipped if s.reason.startswith("possible secret")]
+    if flagged:
+        lines.append("")
+        lines.append("Skipped because they may hold a secret. Check each one; if it's safe, add it to [secrets] allow:")
+        lines += [f"  {s.path} ({s.reason})" for s in flagged]
+    counts = {}
+    for s in result.skipped:
+        if not s.reason.startswith("possible secret"):
+            counts[s.reason] = counts.get(s.reason, 0) + 1
+    if counts:
+        lines.append("")
+        lines.append("Other skips:")
+        lines += [f"  {reason}: {n}" for reason, n in sorted(counts.items())]
+    return "\n".join(lines)

code_recycle/index.py

@@ -0,0 +1,101 @@
+"""Build the search index: one SQLite file with a full-text (FTS5) table of chunks.
+
+Sources: SQLite FTS5 page, overview examples, UNINDEXED column option, porter tokenizer
+(research plan Topic 6). Each build writes a new file and swaps it in, so a failed build never
+leaves a half-written index behind.
+"""
+import os  # replacing the old index file in one step
+import re  # splitting camelCase names
+import sqlite3  # Python's built-in SQLite, which includes FTS5 on most builds
+from pathlib import Path  # paths that work on every platform
+
+from . import chunks, embed, files  # splitting files into pieces; meaning search; walking the folders
+
+# Searchable columns first, then stored-only ones. Order matters: bm25 weights follow it (search.py).
+COLUMNS = ["name", "path", "url", "taught", "place", "code", "body"]
+STORED = ["kind", "folder", "start", "end", "label"]  # label: the real name, since "name" holds the split-up form
+SCHEMA = (
+    "CREATE VIRTUAL TABLE chunks USING fts5("
+    + ", ".join(COLUMNS)
+    + ", "
+    + ", ".join(f"{c} UNINDEXED" for c in STORED)  # kept for results, never matched
+    + ", tokenize = 'porter unicode61')"  # stemming on top of the default tokenizer
+)
+
+
+class IndexBuildError(RuntimeError):
+    """The index can't be built or opened. The message says what to do."""
+
+
+def check_fts5():
+    """Fail early with a clear message if this Python's SQLite was built without FTS5."""
+    try:
+        con = sqlite3.connect(":memory:")
+        con.execute("CREATE VIRTUAL TABLE t USING fts5(x)")
+        con.close()
+    except sqlite3.OperationalError:
+        raise IndexBuildError(
+            "This Python's SQLite has no FTS5 full-text search, which code_recycle needs. "
+            "Use the python.org installer, Homebrew or your Linux distribution's Python, which include it."
+        )
+
+
+def split_name(name):
+    # "loadSettings" and "HTTPServer" become "load Settings" and "HTTP Server", so either word matches
+    return re.sub(r"(?<=[a-z0-9])(?=[A-Z])|(?<=[A-Z])(?=[A-Z][a-z])", " ", name)
+
+
+def build(settings, progress=print):
+    """Walk, chunk and index every folder. Returns the walk result for the index report."""
+    check_fts5()
+    walked = files.walk(settings)
+    target = Path(settings.index_path)
+    target.parent.mkdir(parents=True, exist_ok=True)  # e.g. create .code_recycle/
+    tmp = target.with_name(target.name + ".building")  # built aside, then swapped in
+    if tmp.exists():
+        tmp.unlink()  # left over from a build that was interrupted
+    con = sqlite3.connect(tmp)
+    try:
+        con.execute(SCHEMA)
+        con.execute("CREATE TABLE vectors (id INTEGER PRIMARY KEY, v BLOB)")  # meaning vectors, same rowid as chunks
+        model = embed.load_model(settings.embed_model) if settings.embed_model else None
+        to_embed = []  # (rowid, text) for the pieces that get a meaning vector
+        rows = 0
+        for folder, rel in walked.included:
+            try:
+                text = (folder / rel).read_text(encoding="utf-8")
+            except UnicodeDecodeError:
+                text = (folder / rel).read_text(encoding="utf-8", errors="replace")  # still searchable
+            for c in chunks.chunk_file(rel, text):
+                f = c.fields
+                cur = con.execute(
+                    f"INSERT INTO chunks ({', '.join(COLUMNS + STORED)}) VALUES ({', '.join('?' * (len(COLUMNS) + len(STORED)))})",
+                    (split_name(c.name), rel, f.get("url", ""), f.get("taught", ""), f.get("where", ""),
+                     f.get("code", ""), c.text, c.kind, str(folder), c.start, c.end, c.name),
+                )
+                rows += 1
+                if model and c.kind in embed.EMBEDDED_KINDS:
+                    to_embed.append((cur.lastrowid, embed.piece_text(c.name, c.text, f.get("taught", ""),
+                                                                     f.get("where", ""), f.get("code", ""))))
+        if to_embed:
+            progress(f"Embedding {len(to_embed)} pieces with {settings.embed_model}...")
+            vecs = embed.embed_pieces(model, [t for _, t in to_embed])
+            con.executemany("INSERT INTO vectors (id, v) VALUES (?, ?)",
+                            [(rid, v.tobytes()) for (rid, _), v in zip(to_embed, vecs)])
+        con.execute("CREATE TABLE meta (key TEXT PRIMARY KEY, value TEXT)")
+        con.execute("INSERT INTO meta VALUES ('embed_model', ?)", (settings.embed_model,))  # search must use the same model
+        con.commit()
+    finally:
+        con.close()
+    os.replace(tmp, target)  # one step, so readers see the old index or the new one, never half of one
+    progress(f"Indexed {rows} pieces from {len(walked.included)} files into {target}")
+    return walked
+
+
+def connect(settings):
+    """Open an existing index read-only."""
+    path = Path(settings.index_path)
+    if not path.exists():
+        raise IndexBuildError(f"No index at {path}. Run 'code-recycle index' first.")
+    # as_uri() percent-encodes spaces, "#" and "?", and writes Windows drive letters correctly
+    return sqlite3.connect(path.resolve().as_uri() + "?mode=ro", uri=True)  # read-only: searching never changes it

code_recycle/search.py

@@ -0,0 +1,176 @@
+"""Search the index, apply the reuse rules, and scan every result for secrets before returning it.
+
+Ranking: SQLite FTS5 bm25() with a weight per column (research plan Topic 6). The weights are a
+starting guess (original applied work), to be tuned only on the tuning half of the test set.
+Reuse rules (no_suggest_from, no_suggest_into) are original applied work (Topic 9).
+"""
+import re  # turning free text into a safe FTS5 query
+from dataclasses import dataclass, field  # plain records for results
+from pathlib import Path  # finding which indexed folder a path belongs to
+from pathspec import GitIgnoreSpec  # the same pattern language as the skip rules
+
+from . import embed, index, secret_check
+
+# One weight per searchable column, in index.COLUMNS order: name, path, url, taught, place, code, body.
+WEIGHTS = (5.0, 2.0, 3.0, 4.0, 1.0, 2.0, 1.0)
+SNIPPET_LINES = 40  # lines of code shown per result; the full text is in the file at the given lines
+POOL = 200  # candidates taken from each ranking before rules and de-duplication
+STOPWORDS = {"a", "an", "and", "the", "to", "of", "in", "for", "on", "with", "how", "do", "i", "is", "it", "my", "what"}
+
+
+@dataclass
+class Result:
+    path: str  # file path relative to its folder
+    folder: str  # the indexed folder it came from
+    start: int  # first line
+    end: int  # last line
+    kind: str  # function, class, research_entry, section, ...
+    name: str  # function or class name, or heading
+    url: str = ""  # research entries: where the source lives online
+    where: str = ""  # research entries: where on that page
+    taught: str = ""  # research entries: what it taught
+    snippet: str = ""  # the code pattern (research entries) or the chunk's text
+    withheld: list = field(default_factory=list)  # secret-check hits; the snippet is withheld if any
+
+
+@dataclass
+class Answer:
+    results: list  # Result records, best first
+    note: str = ""  # anything the user should know, such as a rule that hid matches
+
+
+def to_query(text):
+    """Free text to an FTS5 query: every word, quoted so punctuation can't break the syntax, joined by OR."""
+    words = [w for w in re.findall(r"[A-Za-z0-9]+", index.split_name(text).lower()) if w not in STOPWORDS]
+    return " OR ".join(f'"{w}"' for w in dict.fromkeys(words))  # dict keeps order and drops repeats
+
+
+def search(settings, text, from_path=None, limit=5):
+    """Best matches for `text`. `from_path` is where the search is made from, so reuse rules can apply."""
+    q = to_query(text)
+    if not q:
+        return Answer([], "The search had no usable words.")
+    con = index.connect(settings)
+    try:
+        keyword = con.execute(
+            f"SELECT rowid FROM chunks WHERE chunks MATCH ? ORDER BY bm25(chunks, {', '.join(map(str, WEIGHTS))}) LIMIT ?",
+            (q, POOL),
+        ).fetchall()
+        order = [r for (r,) in keyword]
+        if settings.embed_model:
+            order = embed.fuse(order, _meaning(con, settings, text))  # keyword and meaning, combined by RRF
+        order = order[:POOL]
+        found = {}
+        for rid in order:
+            found[rid] = con.execute(
+                "SELECT label, path, url, taught, place, code, body, kind, folder, start, end FROM chunks WHERE rowid = ?",
+                (rid,)).fetchone()
+        rows = [found[r] for r in order]
+    finally:
+        con.close()
+
+    blocked_from = GitIgnoreSpec.from_lines(settings.no_suggest_from)
+    into_spec = GitIgnoreSpec.from_lines(settings.no_suggest_into)
+    protected = _protected_area(settings, from_path, into_spec)
+
+    out, hidden, seen = [], 0, set()
+    for name, path, url, taught, place, code, body, kind, folder, start, end in rows:
+        # Rules first, then de-duplication: a blocked copy must never hide an allowed one
+        if blocked_from.match_file(path):
+            continue  # never suggested from here, and not counted: the user asked for silence
+        if protected is not None and not (folder == protected[0] and path.startswith(protected[1])):
+            hidden += 1  # a match outside the protected area: say it exists, but don't show it
+            continue
+        if body in seen:
+            continue  # an identical copy (a backup or a vendored file) adds nothing; show the best-ranked one
+        seen.add(body)
+        r = Result(path, folder, int(start), int(end), kind, name, url, place, taught, code or _first_lines(body))
+        r.withheld = sorted({hit for t in (r.snippet, r.url, r.where, r.taught, r.name) for hit in secret_check.scan_text(t)})
+        if r.withheld:
+            r.snippet = r.url = r.where = r.taught = ""  # fail closed: show only where it is, not what it says
+        out.append(r)
+        if len(out) == limit:
+            break
+    note = ""
+    if hidden:
+        note = (f"{hidden} matches exist outside this folder, but the no_suggest_into rule says not to "
+                f"suggest them here.")
+    return Answer(out, note)
+
+
+_vector_cache = {}  # index file -> (modified time, rowids, matrix, model), so vectors load once per server
+
+
+def _meaning(con, settings, text):
+    """Rowids ranked by closeness in meaning to the query."""
+    import numpy as np
+    try:
+        stored = con.execute("SELECT value FROM meta WHERE key = 'embed_model'").fetchone()
+    except __import__("sqlite3").OperationalError:
+        stored = None  # an index from before embeddings existed
+    if not stored or stored[0] != settings.embed_model:
+        raise index.IndexBuildError(
+            f"The index was built with embeddings model {stored[0] if stored else 'none'!r}, but settings say "
+            f"{settings.embed_model!r}. Run 'code-recycle index' again.")
+    key = str(settings.index_path)
+    mtime = Path(key).stat().st_mtime
+    cached = _vector_cache.get(key)
+    if not cached or cached[0] != mtime:
+        rows = con.execute("SELECT id, v FROM vectors").fetchall()
+        ids = [r for r, _ in rows]
+        matrix = np.frombuffer(b"".join(v for _, v in rows), dtype=np.float32).reshape(len(rows), -1) if rows else None
+        cached = (mtime, ids, matrix, embed.load_model(settings.embed_model))
+        _vector_cache[key] = cached
+    _, ids, matrix, model = cached
+    if matrix is None:
+        return []
+    return embed.nearest(matrix, ids, embed.embed_query(model, text), POOL)
+
+
+def _protected_area(settings, from_path, into_spec):
+    """If the search comes from inside a no_suggest_into folder: (indexed folder, that folder's path
+    with a trailing slash). Otherwise None. Only results inside that same area may be shown."""
+    if not from_path:
+        return None
+    p = Path(from_path).expanduser().resolve()
+    for folder in settings.folders:
+        try:
+            rel = p.relative_to(folder).as_posix()
+        except ValueError:
+            continue  # not inside this indexed folder
+        parts = [] if rel == "." else rel.split("/")
+        for i in range(1, len(parts) + 1):  # the shortest enclosing folder the rule names
+            area = "/".join(parts[:i]) + "/"
+            if into_spec.match_file(area):
+                return str(folder), area
+    return None
+
+
+def _first_lines(text):
+    lines = text.splitlines()
+    more = len(lines) - SNIPPET_LINES
+    return "\n".join(lines[:SNIPPET_LINES]) + (f"\n... ({more} more lines)" if more > 0 else "")
+
+
+def format_answer(answer):
+    """Plain text an AI client or a person can read: where each result is, and what it says."""
+    if not answer.results:
+        return answer.note or "No matches in your own work. Searching the web is fine."
+    parts = []
+    for i, r in enumerate(answer.results, start=1):
+        head = f"{i}. {r.name or r.path} ({r.kind}) {Path(r.folder) / r.path}:{r.start}-{r.end}"
+        lines = [head]
+        if r.url:
+            lines.append(f"   Source: {r.url}")
+        if r.where:
+            lines.append(f"   Where on the page: {r.where}")
+        if r.taught:
+            lines.append(f"   What it taught: {r.taught}")
+        if r.withheld:
+            lines.append(f"   Content withheld: it may contain a secret ({', '.join(r.withheld)}). Open the file to check.")
+        elif r.snippet:
+            lines.append("   ```\n" + "\n".join("   " + l for l in r.snippet.splitlines()) + "\n   ```")
+        parts.append("\n".join(lines))
+    if answer.note:
+        parts.append(answer.note)
+    return "\n\n".join(parts)

code_recycle/secret_check.py

@@ -0,0 +1,99 @@
+"""Keep secrets out of the index and out of every result.
+
+The policy is original applied work (research plan Topic 10, after the GPT check). It promises
+only what the tests prove:
+1. Files with a known secret name are never opened.
+2. Every other file is scanned with detect-secrets before it is read in. A flagged file is
+   skipped and listed in the index report, unless the user allows it by name.
+3. Every field returned to a client is scanned again with a small built-in pattern list.
+
+Source for step 2: detect-secrets README, "Usage in Other Python Scripts", "Basic Use" and
+"More Advanced Configuration" (choosing detectors with plugins_used).
+By default only named-credential detectors run. The two "high entropy" detectors flag any
+random-looking text (hashes, logs, notebook images), so they only run with strict = true.
+"""
+import os  # removing the canary file
+import re  # the built-in patterns for step 3
+import tempfile  # a throwaway file for the canary check
+from contextlib import contextmanager  # lets one scanner set-up serve a whole walk
+from pathspec import GitIgnoreSpec  # gitignore-style matching, research plan Topic 9
+from detect_secrets import SecretsCollection  # holds whatever one scan finds
+from detect_secrets.settings import transient_settings  # run the scanner with a chosen set of detectors
+
+# Step 1: names that hold secrets by convention. Never opened, and allow-listing can't override this.
+SECRET_NAMES = GitIgnoreSpec.from_lines([
+    ".env", ".env.*", "*.pem", "*.key", "*.p12", "*.pfx", "id_rsa", "id_dsa", "id_ecdsa", "id_ed25519",
+    ".netrc", ".pypirc", ".npmrc", "credentials.json", "*.keystore", "*.jks",
+])
+
+# Step 3: patterns for the commonest live credentials. Short on purpose; the file scan is the main
+# defence and this is the second line. Each entry: (name shown to the user, compiled pattern).
+TEXT_PATTERNS = [
+    ("AWS access key", re.compile(r"\b(?:AKIA|ASIA)[0-9A-Z]{16}\b")),
+    ("GitHub token", re.compile(r"\bgh[pousr]_[A-Za-z0-9]{36,}\b")),
+    ("GitHub fine-grained token", re.compile(r"\bgithub_pat_[A-Za-z0-9_]{22,}\b")),
+    ("Slack token", re.compile(r"\bxox[abprs]-[A-Za-z0-9-]{10,}\b")),
+    ("OpenAI-style key", re.compile(r"\bsk-[A-Za-z0-9_-]{20,}\b")),
+    ("Private key block", re.compile(r"-----BEGIN [A-Z ]*PRIVATE KEY-----")),
+    ("Telegram bot token", re.compile(r"\b\d{8,10}:[A-Za-z0-9_-]{35}\b")),
+]
+
+# Step 2 detectors, named as in detect-secrets 1.5.0 (pinned in pyproject.toml).
+NAMED_DETECTORS = [
+    "ArtifactoryDetector", "AWSKeyDetector", "AzureStorageKeyDetector", "BasicAuthDetector",
+    "CloudantDetector", "DiscordBotTokenDetector", "GitHubTokenDetector", "GitLabTokenDetector",
+    "IbmCloudIamDetector", "IbmCosHmacDetector", "JwtTokenDetector", "KeywordDetector",
+    "MailchimpDetector", "NpmDetector", "OpenAIDetector", "PrivateKeyDetector", "PypiTokenDetector",
+    "SendGridDetector", "SlackDetector", "SoftlayerDetector", "SquareOAuthDetector", "StripeDetector",
+    "TelegramBotTokenDetector", "TwilioKeyDetector",
+]
+ENTROPY_DETECTORS = ["Base64HighEntropyString", "HexHighEntropyString"]  # only with strict = true
+
+
+def is_secret_name(relpath):
+    """True if this path's name marks it as a secrets file (step 1)."""
+    return SECRET_NAMES.match_file(relpath)  # matches the name at any depth, as .gitignore would
+
+
+@contextmanager
+def file_scanner(strict=False):
+    """Set the scanner up once and yield a function that scans one file (step 2).
+
+    The function returns the names of detectors that fired; an empty list means nothing was found.
+    """
+    names = NAMED_DETECTORS + (ENTROPY_DETECTORS if strict else [])
+    with transient_settings({"plugins_used": [{"name": n} for n in names]}):  # only these detectors run
+        def scan(path):
+            found = SecretsCollection()  # a fresh collection per file, so results never mix
+            found.scan_file(str(path))  # scans the file's lines; it never changes the file
+            return sorted({hit["type"] for hits in found.json().values() for hit in hits})  # unique names
+        _canary(scan)  # fail closed: refuse to run if the scanner can't see a planted fake key
+        yield scan
+
+
+class ScannerBroken(RuntimeError):
+    """The secret scanner missed a planted fake key, so nothing is indexed rather than risk a leak."""
+
+
+def _canary(scan):
+    # AWS's documented example key, built from pieces so this file holds no key-shaped text
+    fake = "AKIA" + "IOSFODNN7" + "EXAMPLE"
+    fd, path = tempfile.mkstemp(suffix=".py")
+    try:
+        with os.fdopen(fd, "w") as f:
+            f.write(f'key = "{fake}"\n')
+        if "AWS Access Key" not in scan(path):
+            raise ScannerBroken("The secret scanner failed its self-check. Nothing was indexed.")
+    finally:
+        os.remove(path)
+
+
+def scan_file(path, strict=False):
+    """Scan a single file. For many files, use file_scanner so set-up happens once."""
+    with file_scanner(strict) as scan:
+        return scan(path)
+
+
+def scan_text(text):
+    """Pattern names that match this text (step 3). An empty list means it's safe to return."""
+    return [name for name, pattern in TEXT_PATTERNS if pattern.search(text)]

code_recycle/server.py

@@ -0,0 +1,42 @@
+"""The MCP server: one read-only search tool that any MCP client can call.
+
+Source: MCP Python SDK v2 (pinned to 2.3.0), "Connect to a real host" example and README
+(research plan Topic 1). mcp.run() with no arguments serves over stdio, which every host speaks.
+"""
+import os  # the folder the host started the server in
+
+from mcp.server import MCPServer  # the SDK's server class
+from mcp.server.mcpserver.exceptions import ToolError  # a clean error the client can show
+
+from . import index, search, settings as settings_mod
+
+
+def build_server(settings_path):
+    """An MCP server bound to one settings file. Settings are read now, so mistakes show at start-up."""
+    s = settings_mod.load(settings_path)
+    mcp = MCPServer("code_recycle")
+
+    @mcp.tool()
+    def search_own_work(query: str, from_path: str = "", limit: int = 5) -> str:
+        """Search the user's own code and verified research sources before searching the web.
+
+        Use this first when about to write code, or to look up a library or API: the user may have
+        solved it before. Results give the file and line numbers, and for research entries the
+        source URL, where on the page, what it taught and the verified code copied from it.
+
+        query: what you need, in plain words, for example "retry an HTTP request with backoff".
+        from_path: the file or folder you are working in, so the user's reuse rules can apply.
+            If left out, the folder the server was started in is used.
+        limit: how many results to return, 1 to 20.
+        """
+        if not 1 <= limit <= 20:
+            raise ToolError("limit must be between 1 and 20")
+        try:
+            # Hosts start the server in the project's folder, so that's the safe default when the
+            # caller doesn't say: a rule like no_suggest_into still applies if from_path is left out
+            answer = search.search(s, query, from_path=from_path or os.getcwd(), limit=limit)
+        except index.IndexBuildError as e:
+            raise ToolError(str(e))
+        return search.format_answer(answer)
+
+    return mcp

code_recycle/settings.py

@@ -0,0 +1,151 @@
+"""Read and check the settings file.
+
+Source: Python docs, tomllib, "Examples" (research plan Topic 8).
+The layout of the file is original applied work (project plan, "Original design").
+"""
+import tomllib  # standard-library TOML reader, built in from Python 3.11
+from dataclasses import dataclass, field  # a plain record type for the checked settings
+from pathlib import Path  # path handling that works the same on Linux, macOS and Windows
+
+# Every key the settings file may use, per section. Anything else is a typo and is refused,
+# because a misspelt rule (say "no_sugest_into") would otherwise be silently ignored.
+ALLOWED = {
+    "index": {"folders", "include", "skip", "use_gitignore", "max_file_kb", "index_path"},
+    "rules": {"no_suggest_from", "no_suggest_into"},
+    "secrets": {"allow", "strict"},
+    "embeddings": {"model"},
+}
+
+# Code and text file types indexed when the settings file doesn't say. Data files (JSON, CSV),
+# notebooks and logs are left out on purpose: they are big, noisy and rarely worth reusing.
+DEFAULT_INCLUDE = [
+    "*.py", "*.pyi", "*.md", "*.rst", "*.txt", "*.toml", "*.yaml", "*.yml", "*.cfg", "*.ini",
+    "*.sh", "*.bash", "*.ps1", "*.sql", "*.js", "*.mjs", "*.ts", "*.tsx", "*.jsx", "*.java",
+    "*.kt", "*.go", "*.rs", "*.c", "*.h", "*.cpp", "*.hpp", "*.cs", "*.rb", "*.php", "*.r", "*.R",
+    "*.swift", "*.scala", "*.lua", "*.html", "*.css", "Dockerfile", "Makefile", "*.mk",
+]
+
+EXAMPLE = """\
+# code_recycle settings. Paths may start with ~ or be relative to this file.
+
+[index]
+folders = ["~/projects"]          # folders to search; each one is indexed separately
+# include = ["*.py", "*.md"]      # file types to index; leave out to use the built-in list of code and text types
+skip = ["node_modules/", "*.min.js", "data/"]  # gitignore-style patterns, relative to each folder
+use_gitignore = true              # also honour the .gitignore at the top of each folder
+max_file_kb = 1024                # bigger files are skipped and listed in the index report
+index_path = ".code_recycle/index.db"  # where the index is stored
+
+[rules]
+# Patterns are relative to each indexed folder, in gitignore style.
+no_suggest_from = []              # never return results that live here
+no_suggest_into = []              # searches made from here only say "matches exist elsewhere"
+
+[secrets]
+allow = []                        # files the secret scanner flagged that you have checked and allow
+strict = false                    # true also flags random-looking text: safer, but many false alarms
+
+[embeddings]
+# Search by meaning as well as by keyword. Needs: pip install "code-recycle[embed]"
+# The model (about 67 MB) downloads once, then everything runs offline.
+model = ""                        # "" for keyword only, or "BAAI/bge-small-en-v1.5"
+"""
+
+
+class SettingsError(ValueError):
+    """The settings file is missing, unreadable or has a mistake. The message says what to fix."""
+
+
+@dataclass
+class Settings:
+    folders: list[Path]  # absolute folders to index
+    skip: list[str] = field(default_factory=list)  # extra gitignore-style skip patterns
+    use_gitignore: bool = True  # honour each folder's top-level .gitignore
+    max_file_kb: int = 1024  # size limit per file, in kilobytes
+    index_path: Path = Path(".code_recycle/index.db")  # absolute path of the index file
+    no_suggest_from: list[str] = field(default_factory=list)  # results never come from here
+    no_suggest_into: list[str] = field(default_factory=list)  # queries from here get no suggestions
+    secrets_allow: list[str] = field(default_factory=list)  # scanner-flagged files the user allows
+    secrets_strict: bool = False  # also run the high-entropy detectors
+    include: list[str] = field(default_factory=lambda: list(DEFAULT_INCLUDE))  # file types to index
+    embed_model: str = ""  # "" means keyword search only
+
+
+def _str_list(value, name):
+    # A list of strings is the only shape a pattern or folder list may take
+    if not isinstance(value, list) or not all(isinstance(v, str) for v in value):
+        raise SettingsError(f"'{name}' must be a list of strings, for example [\"src/\"]")
+    return value
+
+
+def _resolve(raw, base):
+    # Expand ~ and make a relative path relative to the settings file, not the working directory,
+    # because an MCP host starts the tool from its own working directory
+    p = Path(raw).expanduser()
+    return p if p.is_absolute() else (base / p).resolve()
+
+
+def load(path):
+    """Read a settings file and return checked Settings, or raise SettingsError saying what is wrong."""
+    path = Path(path).expanduser().resolve()  # absolute, so error messages name the real file
+    try:
+        with open(path, "rb") as f:  # tomllib needs the file opened in binary mode
+            data = tomllib.load(f)  # parse the TOML into plain dicts and lists
+    except FileNotFoundError:
+        raise SettingsError(f"No settings file at {path}. Run 'code-recycle init' to write an example.")
+    except tomllib.TOMLDecodeError as e:
+        raise SettingsError(f"{path} is not valid TOML: {e}")
+
+    for section, keys in data.items():  # refuse unknown sections and keys, so typos are caught
+        if section not in ALLOWED:
+            raise SettingsError(f"Unknown section [{section}] in {path}. Allowed: {', '.join(ALLOWED)}")
+        if not isinstance(keys, dict):
+            raise SettingsError(f"[{section}] in {path} must be a table of settings")
+        for key in keys:
+            if key not in ALLOWED[section]:
+                raise SettingsError(f"Unknown setting '{key}' in [{section}]. Allowed: {', '.join(sorted(ALLOWED[section]))}")
+
+    index = data.get("index", {})  # the only section that must exist, because folders are required
+    rules = data.get("rules", {})
+    secrets = data.get("secrets", {})
+    base = path.parent  # relative paths are read from the settings file's own folder
+
+    folders = [_resolve(f, base) for f in _str_list(index.get("folders", []), "folders")]
+    if not folders:
+        raise SettingsError("[index] folders is empty. List at least one folder to search.")
+    missing = [str(f) for f in folders if not f.is_dir()]  # a folder that doesn't exist is a mistake
+    if missing:
+        raise SettingsError(f"These folders don't exist: {', '.join(missing)}")
+
+    use_gitignore = index.get("use_gitignore", True)
+    if not isinstance(use_gitignore, bool):  # TOML true/false only, not "yes"
+        raise SettingsError("'use_gitignore' must be true or false")
+    max_kb = index.get("max_file_kb", 1024)
+    if not isinstance(max_kb, int) or isinstance(max_kb, bool) or max_kb <= 0:  # bool is an int in Python
+        raise SettingsError("'max_file_kb' must be a whole number above 0")
+    strict = secrets.get("strict", False)
+    if not isinstance(strict, bool):
+        raise SettingsError("'strict' must be true or false")
+    include = _str_list(index.get("include", DEFAULT_INCLUDE), "include")
+    if not include:
+        raise SettingsError("[index] include is empty. Leave it out to use the built-in list.")
+    embed_model = data.get("embeddings", {}).get("model", "")
+    if not isinstance(embed_model, str):
+        raise SettingsError("[embeddings] model must be a model name in quotes, or \"\" for keyword only")
+    index_path = index.get("index_path", ".code_recycle/index.db")
+    if not isinstance(index_path, str):
+        raise SettingsError("'index_path' must be a path in quotes")
+
+    return Settings(
+        folders=folders,
+        skip=_str_list(index.get("skip", []), "skip"),
+        use_gitignore=use_gitignore,
+        max_file_kb=max_kb,
+        index_path=_resolve(index_path, base),
+        no_suggest_from=_str_list(rules.get("no_suggest_from", []), "no_suggest_from"),
+        no_suggest_into=_str_list(rules.get("no_suggest_into", []), "no_suggest_into"),
+        secrets_allow=_str_list(secrets.get("allow", []), "allow"),
+        secrets_strict=strict,
+        include=list(include),
+        embed_model=embed_model,
+    )

code_recycle-0.1.0.dist-info/METADATA

@@ -0,0 +1,176 @@
+Metadata-Version: 2.5
+Name: code-recycle
+Version: 0.1.0
+Summary: Search your own code and verified sources before searching the web. Local, private, and usable from any MCP client.
+License-Expression: MIT
+License-File: LICENSE
+Requires-Python: >=3.11
+Requires-Dist: detect-secrets==1.5.0
+Requires-Dist: mcp==2.3.0
+Requires-Dist: pathspec==1.1.1
+Provides-Extra: dev
+Requires-Dist: anyio; extra == 'dev'
+Requires-Dist: numpy; extra == 'dev'
+Requires-Dist: pyarrow==22.0.0; extra == 'dev'
+Requires-Dist: pytest==9.1.1; extra == 'dev'
+Provides-Extra: embed
+Requires-Dist: fastembed==0.9.1; extra == 'embed'
+Description-Content-Type: text/markdown
+
+# code_recycle
+
+Search your own code and your own verified sources before you search the web.
+
+Every new project I start, I end up looking for the same things again: how to retry a request,
+how to read a settings file, which page had that bootstrap example. Most of the time I've already
+solved it in another project, and I've already checked the source. I just can't find it.
+
+Companies have the same problem, with a twist. Pasting internal code or questions into outside
+tools is a risk, so the search needs to stay on the machine.
+
+code_recycle indexes your folders and answers "have I already solved this?" with the file and line
+numbers. If you keep research notes with a source URL and the code you copied from it, it finds
+those too, with the link and what you learned. It runs locally, needs no account and no API key,
+and plugs into Claude Code, Codex, Cursor or any MCP client.
+
+## Try it in 5 minutes
+
+Needs Python 3.11 or newer.
+
+```bash
+git clone https://github.com/Z33MScEssex/code_recycle
+cd code_recycle
+pip install .
+cd examples/demo
+code-recycle index
+code-recycle search "read a settings file"
+```
+
+The first result is a research note: the Python docs page for `tomllib`, where on the page, what it
+taught, and the code copied from it. Try `code-recycle search "try a request again with a wait"`
+and you get the `retry` function from `tools/http_utils.py`, lines 14 to 24.
+
+## Use it on your own folders
+
+```bash
+code-recycle init          # writes code_recycle.toml here; edit the folders list
+code-recycle index         # builds the index and prints what was skipped and why
+code-recycle search "parse an iso date"
+```
+
+Re-run `code-recycle index` after big changes. It reads everything again; on my laptop that's about
+40 seconds for 1,500 files.
+
+## Connect it to an AI coding assistant
+
+The tool is called `search_own_work`. It's read-only. Hosts start it with a near-empty environment,
+so give the settings file's full path.
+
+Claude Code:
+
+```bash
+claude mcp add code-recycle -e CODE_RECYCLE_SETTINGS=/full/path/to/code_recycle.toml -- code-recycle serve
+```
+
+Codex:
+
+```bash
+codex mcp add code-recycle --env CODE_RECYCLE_SETTINGS=/full/path/to/code_recycle.toml -- code-recycle serve
+```
+
+If the host can't find `code-recycle`, use the full path from `which code-recycle` (or
+`where code-recycle` on Windows).
+
+## Research notes it understands
+
+A Markdown section becomes a research entry when it has a URL field. These labels are read, bold
+or plain:
+
+```markdown
+## Topic 1: Read a TOML settings file
+
+- **URL:** https://docs.python.org/3/library/tomllib.html
+- **Where on the page:** "Examples", the first example.
+- **What it taught:** the file must be opened in binary mode.
+- **Code pattern from source:**
+    (a fenced code block)
+```
+
+Everything else is still indexed: Python by function, method and class with exact line numbers,
+other Markdown by heading, and other text files in overlapping windows of 60 lines.
+
+## How well it works
+
+Measured, not assumed. The test design was written down before the first run, and nothing was
+tuned on the test sets. Full write-up: [results/accuracy_2026-11-05.md](results/accuracy_2026-11-05.md).
+
+| Test | What it asks | Result |
+|---|---|---|
+| CodeSearchNet, Python | Find a function from its docstring among 999 others (Husain et al. 2019) | MRR 0.797, right function in the top 5 87% of the time |
+| My own research notes | Find another note that used the same source, from a topic heading | MRR 0.308, top 5 39% of the time, 51 cases |
+
+The second test is the real job, and it's much harder. The same source often gets described in
+different words in different projects, and keyword search can't bridge that.
+
+I tried meaning-based search with a small local embedding model. It lifted CodeSearchNet to 0.907,
+but it didn't help on my own notes (0.250 against 0.280 at the time), and the rule set beforehand
+said it had to improve both. So v0.1 ships keyword search. The embeddings code is in the repo,
+switched off and marked experimental: [results/prereg_embeddings.md](results/prereg_embeddings.md).
+
+## Privacy and secrets
+
+- Nothing leaves your machine. There are no network calls, unless you switch on the experimental
+  embeddings, which download a model once.
+- Files named like secrets (`.env`, `*.pem`, `id_rsa` and others) are never opened.
+- Every other file is scanned with [detect-secrets](https://github.com/Yelp/detect-secrets) before
+  it's indexed. A flagged file is skipped and listed in the index report. If you've checked it's
+  safe, add it to `[secrets] allow`.
+- Every result is scanned again before it's returned, for common live key formats (AWS, GitHub,
+  Slack, OpenAI-style, Telegram bot tokens, private key blocks). If one matches, the content is
+  withheld and only the location is shown.
+- If the scanner fails its own self-check (it must spot a planted fake key), nothing is indexed.
+
+Each of these behaviours has a test with fake keys. That isn't a guarantee: a pattern scanner can't
+catch every possible secret, so don't point the tool at folders you know hold live credentials.
+
+## Reuse rules
+
+Some folders shouldn't borrow from others. I have one project where every requirement must be
+researched fresh, so code from other projects must never be suggested into it.
+
+```toml
+[rules]
+no_suggest_from = ["private/"]    # never return results from here
+no_suggest_into = ["clean-room/"] # searches made from here only see this folder's own code,
+                                  # plus a note that other matches exist
+```
+
+The MCP tool takes a `from_path`, so the assistant can say where it's working. If it doesn't, the
+folder the server was started in is used. The rules are a guard against accidents, not a security
+boundary: they trust the path they're given.
+
+## Settings
+
+`code-recycle init` writes a commented example. The settings file is found in this order:
+`--settings`, then `$CODE_RECYCLE_SETTINGS`, then `./code_recycle.toml`, then
+`~/.config/code_recycle/code_recycle.toml`. Typos are refused with a clear message, because a
+misspelt rule silently ignored would be worse.
+
+## Limits
+
+- Python is the only language split by function. Others are searched in windows of lines.
+- The index is rebuilt in full each time. Fine for a few thousand files; not built for huge monorepos.
+- Search is keyword-based (SQLite FTS5 with stemming), so it finds shared words, not shared ideas.
+- So far only I have used it. It has had no outside users yet.
+
+## Development
+
+```bash
+pip install -e ".[dev]"
+pytest
+```
+
+87 tests. The CI workflow runs them on Linux, macOS and Windows with Python 3.11 to 3.13 on every push. How sources were
+checked and how accuracy was measured: [METHOD.md](METHOD.md). Benchmarks: [bench/README.md](bench/README.md).
+
+MIT licence.

code_recycle-0.1.0.dist-info/RECORD

@@ -0,0 +1,15 @@
+code_recycle/__init__.py,sha256=DIXyXkoxhDCDmwRFVwoY9ea-D1EH7KECZ5YKUkSjFP0,147
+code_recycle/chunks.py,sha256=GjVb_x3DBqsr7TEutfJIKkr9qd1kDHXpr49AXfoK9zY,7637
+code_recycle/cli.py,sha256=ehTgWCpHuG4LBSS768DyIRjb37Nb8oVnvuYWKmVt1HA,3126
+code_recycle/embed.py,sha256=4jFXy1ZJ-ALwGjJWOjddDNrN2GoVXM17KOUVwHFwx4s,3242
+code_recycle/files.py,sha256=ygdzX87d-4E7aC2OQLXf3EncA6PbpK7YI3zEAbF_qcs,7073
+code_recycle/index.py,sha256=SMYuVQtd_7a7DNRaTNauMgPSmd9cGbNPuyMyAbvwQ1w,5163
+code_recycle/search.py,sha256=FWB_De2NikfuwcCf3DwPIYm1eKshCJJWGi2M2mDk6ts,8353
+code_recycle/secret_check.py,sha256=lJXAKK-AH4Kse7wSsaqJGka51WKzs-LIP195r72I324,5171
+code_recycle/server.py,sha256=0y9k8uelWP5hSSXXu8u3y04nnnq0-A0qJXvs_XFco4I,2090
+code_recycle/settings.py,sha256=5wZ32AEevGXVu8_DglAcGHMq0yKBMfwxUy0-okkZKHU,7993
+code_recycle-0.1.0.dist-info/METADATA,sha256=f4-ggLDRMwMF9ww5fhQf-DM2Y00w6tTbUJqk_uY8zZI,7420
+code_recycle-0.1.0.dist-info/WHEEL,sha256=2iZrJkDIkeDh0k9SikKMqFuRP-ua6mb4Ir1AsER4BEw,87
+code_recycle-0.1.0.dist-info/entry_points.txt,sha256=xvVUqifZ8QOCIPK9f7uj61LL9GP8pDcGnZ7f7wvPYDQ,55
+code_recycle-0.1.0.dist-info/licenses/LICENSE,sha256=4Y_gs5zJAlvyws-IC3gat3Q6aLzHaxruEtfNY-ogpQc,1068
+code_recycle-0.1.0.dist-info/RECORD,,

code_recycle-0.1.0.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: hatchling 1.33.0
+Root-Is-Purelib: true
+Tag: py3-none-any

code_recycle-0.1.0.dist-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+code-recycle = code_recycle.cli:main

code_recycle-0.1.0.dist-info/licenses/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Zee Mehmood
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.