hugpy-tools 0.2.0a0__py3-none-any.whl

hugpy_tools/__init__.py

@@ -0,0 +1,84 @@
+"""hugpy-tools: a slim, stdlib-only capability suite for autonomous agents.
+
+Extracted from the sprawl of ``abstract_utilities`` / ``abstract_webtools`` and
+rebuilt with no third-party runtime dependency, so a machine with only Python
+can use it. It imports NO other ``hugpy_*`` package (foundation layer).
+
+Modules:
+
+    paths     path confinement (jail) + path-part helpers + filename sanitising
+    fs        safe file/dir ops: encoding-detecting read, line-range read,
+              atomic write, exact-string edit, list/tree, rich file info
+    hashkit   sha256 (text/file, streamed) + a cheap size+head quick hash
+    text      approximate token counting, token/line chunking, unified diffs
+    data      JSON/TOML/YAML read + atomic JSON write (safe dumping)
+    timekit   UTC ISO / epoch conversions
+    web       assess_webpage / prescreen_webpage — assessManager-parity webpage
+              assessment over urllib + html.parser (opt-in JS render)
+    search    reserved namespace (owned elsewhere; not implemented here)
+
+Every ``fs`` function takes an already-confined absolute path — call
+``paths.confine(root, path)`` first to enforce a jail. The web fetch never
+follows a redirect off http(s) and disables ``file://`` / ``ftp://``.
+"""
+from __future__ import annotations
+
+from hugpy_tools.data import read_data, safe_json_dumps, write_json
+from hugpy_tools.fs import (
+    atomic_write,
+    detect_encoding,
+    edit_replace,
+    file_info,
+    list_dir,
+    read_lines,
+    read_text,
+    tree,
+)
+from hugpy_tools.hashkit import quick_hash, sha256_file, sha256_text
+from hugpy_tools.paths import PathEscape, confine, file_parts, sanitize_filename
+from hugpy_tools.text import (
+    chunk_by_lines,
+    chunk_by_tokens,
+    count_tokens,
+    unified_diff,
+)
+from hugpy_tools.timekit import epoch_to_iso, iso_to_epoch, now_epoch, now_iso
+from hugpy_tools.web import assess_webpage, prescreen_webpage
+
+__all__ = [
+    # paths
+    "PathEscape",
+    "confine",
+    "file_parts",
+    "sanitize_filename",
+    # fs
+    "atomic_write",
+    "detect_encoding",
+    "edit_replace",
+    "file_info",
+    "list_dir",
+    "read_lines",
+    "read_text",
+    "tree",
+    # hashkit
+    "quick_hash",
+    "sha256_file",
+    "sha256_text",
+    # text
+    "chunk_by_lines",
+    "chunk_by_tokens",
+    "count_tokens",
+    "unified_diff",
+    # data
+    "read_data",
+    "safe_json_dumps",
+    "write_json",
+    # timekit
+    "epoch_to_iso",
+    "iso_to_epoch",
+    "now_epoch",
+    "now_iso",
+    # web
+    "assess_webpage",
+    "prescreen_webpage",
+]

hugpy_tools/data.py

@@ -0,0 +1,74 @@
+"""Structured-data read/write: JSON always, TOML via stdlib ``tomllib``, YAML
+only when PyYAML happens to be installed. Writing is JSON-only on purpose
+(stdlib has no TOML/YAML serializer); the write path is atomic.
+
+Every format degrades honestly: an unavailable parser raises a clear error
+naming exactly what is missing, never a silent wrong-format guess. Ported in
+spirit from abstract_utilities.json_utils (safe_dump / safe_read) but slimmed
+to the two calls an agent needs.
+"""
+from __future__ import annotations
+
+import json
+import os
+
+from . import fs
+
+# Map a lowercase extension to a logical format.
+_EXT_FORMAT = {
+    ".json": "json",
+    ".toml": "toml",
+    ".yaml": "yaml",
+    ".yml": "yaml",
+}
+
+
+def _format_for(path: str, explicit: str | None) -> str:
+    if explicit:
+        return explicit.lower()
+    return _EXT_FORMAT.get(os.path.splitext(path)[1].lower(), "json")
+
+
+def read_data(path: str, fmt: str | None = None):
+    """Parse a JSON/TOML/YAML file to a Python object. Format is inferred from
+    the extension unless ``fmt`` is given. Raises ``RuntimeError`` when a
+    parser is unavailable, ``ValueError`` on an unknown format."""
+    fmt = _format_for(path, fmt)
+    if fmt == "json":
+        with open(path, "rb") as fh:
+            return json.loads(fh.read().decode("utf-8", errors="replace"))
+    if fmt == "toml":
+        try:
+            import tomllib  # stdlib >= 3.11
+        except ModuleNotFoundError as exc:
+            raise RuntimeError(
+                "TOML reading needs Python 3.11+ (stdlib 'tomllib'); this "
+                "interpreter is older. Install the 'tomli' backport and read "
+                "it yourself, or upgrade Python.") from exc
+        with open(path, "rb") as fh:
+            return tomllib.load(fh)
+    if fmt == "yaml":
+        try:
+            import yaml  # optional third-party
+        except ModuleNotFoundError as exc:
+            raise RuntimeError(
+                "YAML reading needs PyYAML, which is not installed "
+                "(hugpy_tools is stdlib-only). Install 'PyYAML' to enable "
+                "YAML.") from exc
+        with open(path, "rb") as fh:
+            return yaml.safe_load(fh)
+    raise ValueError("unknown data format %r (use json/toml/yaml)" % fmt)
+
+
+def safe_json_dumps(obj, indent: int = 2, sort_keys: bool = False) -> str:
+    """``json.dumps`` that never explodes on a non-serialisable value: anything
+    the encoder cannot handle is stringified via ``default=str`` and non-ASCII
+    is preserved (``ensure_ascii=False``)."""
+    return json.dumps(obj, indent=indent, sort_keys=sort_keys,
+                      ensure_ascii=False, default=str)
+
+
+def write_json(path: str, obj, indent: int = 2, sort_keys: bool = False) -> dict:
+    """Serialise ``obj`` to JSON and write it atomically. Returns
+    ``{path, bytes, mode}`` (from :func:`fs.atomic_write`)."""
+    return fs.atomic_write(path, safe_json_dumps(obj, indent, sort_keys) + "\n")

hugpy_tools/fs.py

@@ -0,0 +1,231 @@
+"""Safe file/directory operations (stdlib only).
+
+Every function here takes an ALREADY-CONFINED absolute path — confinement is
+the caller's job (see :func:`paths.confine`). That split keeps this module a
+pure file-ops library with no policy of its own, which is exactly what a
+standalone package wants.
+
+Highlights ported from abstract_utilities:
+  * encoding detection on read (BOM sniff -> utf-8 -> cp1252 fallback)
+  * atomic write (temp file in the same dir + ``os.replace``)
+  * exact string edit with an occurrence-count contract
+  * directory listing + bounded recursive tree + rich file info
+"""
+from __future__ import annotations
+
+import os
+import tempfile
+from datetime import datetime, timezone
+
+from . import hashkit
+
+# BOM -> declared encoding
+_BOMS = (
+    (b"\xef\xbb\xbf", "utf-8-sig"),
+    (b"\xff\xfe\x00\x00", "utf-32-le"),
+    (b"\x00\x00\xfe\xff", "utf-32-be"),
+    (b"\xff\xfe", "utf-16-le"),
+    (b"\xfe\xff", "utf-16-be"),
+)
+
+
+def detect_encoding(data: bytes) -> str:
+    """Best-effort encoding guess for ``data`` using only stdlib: honour a BOM,
+    then prefer strict utf-8, then fall back to cp1252 (a superset of latin-1
+    that decodes any byte). Never raises."""
+    for bom, enc in _BOMS:
+        if data.startswith(bom):
+            return enc
+    try:
+        data.decode("utf-8")
+        return "utf-8"
+    except UnicodeDecodeError:
+        return "cp1252"
+
+
+def read_text(path: str, max_bytes: int | None = None) -> dict:
+    """Read a text file with encoding detection. Returns
+    ``{path, encoding, bytes, truncated, text}``."""
+    with open(path, "rb") as fh:
+        if max_bytes is None:
+            raw = fh.read()
+            truncated = False
+        else:
+            raw = fh.read(max_bytes + 1)
+            truncated = len(raw) > max_bytes
+            raw = raw[:max_bytes]
+    enc = detect_encoding(raw)
+    return {
+        "path": path,
+        "encoding": enc,
+        "bytes": len(raw),
+        "truncated": truncated,
+        "text": raw.decode(enc, errors="replace"),
+    }
+
+
+def read_lines(path: str, start: int = 1, end: int | None = None,
+               number: bool = False) -> dict:
+    """Read a 1-based inclusive line range. ``end=None`` reads to EOF.
+    Returns ``{path, start, end, total_lines, returned, text}``."""
+    start = max(1, int(start))
+    with open(path, "rb") as fh:
+        raw = fh.read()
+    enc = detect_encoding(raw)
+    lines = raw.decode(enc, errors="replace").splitlines()
+    total = len(lines)
+    last = total if end is None else min(int(end), total)
+    chosen = lines[start - 1:last] if start <= total else []
+    if number:
+        width = len(str(last))
+        body = "\n".join("%*d\t%s" % (width, start + i, ln)
+                         for i, ln in enumerate(chosen))
+    else:
+        body = "\n".join(chosen)
+    return {
+        "path": path,
+        "start": start,
+        "end": last,
+        "total_lines": total,
+        "returned": len(chosen),
+        "text": body,
+    }
+
+
+def atomic_write(path: str, content: str, append: bool = False,
+                 encoding: str = "utf-8") -> dict:
+    """Write ``content`` durably. Overwrite is atomic (temp file in the same
+    directory + ``os.replace``, so a reader never sees a half-written file).
+    Append opens the real file directly (append has no atomic swap). Returns
+    ``{path, bytes, mode}``."""
+    os.makedirs(os.path.dirname(path) or ".", exist_ok=True)
+    data = content.encode(encoding, errors="replace")
+    if append:
+        with open(path, "ab") as fh:
+            fh.write(data)
+    else:
+        directory = os.path.dirname(path) or "."
+        fd, tmp = tempfile.mkstemp(dir=directory, prefix=".tmp-", suffix=".part")
+        try:
+            with os.fdopen(fd, "wb") as fh:
+                fh.write(data)
+            os.replace(tmp, path)
+        except BaseException:
+            try:
+                os.unlink(tmp)
+            except OSError:
+                pass
+            raise
+    return {"path": path, "bytes": len(data),
+            "mode": "append" if append else "overwrite"}
+
+
+def edit_replace(path: str, old: str, new: str, count: int | None = None) -> dict:
+    """Exact string replacement in a text file. By default every occurrence is
+    replaced; pass ``count`` to bound it. Raises ``ValueError`` if ``old`` is
+    empty or not found. Returns ``{path, replaced, bytes}``; the write is
+    atomic."""
+    if old == "":
+        raise ValueError("old must be a non-empty string")
+    info = read_text(path)
+    text = info["text"]
+    occurrences = text.count(old)
+    if occurrences == 0:
+        raise ValueError("old string not found in %s" % path)
+    replaced = occurrences if count is None else min(occurrences, int(count))
+    text = text.replace(old, new, -1 if count is None else int(count))
+    out = atomic_write(path, text, encoding="utf-8"
+                       if info["encoding"] in ("utf-8", "utf-8-sig", "cp1252")
+                       else info["encoding"])
+    return {"path": path, "replaced": replaced, "bytes": out["bytes"]}
+
+
+def _iso(ts: float) -> str:
+    return datetime.fromtimestamp(ts, tz=timezone.utc).isoformat()
+
+
+def file_info(path: str, hash_files: bool = False) -> dict:
+    """Rich stat for one path. For a regular file with ``hash_files`` the
+    sha256 is included."""
+    st = os.lstat(path)
+    is_link = os.path.islink(path)
+    real = os.path.realpath(path)
+    is_dir = os.path.isdir(real)
+    info = {
+        "path": path,
+        "name": os.path.basename(path.rstrip(os.sep)) or path,
+        "type": "dir" if is_dir else ("symlink" if is_link and not os.path.exists(real) else "file"),
+        "is_dir": is_dir,
+        "is_symlink": is_link,
+        "size": st.st_size,
+        "mtime": _iso(st.st_mtime),
+    }
+    if hash_files and not is_dir and os.path.isfile(real):
+        info["sha256"] = hashkit.sha256_file(real)
+    return info
+
+
+def list_dir(path: str, glob: str | None = None, files_only: bool = False,
+             dirs_only: bool = False, limit: int = 500) -> dict:
+    """List the immediate entries of a directory (sorted: dirs first, then
+    files, each alphabetical). Optional fnmatch ``glob`` on the entry name.
+    Returns ``{path, count, truncated, entries:[{name,type,size,mtime}]}``."""
+    import fnmatch
+    names = sorted(os.listdir(path))
+    entries = []
+    for name in names:
+        full = os.path.join(path, name)
+        is_dir = os.path.isdir(full)
+        if files_only and is_dir:
+            continue
+        if dirs_only and not is_dir:
+            continue
+        if glob and not fnmatch.fnmatch(name, glob):
+            continue
+        try:
+            st = os.lstat(full)
+            size, mtime = st.st_size, _iso(st.st_mtime)
+        except OSError:
+            size, mtime = None, None
+        entries.append({"name": name,
+                        "type": "dir" if is_dir else "file",
+                        "size": size, "mtime": mtime})
+    entries.sort(key=lambda e: (e["type"] != "dir", e["name"]))
+    truncated = len(entries) > limit
+    return {"path": path, "count": len(entries), "truncated": truncated,
+            "entries": entries[:limit]}
+
+
+def tree(path: str, max_depth: int = 3, max_entries: int = 500,
+         show_files: bool = True) -> dict:
+    """Bounded recursive directory tree rendered as indented text. Skips
+    symlinked directories (never recurses out through a link). Returns
+    ``{path, entries, truncated, text}`` where ``entries`` is the count
+    emitted."""
+    lines: list[str] = []
+    state = {"n": 0, "truncated": False}
+
+    def walk(cur: str, depth: int, prefix: str) -> None:
+        if depth > max_depth or state["truncated"]:
+            return
+        try:
+            names = sorted(os.listdir(cur))
+        except OSError:
+            return
+        dirs = [n for n in names if os.path.isdir(os.path.join(cur, n))
+                and not os.path.islink(os.path.join(cur, n))]
+        files = [n for n in names if not os.path.isdir(os.path.join(cur, n))]
+        ordered = [(n, True) for n in dirs] + \
+                  ([(n, False) for n in files] if show_files else [])
+        for name, is_dir in ordered:
+            if state["n"] >= max_entries:
+                state["truncated"] = True
+                return
+            state["n"] += 1
+            lines.append("%s%s%s" % (prefix, name, "/" if is_dir else ""))
+            if is_dir:
+                walk(os.path.join(cur, name), depth + 1, prefix + "  ")
+
+    walk(path, 1, "")
+    return {"path": path, "entries": state["n"],
+            "truncated": state["truncated"], "text": "\n".join(lines)}

hugpy_tools/hashkit.py

@@ -0,0 +1,35 @@
+"""Content hashing helpers (stdlib ``hashlib`` only).
+
+Ported from abstract_utilities.hash_utils (full_hash/quick_hash) but written
+to take an already-confined path and stream in bounded chunks so a large file
+never loads whole into memory.
+"""
+from __future__ import annotations
+
+import hashlib
+
+_CHUNK = 1024 * 1024  # 1 MiB streaming window
+
+
+def sha256_text(text: str) -> str:
+    return hashlib.sha256(text.encode("utf-8", errors="replace")).hexdigest()
+
+
+def sha256_file(path: str) -> str:
+    """Full SHA-256 of a file, streamed in 1 MiB windows."""
+    h = hashlib.sha256()
+    with open(path, "rb") as fh:
+        for block in iter(lambda: fh.read(_CHUNK), b""):
+            h.update(block)
+    return h.hexdigest()
+
+
+def quick_hash(path: str, bytes_to_read: int = 16384) -> str:
+    """Cheap identity hash: size + first ``bytes_to_read`` bytes. Good for
+    dedup pre-filtering, not a cryptographic guarantee."""
+    import os
+    h = hashlib.sha256()
+    h.update(str(os.path.getsize(path)).encode())
+    with open(path, "rb") as fh:
+        h.update(fh.read(max(0, int(bytes_to_read))))
+    return h.hexdigest()

hugpy_tools/paths.py

@@ -0,0 +1,80 @@
+"""Path confinement and path-part helpers (stdlib only).
+
+`confine` is the jail primitive ported from the agent's tools/fs.py: a path
+(absolute or root-relative) is resolved through ``os.path.realpath`` and must
+land under the realpath of ``root``. This rejects ``..`` escapes AND symlink
+escapes in one check. For writes the *parent* directory is confined (the
+target file may not exist yet, but its directory does), then the basename is
+re-attached — so a dangling path with a symlinked ancestor cannot slip through.
+
+Fail-closed: any ambiguity about where a path lands is a ``PathEscape``, never
+a guess. Everything here is pure stdlib so the module ports cleanly into a
+standalone package.
+"""
+from __future__ import annotations
+
+import os
+import re
+
+
+class PathEscape(ValueError):
+    """A path resolved outside the confinement root."""
+
+
+def confine(root: str, path: str, for_write: bool = False) -> str:
+    """Resolve ``path`` to a real absolute path confined under ``root``.
+
+    Raises :class:`PathEscape` on any escape (``..``, absolute-outside, or a
+    symlink pointing out of the jail).
+    """
+    real_root = os.path.realpath(root)
+    candidate = path if os.path.isabs(path) else os.path.join(real_root, path)
+    if for_write:
+        parent = os.path.realpath(os.path.dirname(candidate) or real_root)
+        resolved = os.path.join(parent, os.path.basename(candidate))
+    else:
+        resolved = os.path.realpath(candidate)
+    if resolved != real_root and not resolved.startswith(real_root + os.sep):
+        raise PathEscape("path %r escapes the root (%s)" % (path, real_root))
+    return resolved
+
+
+def file_parts(path: str) -> dict:
+    """Decompose ``path`` into its named components (dir/base/name/ext plus the
+    two enclosing directory levels), never raising on a bare or odd path."""
+    path = str(path)
+    dirname = os.path.dirname(path)
+    basename = os.path.basename(path)
+    filename, ext = os.path.splitext(basename)
+    parent_dirname = os.path.dirname(dirname)
+    super_dirname = os.path.dirname(parent_dirname)
+    return {
+        "file_path": path,
+        "dirname": dirname,
+        "basename": basename,
+        "filename": filename,
+        "ext": ext,
+        "dirbase": os.path.basename(dirname),
+        "parent_dirname": parent_dirname,
+        "parent_dirbase": os.path.basename(parent_dirname),
+        "super_dirname": super_dirname,
+        "super_dirbase": os.path.basename(super_dirname),
+    }
+
+
+_UNSAFE_NAME = re.compile(r"[^A-Za-z0-9._-]+")
+
+
+def sanitize_filename(name: str, replacement: str = "_", max_len: int = 255) -> str:
+    """Reduce ``name`` to a safe single path component: strip directory
+    separators, collapse unsafe characters, and bound the length. Never
+    returns an empty string (falls back to ``"file"``)."""
+    name = os.path.basename(str(name)).strip()
+    name = _UNSAFE_NAME.sub(replacement, name).strip("._-" + replacement)
+    if not name:
+        name = "file"
+    if len(name) > max_len:
+        stem, ext = os.path.splitext(name)
+        keep = max_len - len(ext)
+        name = (stem[:keep] if keep > 0 else stem[:max_len]) + (ext if keep > 0 else "")
+    return name

hugpy_tools/search/README.md

@@ -0,0 +1,173 @@
+# `hugpy_tools.search` — fast, self-contained local content search
+
+Stdlib-only. Imports nothing from the rest of hugpy. Give it roots + terms; it
+returns line-level hits. It replaces the need for `abstract_search` /
+`abstract_utilities` in the agent's filesystem search path: same behaviour and
+API, correct include/exclude, materially faster.
+
+## Why
+
+`abstract_search` shells out to Unix `find` (a subprocess per query, no
+directory pruning) and its include/exclude filters are wrong (see
+"abstract_search bugs" at the bottom). This module walks with `os.scandir`,
+prunes excluded directories at walk time (they are never entered), filters by
+extension before any `stat`/`open`, and matches literals on bytes — decoding
+only the lines it needs.
+
+## NO implicit filtering
+
+A search engine must not allow/exclude anything silently. **With no user
+filters, you get EVERYTHING under the roots** — dotfiles, `node_modules`,
+`.venv`, binaries. There is no default extension whitelist, no default excluded
+dirs, no default hidden-file skipping.
+
+Opt into the old conveniences EXPLICITLY with named **presets**:
+
+| preset | applies |
+|---|---|
+| `"code"` | `CODE_TEXT_EXTS` allow-list + `EXCLUDE_NOISE_DIRS` + skip hidden |
+| `"noise"` | `EXCLUDE_NOISE_DIRS` + `BINARY_EXTS` excludes + skip hidden |
+| `"text"` | `BINARY_EXTS` + `EXCLUDE_NOISE_DIRS` excludes + skip hidden |
+
+`CODE_TEXT_EXTS` / `EXCLUDE_NOISE_DIRS` / `BINARY_EXTS` are exported; pass them
+as filter kwargs if you want.
+
+**Preset merge = UNION (never be surprised).** Presets and your kwargs are
+unioned field-by-field. Good for excludes; but the allow-whitelist is unioned
+too, so a whitelist preset **broadens**: `preset="code"` + `allowed_exts=[".rs"]`
+matches code/text **and** `.rs`, not just `.rs`. To **narrow** by extension use
+`preset="noise"` (excludes only) + your own `allowed_exts`, or no preset.
+Everything applied shows in the report's `effective_filters`.
+
+Nothing is dropped silently. Every `find_content` / `get_files_and_dirs` /
+`search_content` / `collect` records **`last_report()`**: `effective_filters`
+(exactly what applied, incl. presets), `scanned`, `matched` (search) and
+`skipped` counts by reason (`binary` / `unreadable` / `too_large` /
+`symlink_loop`). Content search still skips binary files (a NUL-byte match is
+meaningless) — but reports them. Pass `report=True` for `(result, report)`
+inline.
+
+## Include / exclude semantics (when you DO filter)
+
+- **Exclude always wins.** Anything matched by an exclude is gone, even if an
+  include also matches it.
+- **Include narrows.** No include ⇒ everything not excluded is in scope.
+- **Directory excludes** match an exact **path segment** and are **pruned**
+  during the walk: `build` drops `.../build/...` but never `.../rebuild/...`,
+  and the excluded subtree is never descended.
+- **Globs** (`include_globs` / `exclude_globs`, aliases `allowed_patterns` /
+  `exclude_patterns`) match with `fnmatch` against **both** the root-relative
+  path **and** the basename, so `**/mct/*.py` and `*session*` both work (`*`
+  crosses `/`; a leading `**/` is normalised to `*`).
+- **Extensions** (`exts` / `exclude_exts`) compare lower-cased, with a leading
+  dot.
+- **Case:** name/glob/extension matching is case-insensitive. Literal content
+  matching is case-insensitive unless `case_sensitive=True`.
+- **Symlinks** are not followed by default (least surprising; loops avoided).
+  `max_depth` / `max_bytes` are explicit params (defaults: unbounded, no cap).
+
+`__init__.py` is never treated specially (the old `abstract_search` dropped
+every `__init__.py` by default — one of the bugs this replaced).
+
+```python
+from hugpy_tools.search import get_files_and_dirs, find_content, last_report
+
+files = get_files_and_dirs(directory="/srv/app")[1]          # EVERYTHING
+files = get_files_and_dirs(directory="/srv/app", preset="code")[1]   # code/text, no noise
+hits, rep = find_content(directory="/srv/app", strings=["x"], preset="noise", report=True)
+rep["effective_filters"]; rep["skipped"]                     # or last_report()
+```
+
+## Examples
+
+```python
+from hugpy_tools.search import find_content, iter_files, make_filters, read_any_file
+
+# 1. Enumerate .py files (node_modules etc. only pruned if you ask — preset/exclude).
+files = list(iter_files("/srv/app", make_filters(allowed_exts=[".py"], preset="noise")))
+
+# 2. Literal search (case-insensitive). Returns
+#    [{"file_path": str, "lines": [{"line": int, "content": str}, ...]}, ...]
+hits = find_content(directory="/srv/app", strings=["assure_model_key"])
+
+# 3. Intersection: files containing BOTH terms (total_strings defaults True).
+hits = find_content(directory="/srv/app", strings=["nginx", "ssl"])
+
+# 4. Path-scoped: only .py under any mct/ directory.
+hits = find_content(directory="/srv/app", strings=["submit_pull"],
+                    allowed_patterns=["**/mct/*.py"])
+
+# 5. Regex.
+hits = find_content(directory="/srv/app", strings=[r"def\s+\w+\("], regex=True)
+
+# 6. Read one file as text (binary/oversize -> ValueError; missing -> FileNotFoundError).
+text = read_any_file("/srv/app/README.md")
+```
+
+### Directive helpers used by the steward
+
+`get_file_filters(root, **kw) -> (dirs, cfg, allowed, include_files, recursive)`
+and `get_files_and_dirs(*, directory, cfg, recursive) -> (dirs, files)` mirror
+`abstract_search`'s shapes so callers unpack them unchanged. `all` / `any` /
+`none` directive logic lives in the caller (e.g. hugpy-agent's
+`mct/session.py`): `all` = intersection, `any` = union, `none` = a file-level
+veto (a file with an excluded term is dropped, not merely demoted).
+
+## Speed
+
+`use_rg` (default **False**) enables an optional ripgrep prefilter. It is a
+*pure* prefilter — ripgrep only narrows which files the Python line-matcher
+opens, so results are byte-for-byte identical to the pure-Python path (tested).
+It is off by default because the pure-Python pruned walk benchmarked **faster**
+on real trees (ripgrep run with `-uuu` rescans ignored/venv subtrees the walk
+prunes, and its subprocess cost rarely pays off); turn it on only for very large
+trees scanned with a highly selective term.
+
+## Benchmark (this machine, 2026-09-24)
+
+Enumerate `*.py` (excl. node_modules/build), literal `import`, regex
+`def\s+\w+\(`. Wall time, best-of-2.
+
+| tree | op | hugpy_tools | abstract_search | speedup |
+|---|---|---|---|---|
+| `/srv/hugpy/src/hugpy` | enumerate | 10.6 ms (n=1133) | 210 ms (n=863) | ~20x |
+| | literal | 86 ms (n=1106) | 422 ms (n=860) | ~5x |
+| | regex | 43 ms (n=1011) | 266 ms (n=820) | ~6x |
+| `/srv/pyit/dev` | enumerate | 35 ms (n=3347) | 455 ms (n=4566) | ~13x |
+| | literal | 114 ms (n=3205) | 1000 ms (n=4329) | ~9x |
+| | regex | 108 ms (n=1914) | 853 ms (n=3377) | ~8x |
+
+`abstract_utilities` delegates its search to `abstract_search`, so its numbers
+track `abstract_search`'s (1010 ms literal on `/srv/pyit/dev`).
+
+**Count differences are `abstract_search` bugs, not misses:**
+- `hugpy_tools` finds the `__init__.py` files `abstract_search` drops (its
+  default `__init__*` exclude-pattern + `__init__` exclude-dir): 270 real source
+  files on the first tree, ~1010 on the second.
+- `abstract_search` returns thousands of `.venv` / `site-packages` /
+  hidden-backup files (2229 on `/srv/pyit/dev`) that `hugpy_tools` prunes by
+  default — and pays for descending them.
+
+## abstract_search bugs (for reference; not fixed here)
+
+Paths under `/srv/pyit/dev/abstract_search/src/abstract_search/`:
+- `find_collect.py:89-91` — excluded dirs become `find ... ! -path '*d*'`: a
+  **substring** match on the full path (so `build` also drops `rebuild`,
+  `prebuilder`) that is a result **filter, not a `-prune`**, so `find` still
+  descends the excluded tree (slow).
+- `find_collect.py:93-99` & `filters.py:296-301` — include/exclude *patterns*
+  match the **basename only** (`-name` / `fnmatch(name, …)`), so path globs like
+  `**/mct/session.py` match nothing.
+- `constants.py:98,102` — defaults exclude the `__init__` dir **and** the
+  `__init__*` pattern, silently dropping every `__init__.py`.
+- `filters.py:154-169` (`ensure_patterns`) — a pattern with no `*`/`?` is
+  rewritten to a prefix/suffix glob, so an intended exact match (`session.py`)
+  becomes `session.py*`.
+- `find_content.py:35-44,186` (`_normalize`) — final line matching strips `//`
+  comments and everything after them, so a term inside a comment or after
+  `http://` never matches, even though the file passed the prefilter.
+- `find_content.py:103,173` — each matched file is read from disk **twice**
+  (once in `getPaths`, once in the match loop).
+- `find_content.py:108` (`getPaths`) — literal prefilter uses
+  `tot_strings not in og_content`: **case-sensitive**, which is why callers had
+  to retry every term in both cases.