maskerade 0.1.0__py3-none-any.whl

maskerade/__init__.py

@@ -0,0 +1,53 @@
+"""maskerade — local anonymisation of pentest prompts and HTTP traffic.
+
+Masks sensitive values out of text before it is sent to a third-party AI, and
+restores them in the reply. Everything runs locally; nothing is sent anywhere
+by this package.
+
+    from maskerade import Pipeline
+
+    p = Pipeline()                   # the default tier: "fast"
+    r = p.run(text)
+    r["masked_text"], r["mask_map"], r["entities"], r["validation"]
+    p.unmask(reply, r["mask_map"])
+
+Two tiers:
+  fast  custom + regex + form + keyval + checksum + id
+        pure Python, three dependencies, tens of milliseconds.
+        THE DEFAULT, and what `pip install maskerade` can run.
+  full  the above plus gliner (+ org_evidence). This is C2, the configuration
+        every number in the decision record was measured on, and it needs
+        `pip install maskerade[ner]` — plus a one-off model download,
+        `python -m maskerade.warm --full`.
+
+Both tiers need the BPE encoder, because `form` is in both:
+
+    python -m maskerade.warm         # ~1.7 MB, once, with network
+
+That is not optional. form fails closed without it, and fail-closed means the
+proxy does not start.
+
+WHICH TIER AM I RUNNING?
+------------------------
+    maskerade --version              names the default tier and the extras
+    p.mode_string()                  names the stages actually live
+
+The published numbers — gold 0.919 recall, corpus 0.955 — were measured with
+tier "full" and maskerade[ner]. THE DEFAULT INSTALL IS A DIFFERENT DETECTOR
+SET AND ITS RECALL HAS NOT BEEN MEASURED. Anywhere those numbers appear, they
+describe `maskerade[ner]` at tier full and nothing else.
+
+The tier boundary itself is still not derived from measurement — see the
+Week 1 laptop run. Until it is, "fast" and "full" are names for stage sets,
+not a policy.
+"""
+from .pipeline import (
+    Pipeline, DetectorError, TIER_FAST, TIER_FULL, TIERS, STAGE_KEYS,
+    DEFAULT_TIER,
+)
+
+__all__ = [
+    "Pipeline", "DetectorError",
+    "TIER_FAST", "TIER_FULL", "TIERS", "STAGE_KEYS", "DEFAULT_TIER",
+]
+__version__ = "0.1.0"

maskerade/_extras.py

@@ -0,0 +1,238 @@
+# -*- coding: utf-8 -*-
+"""Which optional extras are installed — answered WITHOUT importing them.
+
+A bug report has to say which install it came from. `pip install maskerade`
+and `pip install maskerade[ner]` are the same version string, the same
+commands and two different detector sets, so "0.1.0.dev0" on its own does not
+identify the thing that ran.
+
+IT MUST NOT IMPORT WHAT IT REPORTS ON
+-------------------------------------
+importlib.util.find_spec() locates a module without executing it, so probing
+for torch costs a path search rather than the measured ~88 s of instrumented
+import time that `import gliner2` triggers once torch is present. That is not
+an optimisation here, it is a correctness requirement: bench/no_torch_check.py
+asserts that nothing heavy appears in sys.modules under the default tier, and
+this module is reachable from `maskerade --version` and from the proxy's
+startup path. A plain `import torch` inside a try/except would make the gate
+fail and would make the light install pay for a question about itself.
+
+find_spec can do more than return None — it raises ModuleNotFoundError when a
+PARENT package is missing, and ValueError on a half-initialised module — so
+every call is wrapped. An extra reported as absent when it is present is a
+cosmetic error in a version banner; an exception there would take out the
+command.
+"""
+from importlib import util as _util
+
+
+# extra name -> the modules that must all be importable for it to be usable.
+#
+# gliner2 alone is not enough and that is the whole reason this mapping is not
+# just the extra's name: `pip install gliner2` without [local] installs a
+# 356 kB wheel with no torch and no transformers, which resolves here, imports
+# there, and fails at model load. The three names below are what
+# gliner2[local] actually brings (read from its wheel metadata, 2.1.0).
+EXTRA_MODULES = {
+    "ner": ("gliner2", "torch", "transformers"),
+}
+
+
+def module_present(name):
+    """True when `name` can be located on the path. Never imports it."""
+    try:
+        return _util.find_spec(name) is not None
+    except (ImportError, ValueError):
+        # ModuleNotFoundError on a missing parent package, ValueError on a
+        # module that is in sys.modules with a None __spec__.
+        return False
+
+
+def module_version(name):
+    """The installed version of a distribution, or None. Never imports it.
+
+    importlib.metadata reads the .dist-info on disk, so asking for torch's
+    version costs a directory scan rather than the seconds an import costs.
+
+    This exists because "ner: installed" is not enough to reproduce a run.
+    gliner2 is pinned, torch and transformers are not, and all three decide
+    what the model produces — so the version banner names the trio and a bug
+    report stops being a guess.
+    """
+    try:
+        from importlib import metadata as _md
+    except ImportError:                      # pragma: no cover - py<3.8
+        return None
+    try:
+        return _md.version(name)
+    except Exception:
+        # PackageNotFoundError, and anything a malformed dist-info raises. A
+        # version we cannot read is reported as unknown, never as absent:
+        # the module may be perfectly importable from a source checkout.
+        return None
+
+
+def _requirements(dist="maskerade"):
+    """(requirements, sources) for `dist`, preferring .dist-info.
+
+    WHY THIS IS NOT JUST importlib.metadata.requires(). Running from the
+    repository root — which is where shipcheck and `maskerade --version` are
+    run — puts the CWD on sys.path, and a `maskerade.egg-info` left there by
+    `pip install -e .` is then discovered as a distribution. It is a
+    SEPARATE, OLDER copy of the metadata: measured 2026-10-17, an egg-info
+    written before the pin landed reported `gliner2[local]` while the
+    installed .dist-info beside the package correctly reported
+    `gliner2[local]==2.0.0`.
+
+    So the probe that exists to avoid a second, drifting copy of the pin was
+    reading a second, drifting copy of the pin. .dist-info wins, and every
+    source found is returned so a caller can say when they disagree.
+    """
+    try:
+        from importlib import metadata as _md
+    except ImportError:                      # pragma: no cover - py<3.8
+        return [], []
+    want = dist.lower().replace("_", "-")
+    found = []
+    try:
+        for d in _md.distributions():
+            try:
+                name = (d.metadata["Name"] or "")
+            except Exception:
+                continue
+            if name.lower().replace("_", "-") != want:
+                continue
+            path = str(getattr(d, "_path", "") or "")
+            # 0 sorts first: a real install beats a build artifact.
+            rank = 0 if path.endswith(".dist-info") else 1
+            try:
+                reqs = list(d.requires or [])
+            except Exception:
+                reqs = []
+            found.append((rank, path, reqs))
+    except Exception:
+        found = []
+    if not found:
+        try:
+            return list(_md.requires(dist) or []), []
+        except Exception:
+            return [], []
+    found.sort(key=lambda t: t[0])
+    return found[0][2], [(p, r) for _rank, p, r in found]
+
+
+def metadata_conflict(module="gliner2", extra="ner", dist="maskerade"):
+    """Distinct requirement strings for `module` across metadata copies.
+
+    More than one entry means a stale egg-info (or a second install) is
+    stating a different constraint from the one in force. Returns a list of
+    (path, requirement) or [] when everything agrees.
+    """
+    _reqs, sources = _requirements(dist)
+    if len(sources) < 2:
+        return []
+    seen = {}
+    for path, reqs in sources:
+        found = None
+        for req in reqs:
+            head = req.split(";")[0].strip()
+            name = head.split("[")[0].split("=")[0].split("<")[0] \
+                       .split(">")[0].split("!")[0].split("~")[0].strip()
+            if name.lower().replace("_", "-") != module.lower().replace("_", "-"):
+                continue
+            if extra and ("extra" not in req or extra not in req):
+                continue
+            found = head
+            break
+        seen[path] = found
+    if len(set(seen.values())) < 2:
+        return []
+    return sorted(seen.items())
+
+
+def declared_requirement(module, extra=None, dist="maskerade"):
+    """What THIS package asks for, read from its own installed metadata.
+
+    The pin lives in pyproject.toml and nowhere else. Restating it as a
+    constant here would create a second copy that drifts silently, and a
+    drifting copy of a pin is worse than no copy, because it would report a
+    constraint the installer never applied. Reading it back from the
+    installed metadata means this can only ever report what is actually in
+    force.
+
+    Returns the requirement string (e.g. "gliner2[local]==2.0.0") or None.
+    """
+    reqs, _sources = _requirements(dist)
+    for req in reqs:
+        head = req.split(";")[0].strip()
+        name = head.split("[")[0].split("=")[0].split("<")[0] \
+                   .split(">")[0].split("!")[0].split("~")[0].strip()
+        if name.lower().replace("_", "-") != module.lower().replace("_", "-"):
+            continue
+        if extra and ("extra" not in req or extra not in req):
+            continue
+        return head
+    return None
+
+
+def pin_mismatch():
+    """(requirement, installed) when an EXACT pin is not what is installed.
+
+    The case this catches: the pin is raised, or an older environment is
+    reused, and the frozen snapshots no longer describe the library doing
+    the detecting. Only `==` is checked — a range is a deliberate statement
+    that several versions are acceptable, so a version inside it is not a
+    mismatch to report.
+    """
+    req = declared_requirement("gliner2", extra="ner")
+    if not req or "==" not in req:
+        return None
+    wanted = req.split("==", 1)[1].strip()
+    got = module_version("gliner2")
+    if got is None or got == wanted:
+        return None
+    return (req, got)
+
+
+def extras_status():
+    """{extra: {"installed": bool, "present": [...], "missing": [...]}}"""
+    out = {}
+    for extra, modules in EXTRA_MODULES.items():
+        present = [m for m in modules if module_present(m)]
+        missing = [m for m in modules if m not in present]
+        out[extra] = {
+            "installed": not missing,
+            "present": present,
+            "missing": missing,
+        }
+    return out
+
+
+def summary():
+    """One line naming the extras, for a version banner.
+
+    A PARTIAL extra is called out rather than rounded down to "absent": the
+    half-installed state is the one that produces a confusing bug report,
+    because the tier a caller asks for decides whether it is fatal.
+    """
+    bits = []
+    for extra, st in sorted(extras_status().items()):
+        if st["installed"]:
+            # Name the versions. All three decide what the model emits, and
+            # only one of them is pinned.
+            seen = []
+            for m in EXTRA_MODULES[extra]:
+                v = module_version(m)
+                seen.append("%s %s" % (m, v if v else "?"))
+            bits.append("%s: installed (%s)" % (extra, ", ".join(seen)))
+        elif st["present"]:
+            bits.append("%s: PARTIAL (missing %s)"
+                        % (extra, ", ".join(st["missing"])))
+        else:
+            bits.append("%s: not installed" % extra)
+    return "; ".join(bits) if bits else "no optional extras defined"
+
+
+def ner_available():
+    """True when the full tier could actually run on this install."""
+    return extras_status()["ner"]["installed"]

maskerade/allowlist/__init__.py

@@ -0,0 +1,274 @@
+"""Technical-vocabulary allow-list — post-detection suppression.
+
+Pipeline component. Pipeline.__init__ loads it ONCE (the agent loop calls
+run() several times per tool call, so a per-run load would re-read 10k terms
+each time) and run() applies it after every detection stage, before span
+reconciliation and masking. Toggled by stages.allowlist; the run(whitelist=)
+parameter still overrides it per call for the benchmark.
+
+Every *.yaml in this directory is merged at load time, so an optional,
+separately-licensed file such as ``allowlist_wappalyzer.yaml`` is picked up
+automatically when present and simply absent otherwise. A missing directory or
+missing pyyaml yields an empty AllowList, which is falsy (``__len__``) and
+therefore disables suppression on its own — it never raises.
+
+What it does
+------------
+Drops detected entities whose whole text is universal technical vocabulary —
+technology/product names, protocol parameter names, standard-format acronyms —
+that identifies nobody. This is the industry-standard mechanism for the
+"technical vocabulary flagged as sensitive" problem (Presidio allow_list, Google
+DLP exclusion dictionaries, Datadog redaction-excluded-identifiers, gitleaks
+allowlists): a sourced suppression pass applied after detection, not a change to
+the detectors.
+
+Matching contract (the safety property)
+---------------------------------------
+  * whole-token, case-insensitive: an entity is suppressed only when its ENTIRE
+    text equals an allow-list term (after casefold + surrounding-quote/space
+    trim). Substrings never match, so allow-listing "oracle" cannot un-mask
+    "oracle-client-db-prod-4471" — that string is not equal to "oracle".
+
+    SCOPE OF THAT SENTENCE, corrected 2026-10-14. It is true of THIS module
+    and it is not true of the pipeline as a whole. `maskerade/stages/id.py`
+    reads the same term list and tests a candidate's PREFIX against it, so
+    `oracle-client-db-prod-4471` IS vetoed there — by the id stage, not by
+    this filter. Measured; owner decision to keep, with the reasoning and
+    the accepted cost recorded at the `_load_allowlist` comment in that file.
+    Anyone relying on the whole-token property must read it as a property of
+    this filter's own suppression pass, not as a guarantee that an
+    allow-listed term can never cause a longer string to go unmasked.
+  * custom-stage entities are never suppressed: explicit pentester intent wins,
+    so a client literally named "Oracle" can be forced masked via a custom rule.
+    (At the benchmark's injection point only NER-slot entities are seen, so this
+    holds by construction, but the guard is explicit for safety.)
+
+Residual risk: a client legitimately named after a technology (company called
+"Oracle") whose name is detected as a bare "Oracle" will not be masked by the
+probabilistic detectors once suppressed here. Mitigation: a custom rule, or
+listing the term under `exclude:` in the YAML. This is the accepted trade of any
+allow-list and is why matching is whole-token and custom-exempt.
+Beyond the vocabulary allow-list, this module also applies a few TYPE-AWARE
+POLICY FILTERS (config-driven, in allowlist_overrides.yaml) that drop whole
+classes of detection the pipeline should not mask:
+  * email_requires_at — an EMAIL_ADDRESS detection with no '@' is never a real
+    address (fixes a field/word merely containing "email" being masked);
+  * drop_types — entity types out of scope for the engagement (e.g. FINANCIAL_DATA);
+  * drop_single_token_types — one-word place names (bare city/region/timezone),
+    which are public geography rather than client PII.
+"""
+import os
+import re
+
+_DEFAULT_PATH = os.path.join(os.path.dirname(os.path.abspath(__file__)), "allowlist.yaml")
+
+
+class AllowList:
+    """A loaded allow-list with a whole-token matcher + type-aware policy filters."""
+
+    def __init__(self, terms_by_group=None, exclude=None, policy=None):
+        # term (casefolded) -> group name, for per-source attribution
+        self._term_group = {}
+        # separator-collapsed term -> group, for space/case/separator-insensitive
+        # matching of multi-part TECH names ("amazon s3" == "AmazonS3",
+        # "content-type" == "ContentType"). Only built for terms that actually
+        # contain a separator and whose collapsed form is >= 5 chars, so short
+        # tokens can't collide; single tokens are already handled by exact match.
+        self._collapsed = {}
+        for group, terms in (terms_by_group or {}).items():
+            for t in terms:
+                key = self._norm(t)
+                if key:
+                    self._term_group.setdefault(key, group)
+                    ck = self._collapse(key)
+                    if ck and len(ck) >= 5 and ck != key:
+                        self._collapsed.setdefault(ck, group)
+        # `exclude` lets a user force-keep a term that would otherwise be
+        # suppressed (e.g. their client really is called "Oracle").
+        for t in (exclude or []):
+            k = self._norm(t)
+            self._term_group.pop(k, None)
+            self._collapsed.pop(self._collapse(k), None)
+        self.policy = policy or {}
+
+    # ── normalisation: casefold + strip surrounding quotes/space ──
+    @staticmethod
+    def _norm(text):
+        if not text:
+            return ""
+        s = text.strip()
+        while len(s) >= 2 and s[0] in "\"'`" and s[0] == s[-1]:
+            s = s[1:-1].strip()
+        return s.casefold()
+
+    # collapse spaces / underscores / hyphens / dots for separator-insensitive
+    # matching of multi-part technology names.
+    @staticmethod
+    def _collapse(norm_text):
+        return re.sub(r"[ _.\-]", "", norm_text)
+
+    def __len__(self):
+        return len(self._term_group)
+
+    def group_of(self, entity_text):
+        """Return the allow-list group that matches this text, or None."""
+        n = self._norm(entity_text)
+        g = self._term_group.get(n)
+        if g is not None:
+            return g
+        # fall back to separator-collapsed match (multi-part tech names)
+        ck = self._collapse(n)
+        if len(ck) >= 5:
+            return self._collapsed.get(ck)
+        return None
+
+    def _policy_reason(self, e):
+        """Return a policy label if this entity should be dropped by a type-aware
+        policy rule, else None. Applied only to non-custom, non-vocab entities.
+        """
+        p = self.policy
+        if not p:
+            return None
+        etype = (e.get("entity_type") or "").upper()
+        text = e.get("text", "") or ""
+
+        # 1. A real email always contains '@'. An EMAIL_ADDRESS detection without
+        #    one is a false positive (a field/word merely containing "email").
+        #    The '@' may be URL-encoded: the email pattern in
+        #    default_config.yaml deliberately matches %40 and %2540, because
+        #    addresses cross the wire encoded. Until 2026-10-21 this check
+        #    tested for a literal '@' only and silently dropped every such
+        #    match, so `jan.novak%40acmebank.cz` was sent whole.
+        if p.get("email_requires_at") and etype in ("EMAIL_ADDRESS", "EMAIL"):
+            if not any(at in text for at in ("@", "%40", "%2540")):
+                return "policy:email_no_at"
+
+        # 2. Entity types dropped entirely (out of scope for the engagement).
+        if etype in {t.upper() for t in (p.get("drop_types") or [])}:
+            return f"policy:drop_type:{etype}"
+
+        # 3. One-word place names are public geography, not client PII. A single
+        #    token = no internal whitespace (multi-word street addresses stay).
+        if etype in {t.upper() for t in (p.get("drop_single_token_types") or [])}:
+            if text.strip() and not re.search(r"\s", text.strip()):
+                return f"policy:single_token:{etype}"
+
+        return None
+
+    def filter(self, entities, text=None):
+        """Split entities into (kept, suppressed).
+
+        Suppressed items are annotated with `_allowlist_group` (a vocabulary
+        group name, or a `policy:...` label). Custom-stage entities are always
+        kept — explicit pentester intent overrides both vocab and policy.
+
+        `text` (the source the entities were detected in) enables the span-aware
+        guard: a vocabulary or single-word-location match is NOT suppressed when
+        the entity is a fragment of a larger identifier in the text (e.g. the
+        region in `...cluster-x.eu-central-1.rds.amazonaws.com`). The guard can
+        only ever KEEP more, so it is strictly recall-safe.
+
+        CALLER CONTRACT: `text` must be the exact string the entities' start/end
+        offsets index into. Entities from a different coordinate frame (the
+        regex stage detects against pre-regex-masking text) must be passed in a
+        SEPARATE call with text=None — offsets that are merely in range but
+        point at the wrong characters make the guard decide confidently and
+        wrongly. See Pipeline.run(), which splits the list for this reason.
+        """
+        kept, suppressed = [], []
+        for e in entities:
+            if e.get("stage") == "custom":
+                kept.append(e)
+                continue
+            group = self.group_of(e.get("text", ""))
+            is_vocab = group is not None
+            reason = group if is_vocab else self._policy_reason(e)
+            if reason is not None:
+                # The embedding guard applies to vocabulary matches and to the
+                # single-word-location policy (tokens that could be a fragment of
+                # a sensitive composite). It does NOT apply to email_no_at or
+                # drop_type, which are type policies unrelated to embedding.
+                guardable = is_vocab or reason.startswith("policy:single_token")
+                if guardable and text is not None and self._is_embedded(e, text):
+                    reason = None
+            if reason is None:
+                kept.append(e)
+            else:
+                item = dict(e)
+                item["_allowlist_group"] = reason
+                suppressed.append(item)
+        return kept, suppressed
+
+    # identifier continuation chars — a run of these around an entity means the
+    # entity is a sub-token of a larger identifier (hostname, ARN, resource id).
+    _ID_CHARS = frozenset(
+        "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789._-"
+    )
+
+    def _is_embedded(self, e, text):
+        """True if the entity is a fragment of a larger identifier in `text`.
+
+        Expands outward from the entity span over identifier characters; if the
+        surrounding maximal run (minus edge punctuation) is longer than the
+        entity itself, the entity is embedded (e.g. a region inside a hostname).
+        """
+        try:
+            s = int(e["start"]); en = int(e["end"])
+        except (KeyError, TypeError, ValueError):
+            return False
+        if s < 0 or en > len(text) or s >= en:
+            return False
+        i = s
+        while i > 0 and text[i - 1] in self._ID_CHARS:
+            i -= 1
+        j = en
+        while j < len(text) and text[j] in self._ID_CHARS:
+            j += 1
+        run = text[i:j].strip("._-")
+        return len(run) > (en - s)
+
+
+def load_allowlist(path=None):
+    """Load the allow-list. Returns an AllowList (empty if unavailable).
+
+    If `path` is a file, load just that file. If `path` is a directory (or None,
+    meaning this package's own directory), load and MERGE every ``*.yaml`` in it
+    — so an optional, separately-licensed file such as ``allowlist_wappalyzer.yaml``
+    is picked up automatically when present and simply absent otherwise.
+
+    Never raises: a missing file/dir or missing pyyaml yields an empty (no-op)
+    allow-list so the benchmark still runs.
+    """
+    try:
+        import yaml
+    except ImportError:
+        print("  [ALLOWLIST] pyyaml not installed — suppression disabled")
+        return AllowList()
+
+    if path is None:
+        path = os.path.dirname(os.path.abspath(__file__))
+
+    if os.path.isdir(path):
+        import glob
+        files = sorted(glob.glob(os.path.join(path, "*.yaml")))
+    elif os.path.exists(path):
+        files = [path]
+    else:
+        print(f"  [ALLOWLIST] {path} not found — suppression disabled "
+              f"(run tests/allowlist/build.py)")
+        return AllowList()
+
+    terms_by_group = {}
+    exclude = []
+    policy = {}
+    for fp in files:
+        with open(fp, encoding="utf-8") as f:
+            doc = yaml.safe_load(f) or {}
+        for name, g in (doc.get("groups") or {}).items():
+            terms_by_group.setdefault(name, [])
+            terms_by_group[name].extend(g.get("terms") or [])
+        exclude.extend(doc.get("exclude") or [])
+        pol = doc.get("policy")
+        if isinstance(pol, dict):
+            policy.update(pol)
+    return AllowList(terms_by_group, exclude=exclude, policy=policy)