devguard-scanner 0.1.0__py3-none-any.whl

devguard/analyzers/content.py

@@ -0,0 +1,240 @@
+"""Declarative YAML content rules (ADR-004).
+
+Rule match expressions:
+    {regex: "...", min_count: 1, ignorecase: false}
+    {all: [expr, ...]}      every child must match (optionally within N lines, see `within`)
+    {any: [expr, ...]}      at least one child matches
+    {not: expr}             child must not match
+`within: file` (default) or `within: "lines:N"` (top-level `all` only).
+"""
+
+from __future__ import annotations
+
+import fnmatch
+import re
+from collections.abc import Iterable
+from dataclasses import dataclass
+from importlib import resources
+from itertools import islice
+from typing import Any, Literal
+
+import yaml
+from pydantic import BaseModel, Field, field_validator
+
+from devguard.analyzers.base import AnalysisContext, FileView
+from devguard.core.models import Confidence, Finding, Severity, Source
+from devguard.rules.catalog import RuleMeta, build_finding, register
+
+_MAX_MATCHES = 500
+
+
+class RuleTests(BaseModel):
+    path: str | None = None
+    positive: list[str] = Field(min_length=1)
+    negative: list[str] = Field(min_length=1)
+
+
+class YamlRule(BaseModel):
+    id: str
+    title: str
+    threat: list[str] = Field(min_length=1)
+    languages: list[str] = Field(default_factory=list)
+    paths: list[str] = Field(default_factory=list)
+    severity: Severity
+    confidence: Confidence
+    category: str = "capability"
+    match: dict[str, Any]
+    within: str = "file"
+    explanation: str
+    benign_context: str = ""
+    next_step: str = "Read the surrounding code. Do not run it. Check where data comes from and goes to."
+    tags: list[str] = Field(default_factory=list)
+    tests: RuleTests
+
+    @field_validator("id")
+    @classmethod
+    def _id_format(cls, v: str) -> str:
+        if not re.fullmatch(r"DG-[A-Z0-9]+(-[A-Z0-9]+)*-\d{3}", v):
+            raise ValueError(f"bad rule id {v!r}")
+        return v
+
+    @field_validator("within")
+    @classmethod
+    def _within_format(cls, v: str) -> str:
+        if v != "file" and not re.fullmatch(r"lines:\d+", v):
+            raise ValueError("within must be 'file' or 'lines:N'")
+        return v
+
+
+class RulePack(BaseModel):
+    pack: str
+    rules: list[YamlRule]
+
+
+@dataclass
+class _Leaf:
+    pattern: re.Pattern[str]
+    min_count: int
+
+
+@dataclass
+class _Node:
+    op: Literal["all", "any", "not", "leaf"]
+    children: list[_Node]
+    leaf: _Leaf | None = None
+
+
+def _compile(expr: dict[str, Any]) -> _Node:
+    if "regex" in expr:
+        flags = re.MULTILINE | (re.IGNORECASE if expr.get("ignorecase") else 0)
+        return _Node("leaf", [], _Leaf(re.compile(expr["regex"], flags), int(expr.get("min_count", 1))))
+    for op in ("all", "any"):
+        if op in expr:
+            return _Node(op, [_compile(e) for e in expr[op]])  # type: ignore[arg-type]
+    if "not" in expr:
+        return _Node("not", [_compile(expr["not"])])
+    raise ValueError(f"unknown match expression: {expr}")
+
+
+Span = tuple[int, int]
+
+
+def _eval(node: _Node, text: str) -> list[Span] | None:
+    if node.op == "leaf":
+        assert node.leaf is not None
+        spans = [m.span() for m in islice(node.leaf.pattern.finditer(text), _MAX_MATCHES)]
+        return spans if len(spans) >= node.leaf.min_count else None
+    if node.op == "not":
+        return [] if _eval(node.children[0], text) is None else None
+    results = [_eval(c, text) for c in node.children]
+    if node.op == "all":
+        if any(r is None for r in results):
+            return None
+        return [s for r in results if r for s in r]
+    hits = [r for r in results if r is not None]
+    return [s for r in hits for s in r] if hits else None
+
+
+@dataclass
+class CompiledRule:
+    spec: YamlRule
+    root: _Node
+    window: int | None
+
+    def applies(self, rel: str, lang: str | None) -> bool:
+        if self.spec.languages and "any" not in self.spec.languages and lang not in self.spec.languages:
+            return False
+        if self.spec.paths:
+            name = rel.rsplit("/", 1)[-1]
+            return any(fnmatch.fnmatchcase(rel, p) or fnmatch.fnmatchcase(name, p) for p in self.spec.paths)
+        return True
+
+    def evaluate(self, text: str, line_of: Any) -> list[Span] | None:
+        if self.window is None or self.root.op != "all":
+            return _eval(self.root, text)
+        per_child: list[list[Span]] = []
+        for child in self.root.children:
+            res = _eval(child, text)
+            if res is None:
+                return None
+            if child.op != "not":
+                per_child.append(res)
+        if not per_child:
+            return []
+        child_lines = [[(line_of(s[0]), s) for s in spans] for spans in per_child]
+        for anchor_line, anchor_span in child_lines[0]:
+            picked = [anchor_span]
+            for others in child_lines[1:]:
+                near = next((s for ln, s in others if abs(ln - anchor_line) <= self.window), None)
+                if near is None:
+                    break
+                picked.append(near)
+            else:
+                return picked
+        return None
+
+
+class ContentRuleEngine:
+    name = "content-rules"
+
+    def __init__(self, rules: list[CompiledRule]) -> None:
+        self.rules = rules
+
+    @classmethod
+    def load_builtin(cls) -> ContentRuleEngine:
+        packs: list[RulePack] = []
+        base = resources.files("devguard.rules").joinpath("content")
+        for entry in sorted(base.iterdir(), key=lambda e: e.name):
+            if entry.name.endswith((".yaml", ".yml")):
+                packs.append(RulePack.model_validate(yaml.safe_load(entry.read_text("utf-8"))))
+        return cls.from_packs(packs)
+
+    @classmethod
+    def from_packs(cls, packs: Iterable[RulePack]) -> ContentRuleEngine:
+        compiled: list[CompiledRule] = []
+        for pack in packs:
+            for r in pack.rules:
+                window = int(r.within.split(":")[1]) if r.within.startswith("lines:") else None
+                compiled.append(CompiledRule(r, _compile(r.match), window))
+                register(
+                    RuleMeta(
+                        id=r.id,
+                        title=r.title,
+                        severity=r.severity,
+                        confidence=r.confidence,
+                        category=r.category,
+                        threat_refs=tuple(r.threat),
+                        explanation=r.explanation.strip(),
+                        benign_context=r.benign_context.strip(),
+                        next_step=r.next_step.strip(),
+                        tags=tuple(r.tags),
+                        origin=pack.pack,
+                    )
+                )
+        return cls(compiled)
+
+    def applies(self, fv: FileView) -> bool:
+        return fv.text is not None
+
+    def analyze(self, fv: FileView, ctx: AnalysisContext) -> list[Finding]:
+        assert fv.text is not None
+        text = fv.text
+        limit = ctx.config.limits.max_regex_text
+        if len(text) > limit:
+            ctx.warn("content-rules", f"only the first {limit} characters were checked by content rules", fv.rel)
+            text = text[:limit]
+        findings: list[Finding] = []
+        for rule in self.rules:
+            if not rule.applies(fv.rel, fv.lang):
+                continue
+            spans = rule.evaluate(text, lambda off: fv.line_col(off)[0])
+            if spans is None:
+                continue
+            findings.append(self._finding(rule, fv, spans))
+        return findings
+
+    def match_text(self, rule_id: str, rel: str, text: str, lang: str | None) -> bool:
+        """Used by the rule self-tests: does `rule_id` fire on this text?"""
+        from devguard.core.safefs import FileRecord, Kind
+
+        rule = next(r for r in self.rules if r.spec.id == rule_id)
+        fv = FileView(FileRecord(rel, Kind.FILE, len(text), 0o644), text.encode(), text, lang, None, False)
+        if not rule.applies(rel, lang):
+            return False
+        return rule.evaluate(text, lambda off: fv.line_col(off)[0]) is not None
+
+    @staticmethod
+    def _finding(rule: CompiledRule, fv: FileView, spans: list[Span]) -> Finding:
+        seen_lines: set[int] = set()
+        evidence = []
+        for start, end in sorted(spans):
+            line = fv.line_col(start)[0]
+            if line in seen_lines:
+                continue
+            seen_lines.add(line)
+            evidence.append(fv.evidence_at_offset(start, end))
+            if len(evidence) == 3:
+                break
+        if not evidence:
+            evidence.append(fv.evidence_at_line(None))
+        return build_finding(rule.spec.id, evidence, source=Source.RULE)

devguard/analyzers/filesystem.py

@@ -0,0 +1,142 @@
+"""Inventory-level checks: symlinks, special files, deceptive names, vendored code (TC-12, TC-11.6)."""
+
+from __future__ import annotations
+
+import posixpath
+import re
+import unicodedata
+from pathlib import PurePosixPath
+
+from devguard.core.models import Evidence, Finding, Severity
+from devguard.core.redact import visible
+from devguard.core.safefs import Inventory
+from devguard.rules.catalog import build_finding, rule
+
+R_SYMLINK = rule(
+    "DG-FS-SYMLINK-ESCAPE-001",
+    "Symlink points outside the repository",
+    "high",
+    "high",
+    "filesystem",
+    "TC-12.1",
+    "A symbolic link resolves outside the project. Build steps or tools that read 'project files' through it "
+    "can read (and leak) files elsewhere on your machine.",
+    "Some monorepos link to sibling checkouts.",
+)
+R_SPECIAL = rule(
+    "DG-FS-SPECIAL-FILE-001",
+    "Special file in the repository",
+    "medium",
+    "high",
+    "filesystem",
+    "TC-12.4",
+    "The repository contains a FIFO, socket, or device file. These do not belong in source trees and can hang "
+    "or confuse tools that read them.",
+)
+R_NAME = rule(
+    "DG-FS-DECEPTIVE-NAME-001",
+    "Deceptive file or directory name",
+    "medium",
+    "high",
+    "filesystem",
+    "TC-12.2",
+    "A path contains invisible/bidi characters, trailing spaces or dots, or consists only of whitespace, "
+    "which makes it look like a different file or hides it in listings.",
+)
+R_VENDORED = rule(
+    "DG-FS-VENDORED-001",
+    "Dependency directories committed to the repository",
+    "info",
+    "high",
+    "deps",
+    "TC-11.6",
+    "Installed dependencies (node_modules, vendor, virtualenvs) are committed. They are a known place to hide "
+    "modified packages. They were inventoried but not deep-scanned (use --include-vendored); files they "
+    "contain that are referenced by a trigger were scanned anyway.",
+)
+R_EXEC_BIT = rule(
+    "DG-FS-EXEC-BIT-001",
+    "Data file marked executable",
+    "info",
+    "medium",
+    "filesystem",
+    "TC-12.6",
+    "A file with a data extension (image, font, JSON, text) has the executable permission bit set.",
+)
+
+_SENSITIVE_TARGET = re.compile(
+    r"(^|/)(\.ssh|\.aws|\.gnupg|\.kube|\.docker|\.config|\.npmrc|\.pypirc|\.netrc|\.git-credentials|"
+    r"Library/Keychains|Login Data|Cookies|wallet)|^/etc/(shadow|passwd|sudoers)|^/proc/|^/dev/|^/root\b"
+)
+_DATA_EXT = (
+    ".png",
+    ".jpg",
+    ".jpeg",
+    ".gif",
+    ".ico",
+    ".svg",
+    ".json",
+    ".md",
+    ".txt",
+    ".woff",
+    ".woff2",
+    ".ttf",
+    ".otf",
+    ".css",
+    ".csv",
+    ".lock",
+    ".yml",
+    ".yaml",
+    ".xml",
+    ".pdf",
+)
+
+
+def _deceptive(name: str) -> str | None:
+    if any(unicodedata.category(c) in ("Cf", "Cc") or 0xFE00 <= ord(c) <= 0xFE0F for c in name):
+        return "contains invisible or control characters"
+    if name != name.rstrip(" .") and name not in (".", ".."):
+        return "ends with a space or dot"
+    if not name.strip():
+        return "consists only of whitespace"
+    return None
+
+
+def filesystem_findings(inv: Inventory) -> list[Finding]:
+    out: list[Finding] = []
+    for link in inv.symlinks:
+        if link.vendored:  # virtualenvs and node_modules/.bin are full of outward symlinks
+            continue
+        target = link.symlink_target or ""
+        parent = posixpath.dirname(link.rel)
+        joined = posixpath.normpath(posixpath.join(parent, target)) if not target.startswith(("/", "~")) else target
+        escapes = target.startswith(("/", "~")) or joined.startswith("..") or re.match(r"^[A-Za-z]:[\\/]", target)
+        if escapes:
+            sev = Severity.CRITICAL if _SENSITIVE_TARGET.search(target) else None
+            out.append(
+                build_finding(R_SYMLINK.id, [Evidence(path=link.rel, excerpt=visible(f"-> {target}"))], severity=sev)
+            )
+    for sp in inv.special:
+        out.append(build_finding(R_SPECIAL.id, [Evidence(path=sp.rel)], detail=f"Type: {sp.special_type}."))
+    seen: set[str] = set()
+    for rel in [*inv.dirs, *(f.rel for f in inv.files), *(s.rel for s in inv.symlinks)]:
+        for part in PurePosixPath(rel).parts:
+            reason = _deceptive(part)
+            if reason and part not in seen:
+                seen.add(part)
+                sev = Severity.HIGH if "invisible" in reason else None
+                out.append(
+                    build_finding(
+                        R_NAME.id,
+                        [Evidence(path=visible(rel))],
+                        severity=sev,
+                        detail=f"Name {visible(part)!r} {reason}.",
+                    )
+                )
+    if inv.vendored_dirs:
+        shown = ", ".join(inv.vendored_dirs[:10]) + (" …" if len(inv.vendored_dirs) > 10 else "")
+        out.append(build_finding(R_VENDORED.id, [Evidence(path=inv.vendored_dirs[0])], detail=f"Directories: {shown}."))
+    for f in inv.files:
+        if f.executable_bit and f.rel.lower().endswith(_DATA_EXT):
+            out.append(build_finding(R_EXEC_BIT.id, [Evidence(path=f.rel)]))
+    return out

devguard/analyzers/hidden.py

@@ -0,0 +1,273 @@
+"""Hidden content: bidi controls, invisible Unicode, off-screen code, encoded blobs (TC-08)."""
+
+from __future__ import annotations
+
+import math
+import re
+from collections import Counter
+
+from devguard.analyzers.base import AnalysisContext, FileView
+from devguard.analyzers.surfaces.ai_assistants import is_ai_instruction_file
+from devguard.core.models import Evidence, Finding, Severity
+from devguard.core.redact import make_excerpt
+from devguard.rules.catalog import build_finding, rule
+
+R_BIDI = rule(
+    "DG-HID-BIDI-001",
+    "Unicode bidirectional control characters",
+    "high",
+    "high",
+    "hidden",
+    "TC-08.2",
+    "The file contains bidi override/isolate characters (Trojan Source, CVE-2021-42574). They make code "
+    "display in a different order than the compiler or interpreter reads it.",
+    "Legitimate in right-to-left natural-language text; not in code or configuration.",
+    "View the file with invisible characters shown (e.g. `cat -A`), not in a normal editor.",
+)
+R_INVISIBLE = rule(
+    "DG-HID-INVISIBLE-001",
+    "Invisible Unicode characters",
+    "medium",
+    "medium",
+    "hidden",
+    "TC-08.3",
+    "The file contains zero-width, tag, or variation-selector characters that render as nothing. They can "
+    "smuggle instructions to AI assistants or encode payloads that reviewers cannot see.",
+    "Zero-width joiners inside emoji and in some scripts (Persian, Indic) are normal.",
+)
+R_OFFSCREEN = rule(
+    "DG-HID-OFFSCREEN-001",
+    "Code hidden off-screen after a run of whitespace",
+    "high",
+    "medium",
+    "hidden",
+    "TC-08.1",
+    "A line contains a long run of spaces/tabs followed by more code, pushing that code far to the right where "
+    "editors and code review tools do not show it. Malicious 'interview' repositories hide loaders this way.",
+    "Generated files and ASCII art occasionally contain long whitespace runs.",
+    "Inspect the full line (e.g. `sed -n 'Np' file | cut -c1-2000`).",
+)
+R_BLOB = rule(
+    "DG-HID-ENCODED-BLOB-001",
+    "Large encoded data blob in source",
+    "low",
+    "medium",
+    "obfuscation",
+    "TC-08.5",
+    "The file contains a long base64/hex string or a high-entropy literal. Encoded blobs are how payloads "
+    "are embedded in otherwise readable code.",
+    "Embedded images, fonts, certificates, and test vectors are common.",
+)
+
+_BIDI = re.compile("[\u202a-\u202e\u2066-\u2069]")
+_TAGS = re.compile("[\U000e0000-\U000e007f]")
+_VS_SUPP = re.compile("[\U000e0100-\U000e01ef]")
+_VS_BASIC = re.compile("[\ufe00-\ufe0e]")  # FE0F (emoji presentation) is excluded on purpose
+_ZERO_WIDTH = re.compile("[\u200b\u200c\u200d\u2060-\u2064\u180e\u115f\u1160\u3164\uffa0\u17b4\u17b5\u034f\u00ad]")
+_WS_RUN = re.compile(r"\S([ \t]{150,})(\S)|^([ \t]{150,})(\S)")
+_B64 = re.compile(r"[A-Za-z0-9+/]{200,}={0,2}")
+_HEX = re.compile(r"\b(?:[0-9a-fA-F]{2}){128,}\b")
+_QUOTED = re.compile(r"""(["'`])([^"'`\n\\]{120,})\1""")
+_DATA_URI = re.compile(r"data:(image|font|application/font|audio|video)/[\w.+-]+;base64,")
+_EXEC_NEARBY = re.compile(r"\b(eval|exec|Function|child_process|spawn|Invoke-Expression|iex|system|popen)\b")
+
+_SKIP_SUFFIXES = (
+    ".map",
+    ".lock",
+    ".svg",
+    ".min.css",
+    ".snap",
+    ".pem",
+    ".crt",
+    ".cer",
+    ".key",
+    ".pub",
+    ".asc",
+    ".gpg",
+    ".sum",
+    ".ipynb",
+)
+_SKIP_NAMES = (
+    "package-lock.json",
+    "yarn.lock",
+    "pnpm-lock.yaml",
+    "poetry.lock",
+    "uv.lock",
+    "Cargo.lock",
+    "go.sum",
+    "composer.lock",
+    "Gemfile.lock",
+    "Pipfile.lock",
+    "bun.lock",
+)
+_PROSE_LANGS = {"markdown", "text", None}
+_CODE_LANGS = {
+    "javascript",
+    "python",
+    "shell",
+    "powershell",
+    "batch",
+    "ruby",
+    "php",
+    "perl",
+    "go",
+    "java",
+    "kotlin",
+    "csharp",
+    "rust",
+    "lua",
+    "groovy",
+    "vbscript",
+    "json",
+    "yaml",
+    "toml",
+    "ini",
+    "xml",
+    "dockerfile",
+    "make",
+    "dotenv",
+}
+
+
+def shannon_entropy(s: str) -> float:
+    if not s:
+        return 0.0
+    counts = Counter(s)
+    n = len(s)
+    return -sum(c / n * math.log2(c / n) for c in counts.values())
+
+
+class HiddenContentAnalyzer:
+    name = "hidden-content"
+
+    def applies(self, fv: FileView) -> bool:
+        return fv.text is not None and not fv.lossy
+
+    def analyze(self, fv: FileView, ctx: AnalysisContext) -> list[Finding]:
+        assert fv.text is not None
+        out: list[Finding] = []
+        text = fv.text
+        ai_file = is_ai_instruction_file(fv.rel)
+        code = fv.lang in _CODE_LANGS
+        out += self._bidi(fv, text, code, ai_file)
+        out += self._invisible(fv, text, code, ai_file)
+        if code or ai_file:
+            out += self._offscreen(fv)
+        if code and not fv.rel.endswith(_SKIP_SUFFIXES) and fv.name not in _SKIP_NAMES:
+            out += self._blobs(fv, text)
+        return out
+
+    def _bidi(self, fv: FileView, text: str, code: bool, ai_file: bool) -> list[Finding]:
+        hits = list(_BIDI.finditer(text))
+        if not hits:
+            return []
+        sev = Severity.HIGH if (code or ai_file) else Severity.LOW
+        ev = [fv.evidence_at_offset(m.start()) for m in hits[:3]]
+        return [build_finding(R_BIDI.id, ev, severity=sev, detail=f"{len(hits)} occurrence(s).")]
+
+    def _invisible(self, fv: FileView, text: str, code: bool, ai_file: bool) -> list[Finding]:
+        tags = list(_TAGS.finditer(text))
+        vs_supp = list(_VS_SUPP.finditer(text))
+        zw = [m for m in _ZERO_WIDTH.finditer(text) if not _benign_zw(text, m.start())]
+        vs_basic = [m for m in _VS_BASIC.finditer(text) if not _after_symbol(text, m.start())]
+        total = len(tags) + len(vs_supp) + len(zw) + len(vs_basic)
+        if total == 0:
+            return []
+        parts = []
+        if tags:
+            decoded = "".join(chr(ord(m.group()) - 0xE0000) for m in tags[:200])
+            parts.append(f"{len(tags)} Unicode tag characters (decoded: {make_excerpt(decoded, limit=160)!r})")
+        if vs_supp:
+            parts.append(f"{len(vs_supp)} supplementary variation selectors")
+        if vs_basic:
+            parts.append(f"{len(vs_basic)} variation selectors outside emoji")
+        if zw:
+            parts.append(f"{len(zw)} zero-width/filler characters")
+        strong = bool(tags or len(vs_supp) >= 2)
+        if strong or ai_file:
+            sev = Severity.HIGH if (code or ai_file) else Severity.MEDIUM
+        elif code or total >= 10:
+            sev = Severity.MEDIUM
+        else:
+            sev = Severity.LOW
+        first = sorted([*tags, *vs_supp, *zw, *vs_basic], key=lambda m: m.start())[:3]
+        ev = [fv.evidence_at_offset(m.start()) for m in first]
+        return [build_finding(R_INVISIBLE.id, ev, severity=sev, detail="; ".join(parts) + ".")]
+
+    def _offscreen(self, fv: FileView) -> list[Finding]:
+        out: list[Finding] = []
+        for i, line in enumerate(fv.lines, 1):
+            if len(line) < 150:
+                continue
+            m = _WS_RUN.search(line)
+            if not m:
+                continue
+            gap_start = m.start(1) if m.group(1) else m.start(3)
+            gap = m.group(1) or m.group(3)
+            hidden_col = gap_start + len(gap) + 1
+            visible_part = line[:gap_start].strip()
+            hidden_part = line[gap_start + len(gap) :]
+            excerpt = make_excerpt(
+                f"{visible_part[:100]} <\u2026{len(gap)} whitespace chars\u2026> {hidden_part[:180]}", limit=320
+            )
+            out.append(
+                build_finding(
+                    R_OFFSCREEN.id,
+                    [Evidence(path=fv.rel, line_start=i, column=hidden_col, excerpt=excerpt)],
+                    detail=f"Hidden content starts at column {hidden_col}.",
+                )
+            )
+            if len(out) >= 5:
+                break
+        return out
+
+    def _blobs(self, fv: FileView, text: str) -> list[Finding]:
+        candidates: list[tuple[int, int, str]] = []
+        for m in _B64.finditer(text):
+            prefix = text[max(0, m.start() - 60) : m.start()]
+            if _DATA_URI.search(prefix):
+                continue
+            candidates.append((m.start(), m.end(), "base64"))
+        for m in _HEX.finditer(text):
+            candidates.append((m.start(), m.end(), "hex"))
+        for m in _QUOTED.finditer(text):
+            s = m.group(2)
+            if " " not in s and shannon_entropy(s) >= 5.0:
+                candidates.append((m.start(2), m.end(2), "high-entropy"))
+        if not candidates:
+            return []
+        candidates.sort()
+        biggest = max(candidates, key=lambda c: c[1] - c[0])
+        exec_nearby = bool(_EXEC_NEARBY.search(text))
+        sev = Severity.MEDIUM if exec_nearby else Severity.LOW
+        kinds = sorted({c[2] for c in candidates})
+        ev = [fv.evidence_at_offset(biggest[0], biggest[1])]
+        return [
+            build_finding(
+                R_BLOB.id,
+                ev,
+                severity=sev,
+                detail=f"{len(candidates)} blob(s) ({', '.join(kinds)}); largest {biggest[1] - biggest[0]} chars"
+                f"{'; the file also uses dynamic execution APIs' if exec_nearby else ''}.",
+            )
+        ]
+
+
+def _benign_zw(text: str, pos: int) -> bool:
+    ch = text[pos]
+    if pos == 0 and ch == "\ufeff":
+        return True
+    prev = text[pos - 1] if pos > 0 else ""
+    nxt = text[pos + 1] if pos + 1 < len(text) else ""
+    if ch == "\u200d":  # ZWJ in emoji sequences
+        return ord(prev or " ") > 0x2000 and ord(nxt or " ") > 0x2000
+    if ch == "\u200c":  # ZWNJ in Persian/Indic text
+        return prev.isalpha() and ord(prev) > 0x0590 and (nxt.isalpha() or nxt == "")
+    if ch == "\u00ad":  # soft hyphen inside words in prose
+        return prev.isalpha() and nxt.isalpha()
+    return False
+
+
+def _after_symbol(text: str, pos: int) -> bool:
+    prev = text[pos - 1] if pos > 0 else ""
+    return bool(prev) and (ord(prev) > 0x2000 or not prev.isascii())