vb6decompiler 0.2.0__py3-none-any.whl

vb6decompiler/literals.py

@@ -0,0 +1,275 @@
+"""Text literals in a native VB6 build — including the ones every scanner misses.
+
+Why this exists
+---------------
+VB6 compiles a string literal to a **BSTR in the data area**: a 4-byte byte-length
+prefix, the text as **UTF-16LE**, then a NUL terminator. It is not an ASCII string,
+and on the reference target it does not live in ``.data`` at all — that section is
+4 KB while ``.text`` is 106 MB, and the literals sit inside ``.text``. Consequences,
+both measured on the same 111 MB file:
+
+* A generic engine that scans data sections for ASCII reported **59 strings**, all
+  of them noise from ``.rsrc``, and **0 occurrences of a table name that is present
+  260 times** in the file. Its own limitation text says "strings are read from data
+  sections", which is exactly the wrong place.
+* A decompiler-listing scanner that only matched ``push <addr> ; "text"`` missed
+  **170,645 literals** (170,555 of them were loaded with ``mov``), because native
+  VB6 loads a string address with whichever instruction the register allocator
+  chose.
+
+So this module reads two sources and keeps them apart:
+
+1. :func:`scan_binary` — BSTR runs from the image itself, with the length prefix
+   checked. That check is what makes a run a literal and not a coincidence.
+2. :func:`scan_listings` — ``<addr>h ; "text"`` comments from a decompiler's text
+   output, *any* instruction, with the VA and the procedure it belongs to.
+3. :func:`merge` — union keyed by (procedure VA, text address), never by text.
+
+Nothing here decides what a literal *means*. SQL detection is a flag, and a flag is
+not a table list: see :mod:`vb6decompiler.sqlstitch` for what can and cannot be
+closed statically.
+"""
+
+from __future__ import annotations
+
+import os
+import re
+from dataclasses import dataclass
+
+from . import pe as pe_mod
+from .mask import mask_text
+
+#: A run must be at least this long to be plausible (shorter runs are alignment
+#: noise). VB6 allows 1-character literals; this threshold is a precision/recall
+#: choice, documented rather than hidden.
+MIN_CHARS = 4
+
+#: Longer than this and it is not a literal someone typed; it is a blob.
+MAX_CHARS = 400
+
+#: Characters allowed inside a recovered literal. Turkish letters are listed because
+#: VB6 stores them as UTF-16 code units above 0x7F: a "high byte must be zero" rule
+#: silently drops ``ş``, ``ğ`` and ``İ`` and splits the literal around them.
+_ALLOWED = (
+    "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789"
+    " _.,;:!?'\"()[]{}<>+-*/=%&|\\@#$^~`\t\r\n"
+    "çÇğĞıİöÖşŞüÜâÂîÎûÛéÉàÀ°²³€£½¼¾"
+)
+_RUN = re.compile("[" + re.escape(_ALLOWED) + "]{%d,%d}" % (MIN_CHARS, MAX_CHARS))
+
+#: SQL verbs. Used as a flag only; a literal with a verb is a *candidate*.
+SQL_HINT = re.compile(
+    r"\b(SELECT\s|INSERT\s+INTO\b|INSERT\b|UPDATE\s+\w+\s+SET\b|UPDATE\b|"
+    r"DELETE\s+FROM\b|DELETE\b|EXEC\b|CREATE\s+TABLE\b|ALTER\s+TABLE\b)",
+    re.I,
+)
+
+#: Any instruction's literal comment: ``0042C3A0h ; "text"``. The instruction is
+#: deliberately not named — that was the measured trap.
+LISTING_LITERAL = re.compile(r"([0-9A-Fa-f]{6,8})h\s*;\s*\"([^\"]{1,400})\"")
+
+#: Procedure header in a decompiler listing; the trailing hex is the procedure VA.
+LISTING_PROC = re.compile(
+    r"^(?:Private|Public|Friend|Static)?\s*"
+    r"(?:Sub|Function|Property\s+(?:Get|Let|Set))\s+([A-Za-z_]\w*)?\s*\(?.*?'\s*([0-9A-Fa-f]{6,8})\s*$"
+)
+
+
+@dataclass
+class Literal:
+    va: int                          # virtual address of the text
+    text: str
+    encoding: str                    # "utf16", "cp1254" or "listing"
+    section: str | None = None
+    length_prefix_ok: bool = False   # the 4 bytes before the text equal 2*len(text)
+    proc_va: int | None = None       # only known when it came from a listing
+    source: str = "binary"           # "binary" or "listing"
+
+    @property
+    def is_sql(self) -> bool:
+        return bool(SQL_HINT.search(self.text))
+
+    def as_row(self) -> dict:
+        return {
+            "va": f"{self.va:X}",
+            "proc_va": f"{self.proc_va:X}" if self.proc_va else "",
+            "encoding": self.encoding,
+            "section": self.section or "",
+            "bstr": "yes" if self.length_prefix_ok else "no",
+            "sql": "yes" if self.is_sql else "no",
+            "source": self.source,
+            "text": mask_text(self.text),
+        }
+
+
+# ---------------------------------------------------------------------------
+# 1. the image itself
+# ---------------------------------------------------------------------------
+
+
+def scan_binary(
+    path: str,
+    min_chars: int = MIN_CHARS,
+    include_cp1254: bool = True,
+    sections: tuple[str, ...] | None = None,
+) -> list[Literal]:
+    """BSTR (UTF-16LE) literals, plus optional ASCII runs, from one binary.
+
+    Both byte alignments are decoded: VB6 does not promise that a literal starts on
+    a 2-byte boundary, and one odd-length neighbour shifts everything after it.
+    """
+    with open(path, "rb") as fh:
+        data = fh.read()
+    image = pe_mod.read(path)
+    out: list[Literal] = []
+    for sec in image.sections:
+        if sections is not None and sec.name not in sections:
+            continue
+        if sec.name.lower() in (".rsrc", ".reloc", ".pdata", ".idata"):
+            continue
+        start = sec.raw_offset
+        end = min(len(data), sec.raw_offset + sec.raw_size)
+        out.extend(_scan_utf16(data, start, end, image, sec))
+        if include_cp1254:
+            out.extend(_scan_ascii(data, start, end, image, sec))
+    return out
+
+
+def _va_of(image, section, offset: int) -> int:
+    return image.image_base + (offset - section.raw_offset) + section.virtual_address
+
+
+def _scan_utf16(data: bytes, start: int, end: int, image, section):
+    out: list[Literal] = []
+    for align in (0, 1):
+        chunk = data[start + align : end]
+        if len(chunk) % 2:
+            chunk = chunk[:-1]
+        for m in _RUN.finditer(chunk.decode("utf-16-le", "replace")):
+            byte_off = start + align + 2 * m.start()
+            prefix_ok = False
+            if byte_off >= 4:
+                declared = int.from_bytes(data[byte_off - 4 : byte_off], "little")
+                prefix_ok = declared == 2 * len(m.group(0))
+            out.append(
+                Literal(
+                    va=_va_of(image, section, byte_off),
+                    text=m.group(0),
+                    encoding="utf16",
+                    section=section.name,
+                    length_prefix_ok=prefix_ok,
+                )
+            )
+    return out
+
+
+_ASCII_RUN = re.compile(rb"[\x20-\x7e]{%d,%d}" % (MIN_CHARS, MAX_CHARS))
+
+
+def _scan_ascii(data: bytes, start: int, end: int, image, section):
+    """ASCII runs terminated by a NUL: connection strings and paths look like this.
+
+    Turkish CP1254 bytes (0x80+) are deliberately excluded here: a literal that
+    needs them is a UI message, recovered by the UTF-16 pass instead. Including
+    0x80+ in this pass turned dense code bytes into thousands of false strings.
+    """
+    out: list[Literal] = []
+    blob = data[start:end]
+    for m in _ASCII_RUN.finditer(blob):
+        if m.end() < len(blob) and blob[m.end()] != 0:
+            continue  # not NUL-terminated: inside a longer binary run
+        out.append(
+            Literal(
+                va=_va_of(image, section, start + m.start()),
+                text=m.group(0).decode("cp1254", "replace"),
+                encoding="cp1254",
+                section=section.name,
+            )
+        )
+    return out
+
+
+# ---------------------------------------------------------------------------
+# 2. decompiler listings
+# ---------------------------------------------------------------------------
+
+
+def scan_listings(directory: str) -> list[Literal]:
+    """Literals from ``.frm/.bas/.cls`` decompiler output, with procedure VAs."""
+    out: list[Literal] = []
+    for root, _dirs, files in os.walk(directory):
+        for name in sorted(files):
+            if not name.lower().endswith((".frm", ".bas", ".cls")):
+                continue
+            current: int | None = None
+            with open(os.path.join(root, name), encoding="cp1254", errors="replace") as fh:
+                for line in fh:
+                    # The procedure header is matched FIRST: it carries no ';', so a
+                    # "skip lines without a literal comment" filter placed above it
+                    # throws every header away and leaves proc_va None on every
+                    # literal (measured: 441,607 literals, 0 procedures).
+                    head = LISTING_PROC.match(line)
+                    if head:
+                        current = int(head.group(2), 16)
+                        continue
+                    if ";" not in line or '"' not in line:
+                        continue
+                    for m in LISTING_LITERAL.finditer(line):
+                        out.append(
+                            Literal(
+                                va=int(m.group(1), 16),
+                                text=m.group(2),
+                                encoding="listing",
+                                proc_va=current,
+                                source="listing",
+                            )
+                        )
+    return out
+
+
+# ---------------------------------------------------------------------------
+# 3. merge / summary
+# ---------------------------------------------------------------------------
+
+
+def merge(*groups: list[Literal]) -> list[Literal]:
+    """Union keyed by (proc_va, va) — never by text.
+
+    The same sentence at two addresses is two literals; an inventory keyed by text
+    hides the second one, and any later count built on it is wrong.
+    """
+    seen: set[tuple[int | None, int]] = set()
+    out: list[Literal] = []
+    for group in groups:
+        for lit in group:
+            key = (lit.proc_va, lit.va)
+            if key in seen:
+                continue
+            seen.add(key)
+            out.append(lit)
+    return out
+
+
+def summary(literals: list[Literal]) -> dict:
+    by_section: dict[str, int] = {}
+    for lit in literals:
+        key = lit.section or "?"
+        by_section[key] = by_section.get(key, 0) + 1
+    return {
+        "literals": len(literals),
+        "sql_candidates": sum(1 for l in literals if l.is_sql),
+        "bstr_length_checked": sum(1 for l in literals if l.length_prefix_ok),
+        "utf16": sum(1 for l in literals if l.encoding == "utf16"),
+        "cp1254": sum(1 for l in literals if l.encoding == "cp1254"),
+        "from_listing": sum(1 for l in literals if l.source == "listing"),
+        "sections": by_section,
+    }
+
+
+def find(path_or_dir: str, needle: str, case_sensitive: bool = False) -> list[Literal]:
+    """Substring search across either source."""
+    pool = scan_listings(path_or_dir) if os.path.isdir(path_or_dir) else scan_binary(path_or_dir)
+    if case_sensitive:
+        return [l for l in pool if needle in l.text]
+    low = needle.lower()
+    return [l for l in pool if low in l.text.lower()]

vb6decompiler/mask.py

@@ -0,0 +1,192 @@
+"""Hide credentials before anything reaches disk.
+
+Why this exists
+---------------
+A recovered VB6 build contains **plain-text connection strings with ``uid=`` and
+``pwd=`` values** (250 of them in the reference target, plus server names and UNC
+shares). Any artefact written from that data — CSV, Markdown, JSON, a commit —
+leaks the credentials of a live SQL Server. Every writer in this package runs its
+text through :func:`mask_text`; :func:`audit` is the gate that proves nothing else
+did.
+
+What is masked
+--------------
+* ``password=`` / ``pwd=`` values      -> ``[REDACTED]``
+* ``user id=`` / ``uid=`` values       -> ``[REDACTED]``
+* ``server=`` / ``data source=`` hosts -> ``[HOST]``
+* ``\\\\host\\share`` UNC prefixes        -> ``\\\\[HOST]\\``
+* IPv4 literals                        -> ``[IP]``
+
+What is *not* masked, on purpose: database names. They are evidence about the
+program's data model, not a credential, and masking them would destroy the very
+information the toolkit exists to produce.
+
+The measured trap: a masking regex written with ``\\s`` crosses newlines. When a
+literal ends with ``Server=``, ``\\s`` swallows the line break and "masks" the first
+word of the next line — corrupting the file and producing a false count. All
+whitespace classes here are ``[ \\t]``, never ``\\s``, and the file-level pass counts
+lines before and after and refuses to write when they differ.
+"""
+
+from __future__ import annotations
+
+import os
+import re
+from dataclasses import dataclass
+
+REDACTED = "[REDACTED]"
+HOST = "[HOST]"
+IP = "[IP]"
+
+#: Every pattern is line-bounded: ``[ \t]`` and ``[^;\s"']``-style ranges that can
+#: never cross a newline, and the value stops at the next separator.
+def _keep_key(match: "re.Match[str]") -> str:
+    """Keep the key and separator as written, replace only the value.
+
+    A replacement *string* cannot carry a literal backslash: ``re`` reads ``\\``
+    and ``\\1`` as escapes, and a UNC replacement raised "bad escape". A callable
+    has no such problem, so every pattern below uses one.
+    """
+    return match.group(1) + match.group(2) + REDACTED
+
+
+def _keep_host_key(match: "re.Match[str]") -> str:
+    return match.group(1) + match.group(2) + HOST
+
+
+def _unc(match: "re.Match[str]") -> str:
+    return "\\\\" + HOST + "\\"
+
+
+#: RFC 5737 documentation ranges.  Docs and tests use them on purpose as the
+#: stand-in for a real address, so they are not leaks and must survive masking —
+#: a gate that redacts its own examples teaches people to disable the gate.
+DOC_NETS = ("192.0.2.", "198.51.100.", "203.0.113.")
+_LIVE_IP = re.compile(r"\b(?!(?:192\.0\.2|198\.51\.100|203\.0\.113)\.)(?:\d{1,3}\.){3}\d{1,3}\b")
+
+_PATTERNS: list[tuple[re.Pattern[str], object]] = [
+    (re.compile(r"(?i)\b(password|pwd)([ \t]*=[ \t]*)([^;\"'\r\n \t`]+)"), _keep_key),
+    (re.compile(r"(?i)\b(user[ \t]+id|uid)([ \t]*=[ \t]*)([^;\"'\r\n \t`]+)"), _keep_key),
+    (re.compile(r"(?i)\b(server|data[ \t]+source)([ \t]*=[ \t]*)([^;,\"'\r\n \t`]+)"), _keep_host_key),
+    (re.compile(r"\\\\[A-Za-z0-9._-]+\\"), _unc),
+    (_LIVE_IP, IP),
+]
+
+#: Marker that means a value was already masked; re-masking it produces nested
+#: ``[REDACTED]`` inside ``[REDACTED]`` and then a false "still leaking" alarm.
+_ALREADY = re.compile(r"\[(REDACTED|HOST|IP)\]")
+
+
+def mask_text(text: str) -> str:
+    """Mask one string. Idempotent: masked values stay as they are."""
+    if _ALREADY.search(text):
+        # Still run the patterns: a string can carry one masked and one live value.
+        pass
+    out = text
+    for pattern, repl in _PATTERNS:
+        out = pattern.sub(repl, out)
+    return out
+
+
+@dataclass
+class AuditResult:
+    files: int
+    lines: int
+    findings: dict[str, int]
+
+    @property
+    def clean(self) -> bool:
+        return not any(self.findings.values())
+
+
+def audit(path: str, extensions: tuple[str, ...] = (".csv", ".md", ".json", ".txt", ".sql", ".log")) -> AuditResult:
+    """Count unmasked credentials in the text files under ``path``.
+
+    Only *values* count. A file that mentions the word ``uid=`` with nothing after
+    it is a connection-string fragment, not a leak — counting keywords produced
+    211 false positives against 92 real ones in the reference corpus.
+    """
+    findings = {"password": 0, "uid": 0, "host": 0, "unc": 0, "ip": 0}
+    files = 0
+    lines = 0
+    targets = [path] if os.path.isfile(path) else []
+    if not targets:
+        for root, _dirs, names in os.walk(path):
+            for n in names:
+                if n.lower().endswith(extensions):
+                    targets.append(os.path.join(root, n))
+    # The audit uses the same token rule as the masker: a value must have at least one
+    # character that is not a separator or a markdown backtick. Otherwise prose such as
+    # "`uid=`/`pwd=` values" is counted as two leaks, and a gate that cries wolf on
+    # documentation gets ignored — which is how a real leak survives.
+    live = [
+        (re.compile(r"(?i)\b(password|pwd)[ \t]*=[ \t]*[^;\"'\r\n \t`]"), "password"),
+        (re.compile(r"(?i)\b(user[ \t]+id|uid)[ \t]*=[ \t]*[^;\"'\r\n \t`]"), "uid"),
+        (re.compile(r"(?i)\b(server|data[ \t]+source)[ \t]*=[ \t]*[^;,\"'\r\n \t`]"), "host"),
+        (re.compile(r"\\\\[A-Za-z0-9._-]+\\"), "unc"),
+        (_LIVE_IP, "ip"),
+    ]
+    for f in targets:
+        files += 1
+        with open(f, encoding=_codec_for(f), errors="replace") as fh:
+            for line in fh:
+                lines += 1
+                for pattern, key in live:
+                    if pattern.search(line) and not _ALREADY.search(line):
+                        findings[key] += 1
+    return AuditResult(files=files, lines=lines, findings=findings)
+
+
+def _codec_for(path: str) -> str:
+    """BOM-aware codec pick. A UTF-16 artefact (``command > out.txt`` in
+    PowerShell) read as UTF-8 with ``replace`` turns ``uid=`` into ``u?i?d?``:
+    every pattern misses it and the gate passes a live leak.
+    shortcut: detection is BOM-only; a BOM-less UTF-16 file is still blind to
+    the gate — upgrade if a corpus proves that case is real.
+    """
+    with open(path, "rb") as fh:
+        head = fh.read(2)
+    if head in (b"\xff\xfe", b"\xfe\xff"):
+        return "utf-16"      # the codec strips the BOM on read and rewrites it on write
+    if head == b"\xef\xbb":  # EF BB BF prefix
+        return "utf-8-sig"   # strips on read, rewrites on write
+    return "utf-8"
+
+
+def mask_file(path: str, dry_run: bool = False) -> int:
+    """Mask one file in place. Returns the number of changed lines.
+
+    The line count is compared before and after: a whitespace class that crosses
+    newlines would silently join lines, and this function refuses to write in that
+    case. Decoding is BOM-aware, and undecodable bytes round-trip via
+    surrogateescape: a cp1254 artefact leaves every byte outside the masked
+    values unchanged.
+    """
+    codec = _codec_for(path)
+    with open(path, encoding=codec, errors="surrogateescape", newline="") as fh:
+        original = fh.read()
+    lines = original.splitlines(keepends=True)
+    changed = [mask_text(line) for line in lines]
+    n_changed = sum(1 for a, b in zip(lines, changed) if a != b)
+    if len(changed) != len(lines):
+        raise RuntimeError(
+            f"{path}: masking changed the line count ({len(lines)} -> {len(changed)}); not writing"
+        )
+    if not dry_run and n_changed:
+        with open(path, "w", encoding=codec, errors="surrogateescape", newline="") as fh:
+            fh.write("".join(changed))
+    return n_changed
+
+
+def mask_tree(path: str, extensions: tuple[str, ...] | None = None, dry_run: bool = False) -> dict:
+    """Mask every text file under a directory; returns per-file changed-line counts."""
+    exts = extensions or (".csv", ".md", ".json", ".txt", ".sql", ".log")
+    out: dict[str, int] = {}
+    for root, _dirs, names in os.walk(path):
+        for n in names:
+            if n.lower().endswith(exts):
+                p = os.path.join(root, n)
+                # A symlink would let 'mask my artefacts dir' rewrite a file that
+                # lives outside it; the audit may still read through it (stricter).
+                out[p] = 0 if os.path.islink(p) else mask_file(p, dry_run=dry_run)
+    return out