wherewent 0.1.0__py3-none-any.whl

wherewent/__init__.py

@@ -0,0 +1,3 @@
+"""wherewent — a zero-config SQL flight recorder for SQLAlchemy 2.x batch jobs."""
+
+__version__ = "0.1.0"

wherewent/_shim/__init__.py

@@ -0,0 +1,2 @@
+"""Shim package. Its directory is placed on the child's PYTHONPATH so Python
+auto-imports the adjacent sitecustomize.py at interpreter startup."""

wherewent/_shim/sitecustomize.py

@@ -0,0 +1,18 @@
+"""Auto-imported by Python at startup (this dir is on the child's PYTHONPATH).
+
+Only acts when the CLI marked the environment. Any failure is swallowed so the
+job runs exactly as it would without wherewent.
+
+TODO: chaining the site's original sitecustomize is out of scope for this
+prototype (if the environment already shipped one, it is shadowed here).
+"""
+
+import os
+
+if os.environ.get("WHEREWENT_ACTIVE") == "1":
+    try:
+        from wherewent.recorder import install_from_env
+
+        install_from_env()
+    except Exception:
+        pass

wherewent/callsite.py

@@ -0,0 +1,99 @@
+"""Call-site resolution — find the user-code frame that issued a query.
+
+This runs on the hot path (once per execution), so is_library_file is memoized
+and the full stack capture is reserved for the first few samples of each group.
+"""
+
+import os
+import sys
+import sysconfig
+
+# memoization cache for is_library_file (keyed by raw filename string)
+_LIB_CACHE: "dict[str, bool]" = {}
+
+# directory of the installed wherewent package itself (its frames are "library")
+_WHEREWENT_DIR = os.path.dirname(os.path.abspath(__file__))
+
+# stdlib / install-tree prefixes computed once
+_STDLIB_PREFIXES = set()
+for _key in ("stdlib", "platstdlib", "purelib", "platlib"):
+    try:
+        _p = sysconfig.get_paths().get(_key)
+        if _p:
+            _STDLIB_PREFIXES.add(os.path.abspath(_p))
+    except Exception:
+        pass
+for _p in (getattr(sys, "prefix", None), getattr(sys, "base_prefix", None)):
+    if _p:
+        _STDLIB_PREFIXES.add(os.path.abspath(_p))
+_STDLIB_PREFIXES = tuple(_STDLIB_PREFIXES)
+
+
+def _compute_is_library(filename: str) -> bool:
+    # frozen / built-in / <string> / <stdin> pseudo-files are never user code
+    if not filename or filename.startswith("<"):
+        return True
+    path = os.path.abspath(filename)
+    sep = os.sep
+    if "site-packages" in path or "dist-packages" in path:
+        return True
+    if sep + "sqlalchemy" + sep in path:
+        return True
+    if path.startswith(_WHEREWENT_DIR):
+        return True
+    for prefix in _STDLIB_PREFIXES:
+        if path.startswith(prefix):
+            return True
+    return False
+
+
+def is_library_file(filename: str) -> bool:
+    """True if *filename* belongs to a library/stdlib/wherewent (memoized)."""
+    cached = _LIB_CACHE.get(filename)
+    if cached is not None:
+        return cached
+    result = _compute_is_library(filename)
+    _LIB_CACHE[filename] = result
+    return result
+
+
+def _short(filename: str) -> str:
+    """A compact display name: path relative to cwd, else basename."""
+    try:
+        rel = os.path.relpath(filename, os.getcwd())
+        if not rel.startswith(".."):
+            return rel
+    except Exception:
+        pass
+    return os.path.basename(filename)
+
+
+def resolve_call_site(skip: int = 1) -> "tuple[str, int, str] | None":
+    """Walk up the stack and return the first user-code frame.
+
+    Returns (short_file, lineno, function) or None if only library frames exist.
+    """
+    try:
+        frame = sys._getframe(skip)
+    except ValueError:
+        return None
+    while frame is not None:
+        filename = frame.f_code.co_filename
+        if not is_library_file(filename):
+            return (_short(filename), frame.f_lineno, frame.f_code.co_name)
+        frame = frame.f_back
+    return None
+
+
+def capture_stack(limit: int = 30) -> "list[str]":
+    """Return up to *limit* frames (user + library) as 'file:line in func'."""
+    frames = []
+    try:
+        frame = sys._getframe(1)
+    except ValueError:
+        return frames
+    while frame is not None and len(frames) < limit:
+        code = frame.f_code
+        frames.append(f"{_short(code.co_filename)}:{frame.f_lineno} in {code.co_name}")
+        frame = frame.f_back
+    return frames

wherewent/cli.py

@@ -0,0 +1,90 @@
+"""Console entry point: `wherewent run [--save PATH] <command...>`.
+
+Launches the target command in a subprocess whose PYTHONPATH is arranged so
+Python auto-imports our sitecustomize shim, which installs the recorder before
+the job's own code runs. Zero changes to the job itself.
+"""
+
+import os
+import signal
+import subprocess
+import sys
+
+USAGE = (
+    "usage: wherewent run [--save PATH] <command...>\n"
+    "  e.g. wherewent run python job.py\n"
+    "       wherewent run --save out.json python -m mypkg\n"
+    "       wherewent run job.py            (bare script uses this interpreter)\n"
+)
+
+
+def _usage():
+    sys.stderr.write(USAGE)
+
+
+def _child_env(save):
+    import wherewent
+    import wherewent._shim as shim
+
+    # Directory that literally contains sitecustomize.py (so Python auto-imports it).
+    shim_dir = os.path.dirname(os.path.abspath(shim.__file__))
+    # Directory that contains the `wherewent` package, so the child can import it.
+    pkg_parent = os.path.dirname(os.path.dirname(os.path.abspath(wherewent.__file__)))
+
+    env = os.environ.copy()
+    existing = env.get("PYTHONPATH", "")
+    parts = [shim_dir, pkg_parent]
+    if existing:
+        parts.append(existing)
+    env["PYTHONPATH"] = os.pathsep.join(parts)
+    env["WHEREWENT_ACTIVE"] = "1"
+    env["WHEREWENT_SAVE"] = save or ""
+    return env
+
+
+def main(argv=None):
+    argv = list(sys.argv[1:] if argv is None else argv)
+    if not argv or argv[0] != "run":
+        _usage()
+        return 2
+
+    args = argv[1:]
+    save = None
+    i = 0
+    while i < len(args):
+        a = args[i]
+        if a == "--save":
+            if i + 1 >= len(args):
+                _usage()
+                return 2
+            save = args[i + 1]
+            i += 2
+            continue
+        break  # first non-option token starts the command
+    command = args[i:]
+
+    if not command:
+        _usage()
+        return 2
+
+    # A bare `script.py` runs under this interpreter.
+    if command[0].endswith(".py"):
+        command = [sys.executable] + command
+
+    env = _child_env(save)
+
+    # While the child runs, the parent ignores Ctrl-C so the SIGINT goes to the
+    # child (which finalizes and prints the report); we exit with its code.
+    prev = signal.signal(signal.SIGINT, signal.SIG_IGN)
+    try:
+        proc = subprocess.run(command, env=env)
+    except FileNotFoundError:
+        sys.stderr.write(f"wherewent: command not found: {command[0]}\n")
+        return 127
+    finally:
+        signal.signal(signal.SIGINT, prev)
+    return proc.returncode
+
+
+if __name__ == "__main__":
+    sys.exit(main())

wherewent/normalize.py

@@ -0,0 +1,55 @@
+"""SQL normalization — turn a raw statement into a literal-free group signature.
+
+We NEVER keep parameter values. The output is a shape string used only to group
+structurally-identical queries together.
+"""
+
+import hashlib
+import re
+
+# --- pre-compiled patterns (applied in order) ---------------------------------
+
+# 1. comments: -- to end of line, and /* ... */ blocks (dotall for multiline)
+_LINE_COMMENT = re.compile(r"--[^\n]*")
+_BLOCK_COMMENT = re.compile(r"/\*.*?\*/", re.DOTALL)
+
+# 2. single-quoted string literals, honoring the '' escape sequence
+_STRING = re.compile(r"'(?:[^']|'')*'")
+
+# 3. bind placeholders: %s, %(name)s, :name, $1-style (done before numbers so
+#    that $1 is treated as a placeholder rather than the number 1)
+_PLACEHOLDER = re.compile(r"%\(\w+\)s|%s|\$\d+|(?<![:\w]):\w+")
+
+# 4. numeric literals: standalone numbers only, never the trailing digits of an
+#    identifier like col1 (lookbehind rejects a preceding word char or dot)
+_NUMBER = re.compile(r"(?<![\w.])\d+(?:\.\d+)?(?:[eE][+-]?\d+)?")
+
+# 5. whitespace runs
+_WS = re.compile(r"\s+")
+
+# 6. IN (?, ?, ?)  ->  IN (?)
+_IN_LIST = re.compile(r"(?i)\bIN\s*\(\s*\?(?:\s*,\s*\?)*\s*\)")
+
+# 7. multi-row VALUES (?, ?), (?, ?), ...  ->  VALUES (?, ?)  (keep first tuple)
+_MULTI_VALUES = re.compile(r"(?i)(\bVALUES\s*)(\([^()]*\))(?:\s*,\s*\([^()]*\))+")
+
+
+def normalize_sql(sql: str) -> str:
+    """Return a literal-free, whitespace-collapsed signature for *sql*."""
+    if not sql:
+        return ""
+    s = sql
+    s = _LINE_COMMENT.sub(" ", s)
+    s = _BLOCK_COMMENT.sub(" ", s)
+    s = _STRING.sub("?", s)
+    s = _PLACEHOLDER.sub("?", s)
+    s = _NUMBER.sub("?", s)
+    s = _WS.sub(" ", s).strip()
+    s = _IN_LIST.sub("IN (?)", s)
+    s = _MULTI_VALUES.sub(r"\1\2", s)
+    return s
+
+
+def group_key(normalized: str) -> str:
+    """Stable 12-hex-char group id derived from a normalized statement."""
+    return hashlib.sha1(normalized.encode()).hexdigest()[:12]

wherewent/recorder.py

@@ -0,0 +1,383 @@
+"""The recorder: class-level SQLAlchemy event hooks + finalize/report.
+
+Design invariants (see build contract):
+  * Never crash the host job: every hook body is wrapped in try/except.
+  * Never store parameter VALUES — only counts, durations, and normalized SQL.
+  * Cheap hot path: memoized library detection, bounded normalization cache,
+    full stack capture only for the first 5 samples of each group.
+  * finalize() prints exactly once, to stderr, via atexit AND a signal handler.
+"""
+
+import atexit
+import json
+import os
+import signal
+import statistics
+import sys
+from time import perf_counter
+
+from . import report, rules
+from .callsite import capture_stack, resolve_call_site
+from .normalize import group_key, normalize_sql
+from .stats import GroupSnapshot, RunSnapshot
+
+_NORM_CACHE_MAX = 4096
+
+
+class Recorder:
+    def __init__(self):
+        self.installed = False
+        self.finalized = False
+        self.save_path = None
+
+        self.start_perf = None
+        self._start_times = None
+
+        # per-group accumulators keyed by group_key
+        self.groups = {}
+        # normalization cache keyed by raw SQL string (bounded)
+        self._norm_cache = {}
+        # per-execution start times keyed by id(context)
+        self._exec_start = {}
+        # per-connection transaction generation (kept for --save completeness)
+        self._txn_gen = {}
+
+        self.total_queries = 0
+        self.total_commits = 0
+        self.total_rollbacks = 0
+        self.total_rows = 0
+        self.db_time = 0.0
+        self.overhead_time = 0.0
+
+        self.commit_time = 0.0
+        self.commit_measurable = False   # only True once a dialect is wrapped
+
+        self.sqlalchemy_active = False
+        self._prev_handlers = {}
+
+    # -- installation ----------------------------------------------------------
+
+    def install(self, save_path=None):
+        if self.installed:
+            return
+        self.installed = True
+        self.save_path = save_path or None
+        self.start_perf = perf_counter()
+        try:
+            self._start_times = os.times()
+        except Exception:
+            self._start_times = None
+
+        try:
+            import sqlalchemy  # noqa: F401
+            from sqlalchemy import event
+            from sqlalchemy.engine import Engine
+
+            # class-level: every engine in the process is captured, zero config
+            event.listen(Engine, "before_cursor_execute", self._before, named=True)
+            event.listen(Engine, "after_cursor_execute", self._after, named=True)
+            event.listen(Engine, "begin", self._on_begin)
+            event.listen(Engine, "commit", self._on_commit)
+            event.listen(Engine, "rollback", self._on_rollback)
+            event.listen(Engine, "engine_connect", self._on_connect)
+            self.sqlalchemy_active = True
+        except Exception:
+            self.sqlalchemy_active = False
+
+        atexit.register(self.finalize)
+        self._install_signal_handlers()
+
+    def _install_signal_handlers(self):
+        for sig in (signal.SIGINT, signal.SIGTERM):
+            try:
+                self._prev_handlers[sig] = signal.getsignal(sig)
+                signal.signal(sig, self._handle_signal)
+            except Exception:
+                pass
+
+    def _handle_signal(self, signum, frame):
+        # Ctrl-C mid-job MUST still produce the report.
+        self.finalize()
+        try:
+            signal.signal(signum, signal.SIG_DFL)
+        except Exception:
+            pass
+        sys.exit(130 if signum == signal.SIGINT else 143)
+
+    # -- statement hooks -------------------------------------------------------
+
+    def _before(self, **kw):
+        t0 = perf_counter()
+        try:
+            ctx = kw.get("context")
+            if ctx is not None:
+                self._exec_start[id(ctx)] = perf_counter()
+        except Exception:
+            pass
+        finally:
+            self.overhead_time += perf_counter() - t0
+
+    def _after(self, **kw):
+        t0 = perf_counter()
+        try:
+            ctx = kw.get("context")
+            statement = kw.get("statement") or ""
+            cursor = kw.get("cursor")
+            executemany = bool(kw.get("executemany"))
+
+            start = self._exec_start.pop(id(ctx), None) if ctx is not None else None
+            duration = (perf_counter() - start) if start is not None else 0.0
+
+            self.sqlalchemy_active = True
+            self.total_queries += 1
+            self.db_time += duration
+
+            key, normalized = self._normalize_cached(statement)
+            g = self.groups.get(key)
+            if g is None:
+                g = {
+                    "normalized_sql": normalized,
+                    "calls": 0,
+                    "durations": [],
+                    "total_time": 0.0,
+                    "rows": 0,
+                    "executemany_calls": 0,
+                    "call_sites": {},        # (file,line,func) -> count
+                    "sample_stacks": [],
+                }
+                self.groups[key] = g
+
+            g["calls"] += 1
+            g["durations"].append(duration)
+            g["total_time"] += duration
+            if executemany:
+                g["executemany_calls"] += 1
+
+            try:
+                rc = cursor.rowcount if cursor is not None else None
+                if rc is not None and rc >= 0:
+                    g["rows"] += rc
+                    self.total_rows += rc
+            except Exception:
+                pass
+
+            site = resolve_call_site(skip=1)
+            if site is not None:
+                g["call_sites"][site] = g["call_sites"].get(site, 0) + 1
+
+            # full stack capture only for the first 5 executions per group
+            if g["calls"] <= 5:
+                g["sample_stacks"].append(capture_stack())
+        except Exception:
+            pass
+        finally:
+            self.overhead_time += perf_counter() - t0
+
+    # -- transaction hooks -----------------------------------------------------
+
+    def _on_begin(self, conn):
+        t0 = perf_counter()
+        try:
+            self._txn_gen[id(conn)] = self._txn_gen.get(id(conn), 0) + 1
+        except Exception:
+            pass
+        finally:
+            self.overhead_time += perf_counter() - t0
+
+    def _on_commit(self, conn):
+        t0 = perf_counter()
+        try:
+            self.total_commits += 1
+            self._txn_gen[id(conn)] = self._txn_gen.get(id(conn), 0) + 1
+        except Exception:
+            pass
+        finally:
+            self.overhead_time += perf_counter() - t0
+
+    def _on_rollback(self, conn):
+        t0 = perf_counter()
+        try:
+            self.total_rollbacks += 1
+            self._txn_gen[id(conn)] = self._txn_gen.get(id(conn), 0) + 1
+        except Exception:
+            pass
+        finally:
+            self.overhead_time += perf_counter() - t0
+
+    def _on_connect(self, conn, *args):
+        # engine_connect fires with the Connection positionally (2.0: (conn);
+        # older: (conn, branch)). There is no after-commit event, so wrap the
+        # dialect's do_commit / do_rollback with a perf_counter timer here.
+        # Idempotent per dialect.
+        t0 = perf_counter()
+        try:
+            dialect = getattr(conn, "dialect", None)
+            if dialect is not None and not getattr(dialect, "_wherewent_wrapped", False):
+                self._wrap_dialect(dialect)
+        except Exception:
+            pass
+        finally:
+            self.overhead_time += perf_counter() - t0
+
+    def _wrap_dialect(self, dialect):
+        rec = self
+        orig_commit = dialect.do_commit
+        orig_rollback = dialect.do_rollback
+
+        def timed_commit(dbapi_connection, _orig=orig_commit):
+            t = perf_counter()
+            try:
+                return _orig(dbapi_connection)
+            finally:
+                try:
+                    rec.commit_time += perf_counter() - t
+                except Exception:
+                    pass
+
+        def timed_rollback(dbapi_connection, _orig=orig_rollback):
+            t = perf_counter()
+            try:
+                return _orig(dbapi_connection)
+            finally:
+                try:
+                    rec.commit_time += perf_counter() - t
+                except Exception:
+                    pass
+
+        dialect.do_commit = timed_commit
+        dialect.do_rollback = timed_rollback
+        dialect._wherewent_wrapped = True
+        self.commit_measurable = True
+
+    # -- normalization cache ---------------------------------------------------
+
+    def _normalize_cached(self, statement):
+        cached = self._norm_cache.get(statement)
+        if cached is not None:
+            return cached
+        normalized = normalize_sql(statement)
+        key = group_key(normalized)
+        if len(self._norm_cache) >= _NORM_CACHE_MAX:
+            self._norm_cache.clear()   # bounded: on overflow just drop everything
+        self._norm_cache[statement] = (key, normalized)
+        return key, normalized
+
+    # -- snapshot / finalize ---------------------------------------------------
+
+    def snapshot(self) -> RunSnapshot:
+        wall = (perf_counter() - self.start_perf) if self.start_perf else 0.0
+
+        cpu = None
+        if self._start_times is not None:
+            try:
+                now = os.times()
+                cpu = (now.user - self._start_times.user) + (
+                    now.system - self._start_times.system
+                )
+            except Exception:
+                cpu = None
+
+        groups = []
+        for key, g in self.groups.items():
+            durations = g["durations"]
+            median = statistics.median(durations) if durations else 0.0
+            site = None
+            if g["call_sites"]:
+                site = max(g["call_sites"].items(), key=lambda kv: kv[1])[0]
+            groups.append(
+                GroupSnapshot(
+                    key=key,
+                    normalized_sql=g["normalized_sql"],
+                    calls=g["calls"],
+                    total_time=g["total_time"],
+                    median=median,
+                    rows=g["rows"],
+                    executemany_calls=g["executemany_calls"],
+                    call_site=site,
+                )
+            )
+
+        commit_time = self.commit_time if self.commit_measurable else None
+
+        return RunSnapshot(
+            wall_time=wall,
+            cpu_time=cpu,
+            total_queries=self.total_queries,
+            total_commits=self.total_commits,
+            total_rollbacks=self.total_rollbacks,
+            commit_time=commit_time,
+            total_rows=self.total_rows,
+            db_time=self.db_time,
+            overhead_time=self.overhead_time,
+            sqlalchemy_active=self.sqlalchemy_active,
+            groups=groups,
+        )
+
+    def finalize(self):
+        if self.finalized:
+            return
+        self.finalized = True
+        try:
+            run = self.snapshot()
+            findings = rules.evaluate(run)
+            text = report.render(run, findings)
+            sys.stderr.write(text + "\n")
+            sys.stderr.flush()
+            if self.save_path:
+                self._write_json(run, findings)
+        except Exception:
+            pass
+
+    def _write_json(self, run: RunSnapshot, findings):
+        # groups in the snapshot are sorted for the report; for JSON we walk the
+        # raw accumulators so we can attach the captured sample stacks too.
+        group_by_key = {g.key: g for g in run.groups}
+        groups_json = []
+        for key, gs in group_by_key.items():
+            raw = self.groups.get(key, {})
+            groups_json.append(
+                {
+                    "key": gs.key,
+                    "normalized_sql": gs.normalized_sql,
+                    "calls": gs.calls,
+                    "total_time": gs.total_time,
+                    "median": gs.median,
+                    "rows": gs.rows,
+                    "executemany_calls": gs.executemany_calls,
+                    "call_site": list(gs.call_site) if gs.call_site else None,
+                    "sample_stacks": raw.get("sample_stacks", []),
+                }
+            )
+
+        payload = {
+            "wall_time": run.wall_time,
+            "cpu_time": run.cpu_time,
+            "total_queries": run.total_queries,
+            "total_commits": run.total_commits,
+            "total_rollbacks": run.total_rollbacks,
+            "commit_time": run.commit_time,
+            "total_rows": run.total_rows,
+            "db_time": run.db_time,
+            "overhead_time": run.overhead_time,
+            "sqlalchemy_active": run.sqlalchemy_active,
+            "groups": groups_json,
+            "findings": [
+                {"rule": f.rule, "title": f.title, "detail": f.detail, "seconds": f.seconds}
+                for f in findings
+            ],
+        }
+        try:
+            with open(self.save_path, "w") as fh:
+                json.dump(payload, fh, indent=2)
+        except Exception:
+            pass
+
+
+# module-level singleton
+recorder = Recorder()
+
+
+def install_from_env() -> None:
+    """Entry point called by the shim: install using WHEREWENT_SAVE from env."""
+    save = os.environ.get("WHEREWENT_SAVE") or None
+    recorder.install(save)

wherewent/report.py

@@ -0,0 +1,81 @@
+"""Plain-text report renderer (~100 columns, stderr-friendly).
+
+Golden rule: never print a guessed number. Anything unmeasurable prints as "—".
+"""
+
+from .stats import Finding, RunSnapshot
+
+_WIDTH = 100
+_SQL_COL = 48
+
+
+def _pct(part: float, whole: float) -> str:
+    return f"{part / whole * 100:.1f}%" if whole > 0 else "—"
+
+
+def render(run: RunSnapshot, findings: "list[Finding]") -> str:
+    wall = run.wall_time
+    lines = []
+    lines.append("=" * _WIDTH)
+    lines.append("wherewent — SQL flight recorder")
+    lines.append("-" * _WIDTH)
+
+    if run.cpu_time is not None and wall > 0:
+        cpu_str = f"{run.cpu_time:.2f}s ({run.cpu_time / wall * 100:.0f}% CPU busy)"
+    else:
+        cpu_str = "—"
+    lines.append(
+        f"wall: {wall:.2f}s   cpu: {cpu_str}   "
+        f"queries: {run.total_queries:,}   commits: {run.total_commits:,}   "
+        f"rollbacks: {run.total_rollbacks:,}"
+    )
+    lines.append(
+        f"in-DB time: {run.db_time:.2f}s ({_pct(run.db_time, wall)} of wall; "
+        f"app-observed: includes network+driver+server)"
+    )
+    commit_str = f"{run.commit_time:.2f}s" if run.commit_time is not None else "—"
+    lines.append(f"commit time: {commit_str}   total rows: {run.total_rows:,}")
+    lines.append(
+        f"recording added ~{run.overhead_time:.2f}s (~{_pct(run.overhead_time, wall)} of wall)"
+    )
+    lines.append("=" * _WIDTH)
+
+    if not run.sqlalchemy_active:
+        lines.append("SQLAlchemy was not imported by this process — nothing recorded.")
+        lines.append("=" * _WIDTH)
+        return "\n".join(lines)
+
+    # --- query group table (top 15 by total app time) -------------------------
+    groups = sorted(run.groups, key=lambda g: g.total_time, reverse=True)[:15]
+    lines.append(
+        f"{'QUERY GROUP':<{_SQL_COL}} {'CALLS':>8} {'TOTAL':>9} {'MEDIAN':>10}  CALL SITE"
+    )
+    lines.append("-" * _WIDTH)
+    if not groups:
+        lines.append("(no queries recorded)")
+    for g in groups:
+        sql = g.normalized_sql
+        if len(sql) > _SQL_COL:
+            sql = sql[: _SQL_COL - 3] + "..."
+        if g.call_site:
+            site = f"{g.call_site[0]}:{g.call_site[1]} in {g.call_site[2]}"
+        else:
+            site = "—"
+        tot = f"{g.total_time:.2f}s"
+        med = f"{g.median * 1000:.2f}ms"
+        lines.append(f"{sql:<{_SQL_COL}} {g.calls:>8,} {tot:>9} {med:>10}  {site}")
+
+    # --- findings -------------------------------------------------------------
+    lines.append("=" * _WIDTH)
+    lines.append("FINDINGS")
+    lines.append("-" * _WIDTH)
+    if not findings:
+        lines.append("No findings fired (thresholds: see README).")
+    else:
+        for i, f in enumerate(findings, 1):
+            lines.append(f"{i}. [{f.rule}] {f.title}")
+            for dl in f.detail.split("\n"):
+                lines.append(f"   {dl}")
+            lines.append(f"   ~= {f.seconds:.1f}s attributable")
+    lines.append("=" * _WIDTH)
+    return "\n".join(lines)

wherewent/rules.py

@@ -0,0 +1,119 @@
+"""Deterministic heuristics that turn a RunSnapshot into human findings.
+
+Every threshold and merge rule here is part of the build contract. Numbers must
+stay honest: a rule only fires when its inputs are actually measurable.
+"""
+
+from .stats import Finding, RunSnapshot
+
+
+def _site_str(group) -> str:
+    if group.call_site:
+        f, ln, _fn = group.call_site
+        return f"{f}:{ln}"
+    return "unknown"
+
+
+def _within(a: float, b: float, frac: float) -> bool:
+    """True if a and b are within *frac* of each other (relative)."""
+    hi = max(a, b)
+    if hi <= 0:
+        return False
+    return (min(a, b) / hi) >= (1.0 - frac)
+
+
+def evaluate(run: RunSnapshot) -> "list[Finding]":
+    wall = run.wall_time
+    if wall <= 0:
+        return []
+
+    cpu_busy = (run.cpu_time / wall) if (run.cpu_time is not None and wall > 0) else None
+
+    r1 = r1_group = None
+    r2 = None
+    r3 = None
+
+    # --- R1: chatty query group (worst by total_time) -------------------------
+    candidates = [
+        g for g in run.groups
+        if g.calls > 1000 and g.total_time > 0.10 * wall and g.median < 0.005
+    ]
+    if candidates:
+        g = max(candidates, key=lambda x: x.total_time)
+        r1_group = g
+        detail = (
+            f"{g.calls:,} calls x {g.median * 1000:.2f}ms median "
+            f"~= {g.total_time:.1f}s = {g.total_time / wall * 100:.0f}% of {wall:.1f}s wall, "
+            f"at {_site_str(g)}. Batch it (executemany / IN-list / JOIN)."
+        )
+        r1 = Finding("R1", "Chatty query group", detail, g.total_time)
+
+    # --- R2: commit-per-row ---------------------------------------------------
+    if (
+        run.total_commits > 100
+        and run.commit_time is not None
+        and (run.total_rows / run.total_commits) < 10
+        and run.commit_time > 0.05 * wall
+    ):
+        avg = run.total_rows / run.total_commits
+        detail = (
+            f"{run.total_commits:,} commits for {run.total_rows:,} rows "
+            f"({avg:.1f} rows/commit), {run.commit_time:.1f}s in commit "
+            f"= {run.commit_time / wall * 100:.0f}% of wall. "
+            f"Batch to 1,000+ rows per transaction."
+        )
+        r2 = Finding("R2", "Commit-per-row", detail, run.commit_time)
+
+    # --- R3: DB-wait dominates ------------------------------------------------
+    if run.db_time > 0.60 * wall and cpu_busy is not None and cpu_busy < 0.30:
+        detail = (
+            f"DB {run.db_time:.1f}s of {wall:.1f}s wall "
+            f"({run.db_time / wall * 100:.0f}%), CPU busy only {cpu_busy * 100:.0f}% "
+            f"-> round-trip bound, not compute bound."
+        )
+        r3 = Finding("R3", "DB-wait dominates", detail, run.db_time)
+
+    # --- merge / fold ---------------------------------------------------------
+    findings: "list[Finding]" = []
+    consumed = set()
+
+    # R1 + R2 same-loop merge (calls ~= commits within 25%)
+    if r1 is not None and r2 is not None and _within(r1_group.calls, run.total_commits, 0.25):
+        consumed.update({"r1", "r2"})
+        rule = "R1+R2"
+        detail = r1.detail + "\n" + r2.detail
+        seconds = r1.seconds + r2.seconds
+        if r3 is not None:
+            consumed.add("r3")
+            rule = "R1+R2+R3"
+            detail = detail + "\n" + r3.detail  # R3 overlaps R1's query time; don't add seconds
+        findings.append(Finding(rule, "commit-per-row loop", detail, seconds))
+
+    # R3 + exactly one other, where that other explains >= 80% of the DB wait
+    if "r3" not in consumed and r3 is not None:
+        other = other_name = None
+        if r1 is not None and r2 is None:
+            other, other_name = r1, "r1"
+        elif r2 is not None and r1 is None:
+            other, other_name = r2, "r2"
+        if other is not None and other.seconds >= 0.8 * run.db_time:
+            consumed.update({"r3", other_name})
+            if other_name == "r1":
+                # R1's query time is a subset of db_time: attribute db_time, no double-count
+                seconds = max(other.seconds, r3.seconds)
+                rule = "R1+R3"
+            else:
+                # commit time is disjoint from cursor-execute db_time: additive
+                seconds = other.seconds + r3.seconds
+                rule = "R2+R3"
+            findings.append(Finding(rule, other.title, other.detail + "\n" + r3.detail, seconds))
+
+    # emit any survivors that were not folded into a merge
+    for name, f in (("r1", r1), ("r2", r2), ("r3", r3)):
+        if f is not None and name not in consumed:
+            findings.append(f)
+
+    # suppress trivia, sort, cap at 3
+    findings = [f for f in findings if f.seconds >= 0.05 * wall]
+    findings.sort(key=lambda f: f.seconds, reverse=True)
+    return findings[:3]

wherewent/stats.py

@@ -0,0 +1,42 @@
+"""Shared dataclasses — the load-bearing contract that rules/report/tests depend on.
+
+Field names and types here are frozen: do not rename or reorder without updating
+every consumer.
+"""
+
+from dataclasses import dataclass, field
+
+
+@dataclass
+class GroupSnapshot:
+    key: str                     # 12-hex-char sha1 prefix of normalized_sql
+    normalized_sql: str
+    calls: int
+    total_time: float            # seconds, app-observed (includes driver+network+server)
+    median: float                # seconds; 0.0 if no samples
+    rows: int                    # summed cursor.rowcount where >= 0, else 0 contribution
+    executemany_calls: int
+    call_site: "tuple[str, int, str] | None"  # (file, line, function); None if unresolved
+
+
+@dataclass
+class RunSnapshot:
+    wall_time: float             # seconds
+    cpu_time: "float | None"     # process user+sys CPU seconds (os.times); None if unmeasurable
+    total_queries: int
+    total_commits: int
+    total_rollbacks: int
+    commit_time: "float | None"  # seconds inside DBAPI commit; None if not measurable
+    total_rows: int
+    db_time: float               # sum of all cursor-execute durations
+    overhead_time: float         # recorder's own measured hook time, seconds
+    sqlalchemy_active: bool      # were hooks installed and did SQLAlchemy get used
+    groups: "list[GroupSnapshot]" = field(default_factory=list)
+
+
+@dataclass
+class Finding:
+    rule: str        # "R1" | "R2" | "R3" | "R1+R2" | "R1+R2+R3" | "R1+R3" | "R2+R3"
+    title: str       # one line
+    detail: str      # multi-line: MUST show the arithmetic
+    seconds: float   # estimated seconds attributable

wherewent-0.1.0.dist-info/METADATA

@@ -0,0 +1,204 @@
+Metadata-Version: 2.4
+Name: wherewent
+Version: 0.1.0
+Summary: Zero-config SQL flight recorder that answers 'why did this Python batch job take so long?'
+Author-email: Habiba Faisal <habibafaisal8@gmail.com>
+License-Expression: MIT
+Project-URL: Homepage, https://github.com/habibafaisal/wherewent
+Project-URL: Repository, https://github.com/habibafaisal/wherewent
+Project-URL: Issues, https://github.com/habibafaisal/wherewent/issues
+Project-URL: Changelog, https://github.com/habibafaisal/wherewent/blob/main/CHANGELOG.md
+Keywords: profiling,performance,sqlalchemy,postgresql,sql,observability,batch,n+1,orm
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Developers
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Topic :: Database
+Classifier: Topic :: Software Development :: Debuggers
+Classifier: Topic :: System :: Monitoring
+Classifier: Environment :: Console
+Classifier: Operating System :: OS Independent
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Provides-Extra: psutil
+Requires-Dist: psutil; extra == "psutil"
+Provides-Extra: dev
+Requires-Dist: pytest; extra == "dev"
+Requires-Dist: sqlalchemy>=2.0; extra == "dev"
+Dynamic: license-file
+
+<div align="center">
+
+# wherewent
+
+### Where did the time go? Find out in one command.
+
+**A zero-config recorder that answers "why did this Python batch job take so long?"**
+
+[![PyPI version](https://img.shields.io/pypi/v/wherewent.svg)](https://pypi.org/project/wherewent/)
+[![Python versions](https://img.shields.io/pypi/pyversions/wherewent.svg)](https://pypi.org/project/wherewent/)
+[![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](LICENSE)
+[![CI](https://github.com/habibafaisal/wherewent/actions/workflows/ci.yml/badge.svg)](https://github.com/habibafaisal/wherewent/actions/workflows/ci.yml)
+
+```bash
+wherewent run python your_job.py
+```
+
+</div>
+
+---
+
+## The 0.4ms query that costs you 5 minutes
+
+A query can be individually fast — `0.4ms` — and still sink your job, because it's
+called **500,000 times from a single line of code**. Your app burns 300 seconds on
+round-trips while Postgres itself only worked for 80. Every profiler you've tried shows
+you *"time spent in psycopg"* and stops there.
+
+**wherewent shows you the calling pattern.** It groups queries by shape, counts how
+often each shape ran, sums the wall time, and points at the exact `file:line` in *your*
+code that fired it — then tells you, in plain English with the arithmetic shown, what to
+do about it.
+
+```
+====================================================================================================
+wherewent — SQL flight recorder
+----------------------------------------------------------------------------------------------------
+wall: 26.15s   cpu: 25.41s (97% CPU busy)   queries: 20,004   commits: 20,001   rollbacks: 1
+in-DB time: 5.46s (20.9% of wall; app-observed: includes network+driver+server)
+commit time: 9.06s   total rows: 20,000
+recording added ~1.81s (~6.9% of wall)
+====================================================================================================
+QUERY GROUP                                         CALLS     TOTAL     MEDIAN  CALL SITE
+----------------------------------------------------------------------------------------------------
+INSERT INTO events (name, value) VALUES (?, ?)     20,000     5.46s     0.24ms  demo/naive_job.py:65 in main
+SELECT count(*) AS count_1 FROM events                  1     0.00s     0.16ms  demo/naive_job.py:71 in main
+====================================================================================================
+FINDINGS
+----------------------------------------------------------------------------------------------------
+1. [R1+R2] commit-per-row loop
+   20,000 calls x 0.24ms median ~= 5.5s = 21% of 26.1s wall, at demo/naive_job.py:65. Batch it.
+   20,001 commits for 20,000 rows (1.0 rows/commit), 9.1s in commit = 35% of wall. Batch to 1,000+ rows/txn.
+   ~= 14.5s attributable
+====================================================================================================
+```
+
+## Why it's different
+
+| | Sampling profilers | APM / tracing | **wherewent** |
+|---|:---:|:---:|:---:|
+| Zero code changes | ✅ | ❌ | ✅ |
+| Groups queries by shape | ❌ | ⚠️ | ✅ |
+| Blames *your* call site | ⚠️ | ⚠️ | ✅ |
+| Tells you the fix | ❌ | ❌ | ✅ |
+| Runs anywhere, no server | ✅ | ❌ | ✅ |
+| Works on a Ctrl-C'd partial run | ❌ | ⚠️ | ✅ |
+
+## Install
+
+```bash
+pip install wherewent
+```
+
+That's it — the recorder is **pure standard library**. You only need SQLAlchemy
+because *your job* already uses it.
+
+## Use it
+
+Wrap any command. Your script runs **completely unmodified** — no imports, no decorators,
+no config:
+
+```bash
+wherewent run python your_job.py --some arg
+wherewent run python -m your_package
+wherewent run --save run.json python your_job.py   # also dump machine-readable JSON
+```
+
+- The report prints to **stderr** at exit; your job's own stdout/stderr pass through untouched.
+- **Ctrl-C still produces a report.** Sampling the first 5 minutes of a 14-hour job is the
+  main use case — partial data is the point.
+- **It can never crash or corrupt your job.** Every hook body is wrapped so the recorder
+  fails silent rather than taking your run down with it.
+- **It never records your data.** Only query *shapes* and *counts* are kept — literal
+  values and bind parameters are stripped before anything is stored.
+
+### Try the built-in demo
+
+```bash
+git clone https://github.com/habibafaisal/wherewent && cd wherewent
+pip install -e ".[dev]"
+wherewent run python demo/naive_job.py     # watch the R1+R2 finding fire
+python demo/benchmark.py                    # naive vs fixed, with the overhead gate
+```
+
+## How it works
+
+1. **Injects itself** into the target process via a `PYTHONPATH` sitecustomize shim — no
+   changes to your code, no wrapper imports.
+2. **Listens at the class level** — `event.listen(sqlalchemy.engine.Engine, ...)` — so
+   *every* engine your app creates is captured automatically, config-free.
+3. **Normalizes each statement** into a query *group*: literals, bind params, `IN`-lists
+   and multi-row `VALUES` collapse, so a million distinct inserts become one honest row.
+4. **Resolves the call site** by walking the stack past library frames to the first line
+   of *your* code — cheaply: cached by filename, full stacks only for the first 5 samples
+   per group, so the hot path stays cheap enough to hit its overhead budget.
+5. **Fires deterministic findings** from three rules, each showing its arithmetic.
+
+### The findings engine
+
+| Rule | Fires when | Tells you |
+|---|---|---|
+| **R1 — chatty group** | > 1,000 calls, > 10% of wall, median < 5ms | A fast query is called too many times — batch it (`executemany` / `IN`-list / `JOIN`). |
+| **R2 — commit-per-row** | > 100 commits, < 10 rows/commit, > 5% of wall in commit | You're committing per row — batch to 1,000+ rows per transaction. |
+| **R3 — DB-wait bound** | in-DB time > 60% of wall, CPU busy < 30% | The job is round-trip bound, not compute bound. |
+
+Findings that share a root cause **merge** (e.g. `R1+R2`), everything under 5% of wall is
+suppressed, and at most the top 3 are shown — ranked by seconds attributable.
+
+**Every number is honest.** Query times are labelled *app-observed* (they include network,
+driver, and server time — not just Postgres). Anything that can't be measured prints `—`,
+never a guess. wherewent even times *its own hooks* and reports the overhead it added.
+
+## Roadmap — help wanted 🙌
+
+wherewent is built to grow **beyond SQLAlchemy**. Seven of its eight modules —
+normalization, call-site resolution, the stats model, the rules engine, the report, the
+CLI, and the injection shim — are already **framework-agnostic**. They operate on a plain
+`RunSnapshot` of query events. Only `recorder.py`, which binds SQLAlchemy's event system,
+is framework-specific.
+
+**That means a new backend is a well-contained contribution:** capture query
+start/end/rowcount/txn events from another driver, feed the same `RunSnapshot`, and the
+entire findings-and-report pipeline works for free. Good first backends:
+
+- [ ] **Raw `psycopg` / `psycopg2`** — cursor subclass or connection factory hook
+- [ ] **`asyncpg` / async SQLAlchemy** — the async execution path
+- [ ] **Django ORM** — via `connection.execute_wrapper`
+- [ ] **Generic DB-API 2.0** — a monkeypatch-free `Cursor` proxy
+- [ ] New findings rules (N+1 `SELECT` detection, lock-wait, seq-scan heuristics)
+
+See [`CONTRIBUTING.md`](CONTRIBUTING.md) for the backend contract and the **< 15% overhead
+gate** that every capture path must pass.
+
+## Limitations (today)
+
+- SQLAlchemy **2.x, synchronous** only (1.4 may work; async does not yet).
+- Single process — no multiprocessing or async fan-out.
+- Query times are app-observed (network + driver + server), by design.
+- Commit timing is obtained by wrapping the dialect's commit; if that wrap fails it prints `—`.
+
+These are the honest edges of a validation prototype, not permanent walls — see the roadmap.
+
+## Contributing
+
+Contributions are very welcome — new backends, new rules, docs, bug reports. Start with
+[`CONTRIBUTING.md`](CONTRIBUTING.md), open an issue to discuss anything substantial, and
+run `pytest && python demo/benchmark.py` before you push.
+
+## License
+
+[MIT](LICENSE) © 2026 Habiba Faisal

wherewent-0.1.0.dist-info/RECORD

@@ -0,0 +1,16 @@
+wherewent/__init__.py,sha256=d62PR2Qhvc7BZCSDvffjx6dpoO9soQtS6gVaiSwiIJc,108
+wherewent/callsite.py,sha256=BfLjAqKAkgipiLeNcXc6WQolXS-EwEdhoLQyINhhqjg,3172
+wherewent/cli.py,sha256=xlm2mr3IZU8B8mKwxErO4KgTrxp-pVoUF31yF_h7or8,2577
+wherewent/normalize.py,sha256=1E0zerQefcmuwXHq7qlpvqngh1sorExavhIzhUUV5zU,1950
+wherewent/recorder.py,sha256=upykLxIjcutJMNm6RXxqkWTmdJttiz_uUqK1RuMbjww,13219
+wherewent/report.py,sha256=1cwXhLXFme3e8VzYNeTXR8BHMyAG1BsSR7mDJZHAVB8,2965
+wherewent/rules.py,sha256=EvP-WTv2SlLxIgzUKf8A9kV8bjzkcKn03V_7pXQlDYc,4714
+wherewent/stats.py,sha256=EuQ7EV5JsFWKdPIzhvJoVXXX3imet7PUUJoS2vZmV0I,1647
+wherewent/_shim/__init__.py,sha256=UQaL9OROUZV4txD12NURZdAUl513ugLvXlWHx54XBk4,147
+wherewent/_shim/sitecustomize.py,sha256=rjww2F6hx8ILrZsV_Q-cyg905pNfrmVwjFyBEwQHv-c,545
+wherewent-0.1.0.dist-info/licenses/LICENSE,sha256=Y4N2_d6kVd8vbQz5NTh7fdq9YsRC5u6gzcbyia_ZFgw,1070
+wherewent-0.1.0.dist-info/METADATA,sha256=7BbXzH065KiNT0812lZX9p3G1gJzPwDpik35XTKNXmw,9672
+wherewent-0.1.0.dist-info/WHEEL,sha256=K260EYznzXsJYBQGqmI8VTxEdiZYNvDZwW9cBh9-_MA,91
+wherewent-0.1.0.dist-info/entry_points.txt,sha256=rFsqA2cAQZ1cy-8EtkKXHppn4fJ61-llJvomxf67lHw,49
+wherewent-0.1.0.dist-info/top_level.txt,sha256=LBzV2Qrv0W5rgMSKVYIJJFt9MNeDbBCsPPNN9kYfLVU,10
+wherewent-0.1.0.dist-info/RECORD,,

wherewent-0.1.0.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (83.0.0)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

wherewent-0.1.0.dist-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+wherewent = wherewent.cli:main

wherewent-0.1.0.dist-info/licenses/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Habiba Faisal
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

wherewent-0.1.0.dist-info/top_level.txt

@@ -0,0 +1 @@
+wherewent