pgrecon 0.1.0__py3-none-any.whl

pgrecon/__init__.py

@@ -0,0 +1,5 @@
+"""pgrecon: migration reconnaissance for PostgreSQL."""
+
+from importlib.metadata import version
+
+__version__ = version("pgrecon")

pgrecon/cli.py

@@ -0,0 +1,327 @@
+"""Command-line interface."""
+
+import json
+import logging
+import sqlite3
+import sys
+import textwrap
+from collections import Counter
+from dataclasses import asdict
+from pathlib import Path
+from typing import Annotated
+
+import typer
+
+from pgrecon import __version__
+from pgrecon.effort import estimate as build_estimate
+from pgrecon.effort import person_months
+from pgrecon.extract import needs_legacy, script_text
+from pgrecon.inventory import load_dump
+from pgrecon.rules import Rule, all_rules
+from pgrecon.rules.engine import run_rules, summarize
+
+# sqlglot logs a warning whenever exotic syntax makes it fall back to an
+# opaque statement; the outcome is already recorded in the inventory, so
+# the console noise helps nobody.
+logging.getLogger("sqlglot").setLevel(logging.ERROR)
+
+app = typer.Typer(
+    help="Migration reconnaissance for PostgreSQL.",
+    no_args_is_help=True,
+    add_completion=False,
+)
+
+
+def _show_version(value: bool) -> None:
+    if value:
+        typer.echo(f"pgrecon {__version__}")
+        raise typer.Exit()
+
+
+@app.callback()
+def main(
+    version: Annotated[
+        bool,
+        typer.Option(
+            "--version",
+            callback=_show_version,
+            is_eager=True,
+            help="Print the version and exit.",
+        ),
+    ] = False,
+    verbose: Annotated[
+        int,
+        typer.Option(
+            "--verbose",
+            "-v",
+            count=True,
+            help="Log progress to stderr; repeat for debug detail.",
+        ),
+    ] = 0,
+) -> None:
+    # Logs go to stderr and never carry PL/SQL body text, so stdout
+    # stays clean for reports and the log stays safe to share.
+    logging.basicConfig(
+        level=logging.WARNING - 10 * min(verbose, 2),
+        stream=sys.stderr,
+        format="%(levelname)s: %(message)s",
+    )
+
+
+@app.command()
+def script(
+    out: Annotated[
+        Path, typer.Option(help="Where to write the extraction script.")
+    ] = Path("pgrecon_extract.sql"),
+    source_version: Annotated[
+        str | None,
+        typer.Option(
+            "--source-version",
+            help="Oracle version of the source, e.g. 9.2, 11.2, 19."
+            " Picks the right script variant.",
+        ),
+    ] = None,
+    legacy: Annotated[
+        bool,
+        typer.Option(
+            "--legacy",
+            help="Force the variant for Oracle 9.2-11.1 or old clients.",
+        ),
+    ] = False,
+) -> None:
+    """Write the offline SQL*Plus extraction script for the client DBA."""
+    if source_version is not None:
+        try:
+            legacy = legacy or needs_legacy(source_version)
+        except ValueError as exc:
+            raise typer.BadParameter(str(exc)) from exc
+    if legacy and out.name == "pgrecon_extract.sql":
+        out = out.with_name("pgrecon_extract_legacy.sql")
+    out.write_text(script_text(legacy), encoding="ascii", newline="\n")
+    typer.echo(f"Wrote {out}" + (" (legacy variant)" if legacy else ""))
+    typer.echo("Run it as: sqlplus readonly_user@service @" + out.name + " SCHEMA")
+
+
+@app.command()
+def load(
+    dump_dir: Annotated[
+        Path,
+        typer.Argument(exists=True, file_okay=False, help="Extraction dump folder."),
+    ],
+    db: Annotated[Path, typer.Option(help="Inventory database to create.")] = Path(
+        "inventory.db"
+    ),
+    encoding: Annotated[
+        str,
+        typer.Option(
+            help="Encoding of the dump files, e.g. cp949 for a Korean"
+            " Windows client. Defaults to UTF-8."
+        ),
+    ] = "utf-8-sig",
+) -> None:
+    """Load an extraction dump into a local SQLite inventory."""
+    counts = load_dump(dump_dir, db, encoding)
+    width = max(len(name) for name in counts)
+    for name, count in sorted(counts.items()):
+        typer.echo(f"{name:<{width}}  {count}")
+    typer.echo(f"Inventory written to {db}")
+
+
+@app.command()
+def report(
+    db: Annotated[
+        Path,
+        typer.Option(exists=True, dir_okay=False, help="Inventory database."),
+    ] = Path("inventory.db"),
+    fmt: Annotated[
+        str, typer.Option("--format", help="Output format: text or json.")
+    ] = "text",
+    remedies: Annotated[
+        bool,
+        typer.Option(
+            "--remedies",
+            help="Append each fired rule's remedy to the text report.",
+        ),
+    ] = False,
+) -> None:
+    """Run the assessment rules and print the findings."""
+    findings = run_rules(db)
+    summary = summarize(findings)
+    by_id = {rule.id: rule for rule in all_rules()}
+    fired = {f.rule_id: by_id[f.rule_id] for f in findings}
+
+    if fmt == "json":
+        payload = {
+            "summary": summary,
+            "findings": [asdict(f) for f in findings],
+            "rules": {
+                rule_id: _rule_meta(rule) for rule_id, rule in sorted(fired.items())
+            },
+        }
+        typer.echo(json.dumps(payload, indent=2))
+        return
+    if fmt != "text":
+        raise typer.BadParameter("format must be text or json")
+
+    if not findings:
+        typer.echo("No findings.")
+        return
+    widths = (
+        max(len(f.severity.value) for f in findings),
+        max(len(f.rule_id) for f in findings),
+        max(len(f.name) for f in findings),
+    )
+    for f in findings:
+        typer.echo(
+            f"{f.severity.value:<{widths[0]}}  {f.rule_id:<{widths[1]}}"
+            f"  {f.name:<{widths[2]}}  {f.detail}"
+        )
+    typer.echo("")
+    parts = [f"{n} {sev}" for sev, n in summary["by_severity"].items()]
+    typer.echo(
+        f"{summary['findings']} findings ({', '.join(parts)});"
+        f" effort points {summary['effort_points']}"
+    )
+
+    if remedies:
+        counts = Counter(f.rule_id for f in findings)
+        typer.echo("")
+        typer.echo("Remedies:")
+        for rule_id in dict.fromkeys(f.rule_id for f in findings):
+            rule = fired[rule_id]
+            n = counts[rule_id]
+            tags = [rule.severity.value, f"{n} finding" + ("s" if n != 1 else "")]
+            if rule.extension:
+                tags.append(f"extension {rule.extension}")
+            typer.echo("")
+            typer.echo(f"{rule.id}  {rule.title}  [{', '.join(tags)}]")
+            typer.echo(_wrap(rule.remedy))
+
+
+def _rule_meta(rule: Rule) -> dict[str, object]:
+    return {
+        "title": rule.title,
+        "category": rule.category,
+        "severity": rule.severity.value,
+        "effort": rule.effort,
+        "remedy": rule.remedy,
+        "extension": rule.extension,
+    }
+
+
+def _wrap(text: str) -> str:
+    return textwrap.fill(text, width=76, initial_indent="  ", subsequent_indent="  ")
+
+
+@app.command()
+def estimate(
+    db: Annotated[
+        Path,
+        typer.Option(exists=True, dir_okay=False, help="Inventory database."),
+    ] = Path("inventory.db"),
+    fmt: Annotated[
+        str, typer.Option("--format", help="Output format: text or json.")
+    ] = "text",
+) -> None:
+    """Estimate migration effort in person-days, as a range."""
+    findings = run_rules(db)
+    result = build_estimate(db, findings)
+
+    if fmt == "json":
+        payload = {
+            "components": {c.label: c.person_days for c in result.components},
+            "development_person_days": result.development,
+            "person_days": {
+                "low": result.low,
+                "expected": result.expected,
+                "high": result.high,
+            },
+            "person_months": {
+                "low": person_months(result.low),
+                "expected": person_months(result.expected),
+                "high": person_months(result.high),
+            },
+            "assumptions": list(result.assumptions),
+        }
+        typer.echo(json.dumps(payload, indent=2))
+        return
+    if fmt != "text":
+        raise typer.BadParameter("format must be text or json")
+
+    typer.echo("Migration effort estimate (person-days)")
+    typer.echo("")
+    width = max(len(c.label) for c in result.components)
+    for c in result.components:
+        typer.echo(f"  {c.label:<{width}}  {c.person_days:8.1f}")
+    typer.echo(f"  {'development subtotal':<{width}}  {result.development:8.1f}")
+    typer.echo("")
+    typer.echo("With testing and stabilization:")
+    typer.echo(
+        f"  low {result.low:.0f}, expected {result.expected:.0f},"
+        f" high {result.high:.0f} person-days"
+        f" ({person_months(result.low):.1f} to"
+        f" {person_months(result.high):.1f} person-months)"
+    )
+    typer.echo("")
+    typer.echo("Assumptions:")
+    for line in result.assumptions:
+        typer.echo(_wrap("- " + line))
+
+
+@app.command()
+def explain(
+    rule_id: Annotated[
+        str | None,
+        typer.Argument(help="Rule id, e.g. R-SRC-18. Omit to list the catalog."),
+    ] = None,
+) -> None:
+    """Show one rule's remedy, or list the whole catalog."""
+    rules = all_rules()
+    if rule_id is None:
+        width = max(len(rule.id) for rule in rules)
+        for rule in sorted(rules, key=lambda r: r.id):
+            typer.echo(f"{rule.id:<{width}}  {rule.severity.value:<7}  {rule.title}")
+        typer.echo("")
+        typer.echo(f"{len(rules)} rules")
+        return
+    match = {rule.id: rule for rule in rules}.get(rule_id.upper())
+    if match is None:
+        raise typer.BadParameter(f"unknown rule id: {rule_id}")
+    typer.echo(f"{match.id}: {match.title}")
+    line = (
+        f"severity: {match.severity.value}  effort: {match.effort}"
+        f"  category: {match.category}"
+    )
+    if match.extension:
+        line += f"  extension: {match.extension}"
+    typer.echo(line)
+    typer.echo("")
+    typer.echo(_wrap(match.remedy))
+
+
+@app.command()
+def info(
+    db: Annotated[
+        Path,
+        typer.Option(exists=True, dir_okay=False, help="Inventory database."),
+    ] = Path("inventory.db"),
+) -> None:
+    """Summarize an inventory database."""
+    conn = sqlite3.connect(db)
+    try:
+        for key, value in conn.execute("SELECT key, value FROM meta ORDER BY key"):
+            typer.echo(f"{key}: {value}")
+        rows = conn.execute(
+            "SELECT type, COUNT(*) FROM objects GROUP BY type ORDER BY type"
+        ).fetchall()
+        if rows:
+            typer.echo("objects:")
+            for obj_type, count in rows:
+                typer.echo(f"  {obj_type}: {count}")
+        failed = conn.execute("SELECT COUNT(*) FROM ddl WHERE parse_ok = 0").fetchone()[
+            0
+        ]
+        total = conn.execute("SELECT COUNT(*) FROM ddl").fetchone()[0]
+        typer.echo(f"ddl parsed: {total - failed}/{total}")
+    finally:
+        conn.close()

pgrecon/effort.py

@@ -0,0 +1,161 @@
+"""Turn findings and inventory scale into a person-day estimate.
+
+The estimate is built from named components so it can be argued with
+line by line, which is the point: an opaque total invites either blind
+trust or blind dismissal. Development effort is the sum of a baseline,
+mechanical schema conversion, per-finding remediation, PL/SQL porting
+by volume, and data movement; testing and stabilization is applied as
+a factor range on top, because in field reports it rivals development
+and nobody budgets it.
+
+Every rate here is a default calibration from published field
+experience and ora2pg-era rules of thumb. They are deliberately
+visible and deliberately conservative; real engagements calibrate
+them against the client's team and workload.
+"""
+
+import sqlite3
+from dataclasses import dataclass
+from pathlib import Path
+
+from pgrecon.rules import Finding, Rule, Severity, all_rules
+
+# Person-days. The baseline covers environments, tooling, data-movement
+# setup, and cutover rehearsal scaffolding that exist even for a clean
+# schema.
+BASELINE_PD = 5.0
+PER_TABLE_PD = 0.2
+PER_INDEX_PD = 0.05
+
+# Mechanical conversion rates for objects that mostly translate.
+CONVERT_TABLE_PD = 0.05
+CONVERT_COLUMN_PD = 0.02
+CONVERT_VIEW_PD = 0.1
+CONVERT_SEQUENCE_PD = 0.02
+
+# Porting rate for stored code, including its unit tests. Findings
+# price the hotspots; this prices the bulk that merely needs careful
+# transcription.
+PLSQL_LINES_PER_DAY = 200.0
+
+# Data movement is mostly machine time; the human share scales weakly
+# with volume.
+BYTES_PER_PD = 50e9
+
+# A rule firing n times does not cost n times the first fix: the
+# pattern is learned once. The marginal share of each further finding
+# depends on how mechanical the fix is, which severity approximates.
+MARGINAL = {
+    Severity.INFO: 0.05,
+    Severity.LOW: 0.2,
+    Severity.MEDIUM: 0.5,
+    Severity.HIGH: 0.7,
+    Severity.BLOCKER: 1.0,
+}
+
+# Testing and stabilization as a share of development. Field reports
+# put it between a third and everything-again-and-more.
+TESTING_LOW = 1.3
+TESTING_EXPECTED = 1.6
+TESTING_HIGH = 2.2
+
+WORKDAYS_PER_MONTH = 21.0
+
+
+@dataclass(frozen=True)
+class Component:
+    label: str
+    person_days: float
+
+
+@dataclass(frozen=True)
+class Estimate:
+    components: tuple[Component, ...]
+    development: float
+    low: float
+    expected: float
+    high: float
+    assumptions: tuple[str, ...]
+
+
+def _scalar(conn: sqlite3.Connection, query: str) -> float:
+    value = conn.execute(query).fetchone()[0]
+    return float(value or 0)
+
+
+def _remediation(findings: list[Finding], rules: list[Rule]) -> float:
+    by_id = {rule.id: rule for rule in rules}
+    counts: dict[str, int] = {}
+    for finding in findings:
+        counts[finding.rule_id] = counts.get(finding.rule_id, 0) + 1
+    total = 0.0
+    for rule_id, n in counts.items():
+        rule = by_id.get(rule_id)
+        if rule is None:
+            continue
+        marginal = MARGINAL[rule.severity]
+        total += rule.effort * (1 + marginal * (n - 1))
+    return total
+
+
+def estimate(
+    db_path: Path, findings: list[Finding], rules: list[Rule] | None = None
+) -> Estimate:
+    if rules is None:
+        rules = all_rules()
+    conn = sqlite3.connect(db_path)
+    try:
+        tables = _scalar(conn, "SELECT COUNT(*) FROM tables")
+        columns = _scalar(conn, "SELECT COUNT(*) FROM columns")
+        indexes = _scalar(conn, "SELECT COUNT(*) FROM indexes")
+        views = _scalar(conn, "SELECT COUNT(*) FROM objects WHERE type = 'VIEW'")
+        sequences = _scalar(
+            conn, "SELECT COUNT(*) FROM objects WHERE type = 'SEQUENCE'"
+        )
+        source_lines = _scalar(conn, "SELECT COUNT(*) FROM source")
+        data_bytes = _scalar(
+            conn,
+            "SELECT COALESCE(SUM(num_rows * avg_row_len), 0) FROM tables",
+        )
+    finally:
+        conn.close()
+
+    components = (
+        Component(
+            "baseline and environment",
+            BASELINE_PD + PER_TABLE_PD * tables + PER_INDEX_PD * indexes,
+        ),
+        Component(
+            "schema conversion",
+            CONVERT_TABLE_PD * tables
+            + CONVERT_COLUMN_PD * columns
+            + CONVERT_VIEW_PD * views
+            + CONVERT_SEQUENCE_PD * sequences,
+        ),
+        Component("finding remediation", _remediation(findings, rules)),
+        Component("PL/SQL porting by volume", source_lines / PLSQL_LINES_PER_DAY),
+        Component("data movement", data_bytes / BYTES_PER_PD),
+    )
+    development = sum(c.person_days for c in components)
+    assumptions = (
+        f"stored code ports at {PLSQL_LINES_PER_DAY:.0f} lines per person-day"
+        " including unit tests",
+        "repeated findings of one rule cost a severity-dependent fraction"
+        " of the first fix",
+        f"testing and stabilization runs {TESTING_LOW:.1f}x to"
+        f" {TESTING_HIGH:.1f}x development, {TESTING_EXPECTED:.1f}x expected",
+        "static estimate from schema facts alone: workload, data quality,"
+        " and application coupling are not visible here",
+    )
+    return Estimate(
+        components=components,
+        development=round(development, 1),
+        low=round(development * TESTING_LOW, 1),
+        expected=round(development * TESTING_EXPECTED, 1),
+        high=round(development * TESTING_HIGH, 1),
+        assumptions=assumptions,
+    )
+
+
+def person_months(person_days: float) -> float:
+    return round(person_days / WORKDAYS_PER_MONTH, 1)

pgrecon/extract/__init__.py

@@ -0,0 +1,32 @@
+"""Offline extraction: the reviewable SQL*Plus scripts and their helpers."""
+
+from importlib import resources
+
+
+def script_text(legacy: bool = False) -> str:
+    """Return the offline extraction script shipped with the package.
+
+    The standard script needs a SQL*Plus 12.2+ client and targets Oracle
+    11.2 or newer. The legacy variant runs on any old on-box SQL*Plus
+    against Oracle 9.2 through 11.1: hand-built CSV, no DBMS_METADATA.
+    """
+    name = "pgrecon_extract_legacy.sql" if legacy else "pgrecon_extract.sql"
+    path = resources.files("pgrecon.extract").joinpath(name)
+    return path.read_text(encoding="ascii")
+
+
+def needs_legacy(source_version: str) -> bool:
+    """Decide the script variant from an Oracle version like 9.2 or 19.
+
+    Anything below 11.2 takes the legacy variant. Raises ValueError for
+    input that does not look like an Oracle version.
+    """
+    parts = source_version.strip().split(".")
+    try:
+        major = int(parts[0])
+        minor = int(parts[1]) if len(parts) > 1 else 0
+    except (ValueError, IndexError) as exc:
+        raise ValueError(f"not an Oracle version: {source_version!r}") from exc
+    if not 8 <= major <= 30:
+        raise ValueError(f"not a supported Oracle version: {source_version!r}")
+    return (major, minor) < (11, 2)