mainframe-modernization-toolkit 0.1.0__py3-none-any.whl

mainframe_modernization_toolkit/__init__.py

@@ -0,0 +1,3 @@
+"""Deterministic mainframe modernization tools."""
+
+__version__ = "0.1.0"

mainframe_modernization_toolkit/__main__.py

@@ -0,0 +1,5 @@
+from .application import main
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

mainframe_modernization_toolkit/application.py

@@ -0,0 +1,142 @@
+"""Umbrella command for the mainframe modernization toolkit."""
+from __future__ import annotations
+
+import argparse
+import json
+from pathlib import Path
+import shutil
+import sys
+from typing import Any
+
+from . import __version__
+from .cli.migration_preflight import run_preflight
+from .registry import TOOL_MODULES, run_tool
+from .resources import export_vsix, resource_root, verify_vsix
+
+
+WORKSPACE_RESOURCES = (
+    ("mainframe-migration.json", "mainframe-migration.json"),
+    ("mainframe-migration.schema.json", "mainframe-migration.schema.json"),
+    ("relational-ir.schema.json", "relational-ir.schema.json"),
+    ("customizations/skills/mainframe-jcl-migration/SKILL.md", ".github/skills/mainframe-jcl-migration/SKILL.md"),
+    ("customizations/skills/mainframe-jcl-migration/references/migration-checklist.md", ".github/skills/mainframe-jcl-migration/references/migration-checklist.md"),
+    ("customizations/skills/mainframe-jcl-migration/references/tool-catalog.md", ".github/skills/mainframe-jcl-migration/references/tool-catalog.md"),
+    ("customizations/agents/mainframe-jcl-migrator.agent.md", ".github/agents/mainframe-jcl-migrator.agent.md"),
+)
+
+
+def initialize_workspace(root: str | Path, *, force: bool, dry_run: bool) -> dict[str, list[str]]:
+    target_root = Path(root).expanduser().resolve()
+    if target_root.exists() and not target_root.is_dir():
+        raise ValueError(f"Workspace root is not a directory: {target_root}")
+    copied: list[str] = []
+    skipped: list[str] = []
+    resources = resource_root()
+    for source_name, destination_name in WORKSPACE_RESOURCES:
+        destination = target_root / destination_name
+        if destination.exists() and not force:
+            skipped.append(destination_name)
+            continue
+        copied.append(destination_name)
+        if not dry_run:
+            destination.parent.mkdir(parents=True, exist_ok=True)
+            destination.write_bytes(resources.joinpath(*source_name.split("/")).read_bytes())
+    return {"copied": copied, "skipped": skipped}
+
+
+def doctor(workspace: str | Path | None) -> dict[str, Any]:
+    checks: list[dict[str, Any]] = []
+    checks.append({
+        "name": "python",
+        "ok": sys.version_info >= (3, 10),
+        "detail": f"{sys.version_info.major}.{sys.version_info.minor}.{sys.version_info.micro}",
+    })
+    missing = [name for name, _ in WORKSPACE_RESOURCES if not resource_root().joinpath(*name.split("/")).is_file()]
+    checks.append({"name": "package_resources", "ok": not missing, "detail": missing or "available"})
+    try:
+        vsix_result = verify_vsix()
+    except (FileNotFoundError, KeyError, OSError, json.JSONDecodeError) as error:
+        vsix_result = {"ok": False, "errors": [str(error)]}
+    checks.append({"name": "embedded_vsix", "ok": vsix_result["ok"], "detail": vsix_result.get("errors") or "verified"})
+    code_path = shutil.which("code")
+    checks.append({"name": "vscode_cli", "ok": code_path is not None, "required": False, "detail": code_path or "not found"})
+
+    if workspace is not None:
+        root = Path(workspace).expanduser().resolve()
+        config = root / "mainframe-migration.json"
+        if not config.is_file():
+            checks.append({"name": "workspace_preflight", "ok": False, "detail": f"missing {config}"})
+        else:
+            report = run_preflight(root)
+            checks.append({
+                "name": "workspace_preflight",
+                "ok": not report.blocked,
+                "detail": {"blocked": report.blocked, "findings": len(report.findings)},
+            })
+    required_checks = [check for check in checks if check.get("required", True)]
+    return {"ok": all(check["ok"] for check in required_checks), "checks": checks}
+
+
+def build_parser() -> argparse.ArgumentParser:
+    parser = argparse.ArgumentParser(prog="mainframe-toolkit")
+    parser.add_argument("--version", action="version", version=__version__)
+    commands = parser.add_subparsers(dest="command", required=True)
+
+    run_parser = commands.add_parser("run", help="Run one packaged migration tool")
+    run_parser.add_argument("tool", choices=tuple(TOOL_MODULES))
+    run_parser.add_argument("arguments", nargs=argparse.REMAINDER)
+
+    vsix_parser = commands.add_parser("vsix", help="Export or verify the bundled VSIX")
+    vsix_commands = vsix_parser.add_subparsers(dest="vsix_command", required=True)
+    export_parser = vsix_commands.add_parser("export")
+    export_parser.add_argument("--output", required=True)
+    verify_parser = vsix_commands.add_parser("verify")
+    verify_parser.add_argument("path", nargs="?")
+
+    doctor_parser = commands.add_parser("doctor", help="Check package and workspace readiness")
+    doctor_parser.add_argument("--workspace")
+    doctor_parser.add_argument("--format", choices=("json", "text"), default="text")
+
+    workspace_parser = commands.add_parser("workspace", help="Manage workspace templates")
+    workspace_commands = workspace_parser.add_subparsers(dest="workspace_command", required=True)
+    init_parser = workspace_commands.add_parser("init")
+    init_parser.add_argument("root")
+    init_parser.add_argument("--force", action="store_true")
+    init_parser.add_argument("--dry-run", action="store_true")
+    return parser
+
+
+def main(argv: list[str] | None = None) -> int:
+    args = build_parser().parse_args(argv)
+    if args.command == "run":
+        tool_arguments = args.arguments[1:] if args.arguments[:1] == ["--"] else args.arguments
+        return run_tool(args.tool, tool_arguments)
+    if args.command == "vsix":
+        if args.vsix_command == "export":
+            print(export_vsix(args.output))
+            return 0
+        result = verify_vsix(args.path)
+        print(json.dumps(result, indent=2, ensure_ascii=True))
+        return 0 if result["ok"] else 1
+    if args.command == "doctor":
+        result = doctor(args.workspace)
+        if args.format == "json":
+            print(json.dumps(result, indent=2, ensure_ascii=True))
+        else:
+            for check in result["checks"]:
+                status = "OK" if check["ok"] else "WARN" if not check.get("required", True) else "FAIL"
+                print(f"{status:4} {check['name']}: {check['detail']}")
+        return 0 if result["ok"] else 1
+    if args.command == "workspace" and args.workspace_command == "init":
+        try:
+            result = initialize_workspace(args.root, force=args.force, dry_run=args.dry_run)
+        except (OSError, ValueError) as error:
+            print(f"workspace init failed: {error}", file=sys.stderr)
+            return 1
+        action = "Would copy" if args.dry_run else "Copied"
+        for path in result["copied"]:
+            print(f"{action}: {path}")
+        for path in result["skipped"]:
+            print(f"Skipped existing: {path}")
+        return 0
+    return 2

mainframe_modernization_toolkit/cli/__init__.py

@@ -0,0 +1 @@
+"""Console command implementations."""

mainframe_modernization_toolkit/cli/business_rule_extractor.py

@@ -0,0 +1,263 @@
+#!/usr/bin/env python3
+"""business_rule_extractor.py - Mine IF/EVALUATE decision logic out of a
+COBOL program into a structured, human- and agent-readable rule list.
+
+Why this matters for migration
+-------------------------------
+The hardest part of any COBOL->Python rewrite isn't syntax translation,
+it's *rule harvesting*: recovering the tangle of decades-old IF/EVALUATE
+conditions that encode the actual business logic before anyone can safely
+reimplement it. Big modernization vendors (Micro Focus, IBM, Advanced)
+sell "business rule extraction" as a discrete phase that happens *before*
+code generation - see docs/RESEARCH.md. This script gives an AI agent a
+deterministic first pass at that phase: every IF/WHEN condition, its
+consequence, and the paragraph/COBOL comment context around it, so a human
+or a follow-up LLM step can confirm/refine the rule in plain English
+instead of re-deriving it from raw COBOL.
+
+What it does
+------------
+For a single COBOL file:
+1. Splits into paragraphs (using PROCEDURE DIVISION paragraph headers).
+2. Within each paragraph, extracts top-level IF/ELSE/END-IF and
+   EVALUATE/WHEN/END-EVALUATE constructs with their condition text.
+3. Attaches any COBOL comment lines immediately preceding the construct as
+   "documented_intent" (many legacy shops *did* write down the "why").
+4. Emits a structured rule list ready for a Python `if`/dict-dispatch
+   rewrite, or for a business analyst to review.
+
+Edge cases (never silently mis-extracted)
+--------------------------------------------
+An IF/EVALUATE block that never reaches its matching END-IF/END-EVALUATE
+before end of file (e.g. a missing terminator, or nesting deeper than
+this line-scanner tracks) is still emitted as a rule, but marked
+`"truncated": true` with a `review_note` explaining that its
+condition/branches may be incomplete - never silently merged into the
+next rule or dropped.
+
+Usage
+-----
+    python3 business_rule_extractor.py <file.cbl> [--format json|markdown]
+"""
+from __future__ import annotations
+
+import argparse
+import json
+import re
+import sys
+from pathlib import Path
+
+from ..common.cli_context import add_config_argument, single_file_context_from_args  # noqa: E402
+from ..common.cobol_parser import strip_source_line  # noqa: E402
+from ..common.warnings_model import DEFAULT_TODO_PREFIX, render_todo  # noqa: E402
+
+RE_PARAGRAPH_HEADER = re.compile(r"^\s{0,10}([A-Z0-9][\w-]*)\.\s*$", re.IGNORECASE)
+RE_IF = re.compile(r"^\s*IF\s+(.+?)(?:\s+THEN)?\s*$", re.IGNORECASE)
+RE_ELSE = re.compile(r"^\s*ELSE\b", re.IGNORECASE)
+RE_END_IF = re.compile(r"^\s*END-IF\b", re.IGNORECASE)
+RE_EVALUATE = re.compile(r"^\s*EVALUATE\s+(.+)$", re.IGNORECASE)
+RE_WHEN = re.compile(r"^\s*WHEN\s+(.+)$", re.IGNORECASE)
+RE_END_EVALUATE = re.compile(r"^\s*END-EVALUATE\b", re.IGNORECASE)
+
+
+def is_comment(raw_line: str) -> bool:
+    stripped = raw_line.lstrip()
+    return stripped.startswith("*") and not stripped.startswith("*>")
+
+
+def comment_text(raw_line: str) -> str:
+    stripped = raw_line.lstrip()
+    return stripped.lstrip("*").strip()
+
+
+def extract_rules(text: str) -> list[dict]:
+    lines = text.splitlines()
+    rules: list[dict] = []
+    current_paragraph = "(no paragraph / top level)"
+    pending_comments: list[str] = []
+
+    i = 0
+    while i < len(lines):
+        raw = lines[i]
+        i += 1
+        if is_comment(raw):
+            pending_comments.append(comment_text(raw))
+            continue
+        code = strip_source_line(raw)
+        if not code.strip():
+            continue
+
+        para_m = RE_PARAGRAPH_HEADER.match(code)
+        if para_m and not any(
+            kw in code.upper() for kw in ("END-IF", "END-EVALUATE", "END-PERFORM")
+        ):
+            current_paragraph = para_m.group(1).upper()
+            pending_comments = []
+            continue
+
+        if_m = RE_IF.match(code)
+        if if_m:
+            branch_lines = [code.strip()]
+            depth = 1
+            has_else = False
+            start_line = i
+            while i < len(lines) and depth > 0:
+                nxt_raw = lines[i]
+                i += 1
+                if is_comment(nxt_raw):
+                    continue
+                nxt = strip_source_line(nxt_raw)
+                if not nxt.strip():
+                    continue
+                if RE_IF.match(nxt):
+                    depth += 1
+                elif RE_END_IF.match(nxt):
+                    depth -= 1
+                    if depth == 0:
+                        break
+                elif RE_ELSE.match(nxt) and depth == 1:
+                    has_else = True
+                branch_lines.append(nxt.strip())
+
+            truncated = depth > 0
+            rule: dict = {
+                "kind": "IF",
+                "paragraph": current_paragraph,
+                "condition": if_m.group(1).strip().rstrip("."),
+                "has_else": has_else,
+                "documented_intent": " ".join(pending_comments) if pending_comments else None,
+                "raw_snippet": "\n".join(branch_lines[:12]),
+                "truncated": truncated,
+            }
+            if truncated:
+                rule["review_note"] = (
+                    f"This IF block starting at line {start_line} never reached a matching "
+                    "END-IF before end of file (or before this scanner's nesting tracking gave "
+                    "up) - the condition/branches captured above may be incomplete. Verify "
+                    "manually against the source before trusting this rule."
+                )
+            rules.append(rule)
+            pending_comments = []
+            continue
+
+        eval_m = RE_EVALUATE.match(code)
+        if eval_m:
+            subject = eval_m.group(1).strip()
+            whens: list[str] = []
+            depth = 1
+            start_line = i
+            snippet_lines = [code.strip()]
+            while i < len(lines) and depth > 0:
+                nxt_raw = lines[i]
+                i += 1
+                if is_comment(nxt_raw):
+                    continue
+                nxt = strip_source_line(nxt_raw)
+                if not nxt.strip():
+                    continue
+                if RE_EVALUATE.match(nxt):
+                    depth += 1
+                elif RE_END_EVALUATE.match(nxt):
+                    depth -= 1
+                    if depth == 0:
+                        snippet_lines.append(nxt.strip())
+                        break
+                when_m = RE_WHEN.match(nxt)
+                if when_m and depth == 1:
+                    whens.append(when_m.group(1).strip())
+                snippet_lines.append(nxt.strip())
+
+            truncated = depth > 0
+            rule = {
+                "kind": "EVALUATE",
+                "paragraph": current_paragraph,
+                "subject": subject,
+                "when_clauses": whens,
+                "documented_intent": " ".join(pending_comments) if pending_comments else None,
+                "raw_snippet": "\n".join(snippet_lines[:20]),
+                "truncated": truncated,
+            }
+            if truncated:
+                rule["review_note"] = (
+                    f"This EVALUATE block starting at line {start_line} never reached a matching "
+                    "END-EVALUATE before end of file - the WHEN clauses captured above may be "
+                    "incomplete. Verify manually against the source before trusting this rule."
+                )
+            rules.append(rule)
+            pending_comments = []
+            continue
+
+        pending_comments = []
+
+    return rules
+
+
+def add_rule_todos(
+    rules: list[dict], todo_prefix: str = DEFAULT_TODO_PREFIX
+) -> None:
+    for rule in rules:
+        if not rule.get("truncated"):
+            continue
+        code = f"TRUNCATED_{rule['kind']}_BLOCK"
+        rule["todo"] = render_todo(
+            code,
+            rule["review_note"],
+            "Verify the complete source control-flow block before implementing this rule.",
+            comment_prefix="",
+            todo_prefix=todo_prefix,
+        )
+
+
+def render_markdown(
+    rules: list[dict],
+    file_label: str,
+    todo_prefix: str = DEFAULT_TODO_PREFIX,
+) -> str:
+    add_rule_todos(rules, todo_prefix)
+    lines = [f"# Business rules extracted from `{file_label}`", ""]
+    for idx, rule in enumerate(rules, start=1):
+        title = f"## Rule {idx} - {rule['kind']} in paragraph `{rule['paragraph']}`"
+        if rule.get("truncated"):
+            title += " :warning: TRUNCATED - MANUAL REVIEW REQUIRED"
+        lines.append(title)
+        if rule.get("todo"):
+            lines.append(f"> {rule['todo']}")
+        if rule.get("documented_intent"):
+            lines.append(f"> Original comment: {rule['documented_intent']}")
+        if rule["kind"] == "IF":
+            lines.append(f"- Condition: `{rule['condition']}`")
+            lines.append(f"- Has ELSE branch: {rule['has_else']}")
+        else:
+            lines.append(f"- Subject: `{rule['subject']}`")
+            for w in rule["when_clauses"]:
+                lines.append(f"  - WHEN `{w}`")
+        lines.append("")
+        lines.append("```cobol")
+        lines.append(rule["raw_snippet"])
+        lines.append("```")
+        lines.append("")
+    return "\n".join(lines)
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
+    parser.add_argument("file", help="Path to a single COBOL source file")
+    parser.add_argument("--format", choices=["json", "markdown"], default="markdown")
+    add_config_argument(parser)
+    args = parser.parse_args()
+
+    context = single_file_context_from_args(parser, args, "cobol")
+    path = context.path
+    text = path.read_text(encoding=context.source_encoding, errors="replace")
+    rules = extract_rules(text)
+    add_rule_todos(rules, context.todo_prefix)
+
+    if args.format == "json":
+        print(json.dumps(rules, indent=2))
+    else:
+        print(render_markdown(rules, str(path), context.todo_prefix))
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

mainframe_modernization_toolkit/cli/characterization_test_scaffolder.py

@@ -0,0 +1,180 @@
+#!/usr/bin/env python3
+"""characterization_test_scaffolder.py - Generate a "golden master" /
+characterization-test scaffold for a COBOL program, so its Python port can
+be verified against the original's observed behavior before cutover.
+
+Why this matters for migration
+-------------------------------
+"Golden master testing" (a.k.a. characterization testing) is the single
+most-cited safety practice for legacy rewrites (Michael Feathers'
+"Working Effectively with Legacy Code"; adopted at scale by IBM, Micro
+Focus, and every serious COBOL migration factory - see
+docs/RESEARCH.md). The idea: capture many real input/output pairs from
+the *existing* system, then run the same inputs through the newly-written
+Python code and diff the outputs byte-for-byte. This tool doesn't run the
+mainframe (out of scope/impossible here) - it scaffolds the *pytest*
+harness plus a fixtures directory convention, driven off:
+  - the program's LINKAGE SECTION (its CALL/USING contract), and
+  - any known sample data files placed under test-fixtures/golden/<PROGRAM>/
+
+so an agent (or engineer) only needs to drop in captured
+input/output pairs to get a working regression suite immediately.
+
+Usage
+-----
+    python3 characterization_test_scaffolder.py <file.cbl> [--out FILE]
+"""
+from __future__ import annotations
+
+import argparse
+import sys
+from pathlib import Path
+
+from ..common.cli_context import add_config_argument, single_file_context_from_args  # noqa: E402
+from ..common.cobol_parser import parse_cobol_source  # noqa: E402
+from ..common.warnings_model import DEFAULT_TODO_PREFIX, render_todo  # noqa: E402
+
+
+def to_snake(name: str) -> str:
+    """Program names may start with a digit; prefix so the generated
+    Python module reference stays a valid identifier."""
+    snake = name.lower().replace("-", "_")
+    if snake and snake[0].isdigit():
+        snake = f"p_{snake}"
+    return snake
+
+
+def render_pytest_scaffold(
+    program, todo_prefix: str = DEFAULT_TODO_PREFIX
+) -> str:
+    name = program.program_id or "UNKNOWN"
+    module = to_snake(name)
+    linkage = program.linkage_items or ["<no LINKAGE SECTION found - review manually>"]
+
+    lines = [
+        f'"""Golden-master characterization tests for {name}.',
+        "",
+        "How to use this scaffold:",
+        f"1. Capture real (or representative synthetic) input/output pairs from",
+        f"   the existing mainframe run of {name} - e.g. dump the LINKAGE SECTION",
+        "   record before/after a batch run - and drop them as JSON files under:",
+        f"       test-fixtures/golden/{name}/case_<n>.json",
+        '   each shaped as {"input": {...}, "expected_output": {...}}.',
+        "2. Implement / import the migrated Python equivalent below where",
+        "   marked TODO.",
+        "3. Run: pytest this file. Every golden case is parametrized",
+        "   automatically - no code changes needed as you add more fixtures.",
+        "",
+        f"LINKAGE SECTION contract observed in the COBOL source: {linkage}",
+    ]
+
+    if program.warnings:
+        lines.append("")
+        lines.append("Migration TODOs from the deterministic parser:")
+        for w in program.warnings:
+            lines.append(
+                "  " + render_todo(
+                    w.code,
+                    w.message,
+                    w.suggested_action,
+                    comment_prefix="",
+                    todo_prefix=todo_prefix,
+                )
+            )
+        if program.program_id is None:
+            lines.append(
+                "  NOTE: no PROGRAM-ID was found, so 'UNKNOWN' is used as a placeholder "
+                "throughout this file - fix the source and regenerate before relying on this."
+            )
+
+    lines += [
+        '"""',
+        "from __future__ import annotations",
+        "",
+        "import json",
+        "from pathlib import Path",
+        "",
+        "import pytest",
+        "",
+        render_todo(
+            "CHARACTERIZATION_IMPORT",
+            f"The migrated implementation for {name} is not imported.",
+            "Import the approved implementation after its callable contract is established.",
+            todo_prefix=todo_prefix,
+        ),
+        f"# from {module} import {module}_main",
+        "",
+        f'GOLDEN_DIR = Path(__file__).parent / "golden" / "{name}"',
+        "",
+        "",
+        "def _load_golden_cases() -> list[dict]:",
+        "    if not GOLDEN_DIR.exists():",
+        "        return []",
+        '    return [json.loads(p.read_text()) for p in sorted(GOLDEN_DIR.glob("case_*.json"))]',
+        "",
+        "",
+        "GOLDEN_CASES = _load_golden_cases()",
+        "",
+        "",
+        "@pytest.mark.skipif(not GOLDEN_CASES, reason=\"No golden fixtures captured yet - see module docstring\")",
+        '@pytest.mark.parametrize("case", GOLDEN_CASES, ids=lambda c: c.get("name", "case"))',
+        "def test_matches_golden_master(case: dict) -> None:",
+        f'    """Replays a captured {name} input and diffs against the recorded',
+        "    mainframe output. This is the safety net that allows the COBOL",
+        "    source to eventually be retired.",
+        '    """',
+        "    " + render_todo(
+            "CHARACTERIZATION_EXECUTION",
+            f"Golden cases are not yet executed through {module}_main.",
+            "Call the migrated implementation with the verified input adapter.",
+            todo_prefix=todo_prefix,
+        ),
+        "    actual_output = None  # placeholder until the port is implemented",
+        "    error_msg = (",
+        f'        f"{name} Python port diverges from golden master for case "',
+        '        f"{case.get(\'name\')}"',
+        "    )",
+        "    assert actual_output == case[\"expected_output\"], error_msg",
+        "",
+        "",
+        "def test_golden_fixtures_present() -> None:",
+        f'    """Fails loudly if nobody has captured fixtures for {name} yet -',
+        "    better than a silently-skipped, false-green test suite.",
+        '    """',
+        "    assert GOLDEN_CASES, (",
+        '        f"No golden-master fixtures found under {GOLDEN_DIR}. Capture real "',
+        '        "input/output pairs before porting business logic."',
+        "    )",
+    ]
+    return "\n".join(lines)
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
+    parser.add_argument("file")
+    parser.add_argument("--out", help="Write scaffold to this file instead of stdout")
+    add_config_argument(parser)
+    args = parser.parse_args()
+
+    context = single_file_context_from_args(parser, args, "cobol")
+    path = context.path
+    text = path.read_text(encoding=context.source_encoding, errors="replace")
+    program = parse_cobol_source(text, file_path=str(path))
+    blocking = [warning for warning in program.warnings if warning.severity == "ERROR"]
+    if blocking:
+        parser.error(
+            "Characterization scaffold generation is blocked by parser errors: "
+            + "; ".join(f"[{warning.code}] {warning.message}" for warning in blocking)
+        )
+    scaffold = render_pytest_scaffold(program, context.todo_prefix)
+
+    if args.out:
+        Path(args.out).write_text(scaffold, encoding="utf-8")
+        print(f"Wrote characterization test scaffold to {args.out}")
+    else:
+        print(scaffold)
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())