mainframe-migration-toolkit 0.2.0__py3-none-any.whl

mainframe_toolkit/cli.py

@@ -0,0 +1,361 @@
+from __future__ import annotations
+
+import argparse
+import csv
+import hashlib
+import json
+import os
+import shutil
+import subprocess
+import sys
+from pathlib import Path
+from typing import Any, Sequence
+
+from .jcl import parse_jcl
+from .sequential import RecordLayout, convert_sequential
+from .similarity import (
+    analyze_codebase,
+    apply_ai_reviews,
+    write_ai_review_payload,
+    write_results,
+)
+from .specs import inspect_project, write_inspection
+from .synthetic import synthetic_rows
+from .workspace import init_workspace
+
+
+def _json_dump(value: Any, path: str | Path | None = None) -> None:
+    content = json.dumps(value, ensure_ascii=False, indent=2, default=str) + "\n"
+    if path is None or str(path) == "-":
+        sys.stdout.write(content)
+        return
+    output = Path(path)
+    output.parent.mkdir(parents=True, exist_ok=True)
+    output.write_text(content, encoding="utf-8")
+
+
+def _inspect(args: argparse.Namespace) -> int:
+    result = inspect_project(args.project, strict=args.strict)
+    if args.output:
+        write_inspection(result, args.output)
+    else:
+        _json_dump(result.to_dict())
+    return 0 if result.valid else 2
+
+
+def _init_workspace(args: argparse.Namespace) -> int:
+    result = init_workspace(args.destination, force=args.force)
+    _json_dump(result.to_dict())
+    return 0
+
+
+def _jcl_parse(args: argparse.Namespace) -> int:
+    _json_dump(parse_jcl(args.source).to_dict(), args.output)
+    return 0
+
+
+def _copybook_parse(args: argparse.Namespace) -> int:
+    from .copybook import layout_to_dict, parse_copybook
+
+    parsed = parse_copybook(args.source)
+    _json_dump(layout_to_dict(parsed), args.output)
+    return 0
+
+
+def _sequential_convert(args: argparse.Namespace) -> int:
+    layout = RecordLayout.from_json(args.layout)
+    result = convert_sequential(
+        args.source,
+        args.destination,
+        layout,
+        output_format=args.format,
+        mode=args.mode,
+        delimiter=args.delimiter,
+        on_error=args.on_error,
+        batch_size=args.batch_size,
+    )
+    _json_dump(result)
+    return 0
+
+
+def _similarity_scan(args: argparse.Namespace) -> int:
+    report = analyze_codebase(
+        args.root,
+        threshold=args.threshold,
+        shingle_size=args.shingle_size,
+        exhaustive_limit=args.exhaustive_limit,
+    )
+    write_results(report, args.json, args.csv)
+    write_ai_review_payload(report, args.review_request)
+    _json_dump(
+        {
+            **report.statistics,
+            "json": str(Path(args.json)),
+            "csv": str(Path(args.csv)),
+            "review_request": str(Path(args.review_request)),
+        }
+    )
+    return 0
+
+
+def _similarity_apply(args: argparse.Namespace) -> int:
+    json_output = args.json or args.report
+    csv_output = args.csv or str(Path(json_output).with_suffix(".csv"))
+    result = apply_ai_reviews(
+        args.report,
+        args.reviews,
+        json_path=json_output,
+        csv_path=csv_output,
+    )
+    records = result.records if hasattr(result, "records") else result.get("arquivos", [])
+    missing = sum(not row.get("parecer_ia") or not row.get("justificativa_ia") for row in records)
+    _json_dump({"json": json_output, "csv": csv_output, "rows": len(records), "reviews_missing": missing})
+    return 0 if missing == 0 else 1
+
+
+def _synthetic(args: argparse.Namespace) -> int:
+    payload = json.loads(Path(args.schema).read_text(encoding="utf-8"))
+    schema = payload.get("schema", payload) if isinstance(payload, dict) else payload
+    if not isinstance(schema, dict) or not all(isinstance(value, str) for value in schema.values()):
+        raise ValueError("schema must be a JSON object mapping field names to types")
+    rows = synthetic_rows(schema, count=args.rows, seed=args.seed)
+    output = Path(args.output)
+    output.parent.mkdir(parents=True, exist_ok=True)
+    if args.format == "jsonl":
+        with output.open("w", encoding="utf-8") as stream:
+            for row in rows:
+                stream.write(json.dumps(row, ensure_ascii=False, default=str) + "\n")
+    else:
+        with output.open("w", encoding="utf-8", newline="") as stream:
+            writer = csv.DictWriter(stream, fieldnames=list(schema), delimiter=args.delimiter)
+            writer.writeheader()
+            writer.writerows(rows)
+    digest = hashlib.sha256(output.read_bytes()).hexdigest()
+    manifest = Path(args.manifest) if args.manifest else output.with_suffix(output.suffix + ".manifest.json")
+    _json_dump(
+        {
+            "synthetic": True,
+            "output": str(output),
+            "format": args.format,
+            "rows": len(rows),
+            "seed": args.seed,
+            "schema": schema,
+            "sha256": digest,
+        },
+        manifest,
+    )
+    _json_dump({"output": str(output), "manifest": str(manifest), "rows": len(rows), "seed": args.seed})
+    return 0
+
+
+def _toolkit_root() -> Path:
+    override = os.environ.get("MAINFRAME_TOOLKIT_ROOT")
+    if override:
+        return Path(override).resolve()
+    candidate = Path(__file__).resolve().parents[2]
+    return candidate if (candidate / "validator-java" / "pom.xml").is_file() else Path.cwd()
+
+
+def _validator_dir(value: str | None) -> Path:
+    return Path(value).resolve() if value else _toolkit_root() / "validator-java"
+
+
+def _maven_executable(directory: Path) -> str:
+    wrapper = directory / ("mvnw.cmd" if os.name == "nt" else "mvnw")
+    if wrapper.is_file():
+        return str(wrapper)
+    executable = shutil.which("mvn") or shutil.which("mvn.cmd")
+    if not executable:
+        raise RuntimeError("Maven 3.9+ or the validator-java Maven wrapper is required")
+    return executable
+
+
+def _golden_build(args: argparse.Namespace) -> int:
+    directory = _validator_dir(args.validator_dir)
+    pom = directory / "pom.xml"
+    if not pom.is_file():
+        raise FileNotFoundError(pom)
+    command = [_maven_executable(directory), "-q", "-f", str(pom), "package"]
+    if args.skip_tests:
+        command.insert(-1, "-DskipTests")
+    process_environment = None
+    java_home = getattr(args, "java_home", None)
+    if java_home:
+        process_environment = os.environ.copy()
+        process_environment["JAVA_HOME"] = str(Path(java_home).resolve())
+    return subprocess.run(command, check=False, env=process_environment).returncode
+
+
+def _resolve_validator_jar(args: argparse.Namespace) -> Path:
+    if args.jar:
+        jar = Path(args.jar).resolve()
+    elif os.environ.get("MAINFRAME_VALIDATOR_JAR"):
+        jar = Path(os.environ["MAINFRAME_VALIDATOR_JAR"]).resolve()
+    else:
+        jar = _validator_dir(args.validator_dir) / "target" / "dataset-validator.jar"
+    if not jar.is_file() and args.build:
+        code = _golden_build(
+            argparse.Namespace(
+                validator_dir=args.validator_dir,
+                skip_tests=False,
+                java_home=(str(Path(args.java).resolve().parent.parent) if args.java else None),
+            )
+        )
+        if code != 0:
+            raise RuntimeError(f"validator-java build failed with exit code {code}")
+    if not jar.is_file():
+        raise FileNotFoundError(jar)
+    return jar
+
+
+def _golden_validate(args: argparse.Namespace) -> int:
+    requested_java = args.java or os.environ.get("MAINFRAME_JAVA")
+    java = str(Path(requested_java).resolve()) if requested_java else shutil.which("java")
+    if not java:
+        raise RuntimeError("Java 17+ is required to run the dataset validator")
+    command = [java, "-jar", str(_resolve_validator_jar(args)), args.expected, args.actual]
+    scalar_options = {
+        "--format": args.format,
+        "--expected-format": args.expected_format,
+        "--actual-format": args.actual_format,
+        "--delimiter": args.delimiter,
+        "--expected-delimiter": args.expected_delimiter,
+        "--actual-delimiter": args.actual_delimiter,
+        "--header": args.header,
+        "--expected-header": args.expected_header,
+        "--actual-header": args.actual_header,
+        "--charset": args.charset,
+        "--max-differences": args.max_differences,
+        "--report": args.report,
+    }
+    for option, value in scalar_options.items():
+        if value is not None:
+            command.extend((option, str(value)))
+    if args.no_header:
+        command.append("--no-header")
+    if args.trim:
+        command.append("--trim")
+    if args.normalize_decimal:
+        command.append("--normalize-decimal")
+    for value in args.normalize:
+        command.extend(("--normalize", value))
+    for value in args.key:
+        command.extend(("--key", value))
+    return subprocess.run(command, check=False).returncode
+
+
+def _parser() -> argparse.ArgumentParser:
+    parser = argparse.ArgumentParser(prog="mainframe-toolkit")
+    subcommands = parser.add_subparsers(dest="command", required=True)
+
+    initialize = subcommands.add_parser("init-workspace", help="materialize a migration workspace")
+    initialize.add_argument("destination", nargs="?", default=".")
+    initialize.add_argument(
+        "--force",
+        action="store_true",
+        help="update conflicting toolkit-managed files without replacing project inputs",
+    )
+    initialize.set_defaults(handler=_init_workspace)
+
+    inspect = subcommands.add_parser("inspect", help="validate the migration project contract")
+    inspect.add_argument("project", nargs="?", default=".")
+    inspect.add_argument("--output", "-o")
+    inspect.add_argument("--strict", action="store_true")
+    inspect.set_defaults(handler=_inspect)
+
+    jcl = subcommands.add_parser("jcl", help="JCL source tools").add_subparsers(dest="jcl_command", required=True)
+    jcl_parse = jcl.add_parser("parse")
+    jcl_parse.add_argument("source")
+    jcl_parse.add_argument("--output", "-o")
+    jcl_parse.set_defaults(handler=_jcl_parse)
+
+    copybook = subcommands.add_parser("copybook", help="copybook layout tools").add_subparsers(dest="copybook_command", required=True)
+    copybook_parse = copybook.add_parser("parse")
+    copybook_parse.add_argument("source")
+    copybook_parse.add_argument("--output", "-o")
+    copybook_parse.set_defaults(handler=_copybook_parse)
+
+    sequential = subcommands.add_parser("sequential", help="sequential dataset tools").add_subparsers(dest="sequential_command", required=True)
+    convert = sequential.add_parser("convert")
+    convert.add_argument("source")
+    convert.add_argument("destination")
+    convert.add_argument("--layout", required=True)
+    convert.add_argument("--format", choices=("csv", "parquet"))
+    convert.add_argument("--mode", choices=("line", "fixed"), default="line")
+    convert.add_argument("--delimiter", default=",")
+    convert.add_argument("--on-error", choices=("raise", "skip", "null"), default="raise")
+    convert.add_argument("--batch-size", type=int, default=10_000)
+    convert.set_defaults(handler=_sequential_convert)
+
+    similarity = subcommands.add_parser("similarity", help="deterministic source clustering").add_subparsers(dest="similarity_command", required=True)
+    scan = similarity.add_parser("scan")
+    scan.add_argument("root")
+    scan.add_argument("--threshold", type=float, default=80.0)
+    scan.add_argument("--shingle-size", type=int, default=3)
+    scan.add_argument("--exhaustive-limit", type=int, default=80)
+    scan.add_argument("--json", default="similarity.json")
+    scan.add_argument("--csv", default="similarity.csv")
+    scan.add_argument("--review-request", default="similarity-review-request.json")
+    scan.set_defaults(handler=_similarity_scan)
+    review = similarity.add_parser("apply-review")
+    review.add_argument("report")
+    review.add_argument("reviews")
+    review.add_argument("--json")
+    review.add_argument("--csv")
+    review.set_defaults(handler=_similarity_apply)
+
+    synthetic = subcommands.add_parser("synthetic", help="create deterministic fallback rows")
+    synthetic.add_argument("--schema", required=True)
+    synthetic.add_argument("--output", required=True)
+    synthetic.add_argument("--manifest")
+    synthetic.add_argument("--rows", type=int, default=10)
+    synthetic.add_argument("--seed", default="0")
+    synthetic.add_argument("--format", choices=("jsonl", "csv"), default="jsonl")
+    synthetic.add_argument("--delimiter", default=",")
+    synthetic.set_defaults(handler=_synthetic)
+
+    golden = subcommands.add_parser("golden", help="Java cell-level dataset validator").add_subparsers(dest="golden_command", required=True)
+    build = golden.add_parser("build")
+    build.add_argument("--validator-dir")
+    build.add_argument("--java-home")
+    build.add_argument("--skip-tests", action="store_true")
+    build.set_defaults(handler=_golden_build)
+    validate = golden.add_parser("validate")
+    validate.add_argument("expected")
+    validate.add_argument("actual")
+    validate.add_argument("--jar")
+    validate.add_argument("--java")
+    validate.add_argument("--validator-dir")
+    validate.add_argument("--build", action=argparse.BooleanOptionalAction, default=True)
+    validate.add_argument("--format", choices=("auto", "csv", "parquet"), default="auto")
+    validate.add_argument("--expected-format", choices=("auto", "csv", "parquet"))
+    validate.add_argument("--actual-format", choices=("auto", "csv", "parquet"))
+    validate.add_argument("--delimiter", default=",")
+    validate.add_argument("--expected-delimiter")
+    validate.add_argument("--actual-delimiter")
+    validate.add_argument("--no-header", action="store_true")
+    validate.add_argument("--header", choices=("true", "false"))
+    validate.add_argument("--expected-header", choices=("true", "false"))
+    validate.add_argument("--actual-header", choices=("true", "false"))
+    validate.add_argument("--charset", default="UTF-8")
+    validate.add_argument("--normalize", action="append", default=[])
+    validate.add_argument("--trim", action="store_true")
+    validate.add_argument("--normalize-decimal", action="store_true")
+    validate.add_argument("--key", action="append", default=[])
+    validate.add_argument("--max-differences", type=int, default=100)
+    validate.add_argument("--report", "-r", default="-")
+    validate.set_defaults(handler=_golden_validate)
+    return parser
+
+
+def main(argv: Sequence[str] | None = None) -> int:
+    try:
+        args = _parser().parse_args(argv)
+        return int(args.handler(args))
+    except (FileNotFoundError, ValueError, RuntimeError) as exc:
+        sys.stderr.write(f"error: {exc}\n")
+        return 2
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

mainframe_toolkit/cobol.py

@@ -0,0 +1,106 @@
+from __future__ import annotations
+
+from decimal import Decimal, InvalidOperation, ROUND_DOWN, ROUND_HALF_UP
+from typing import Any
+
+from .errors import DataFormatError
+
+
+_POSITIVE_OVERPUNCH = "{ABCDEFGHI"
+_NEGATIVE_OVERPUNCH = "}JKLMNOPQR"
+
+
+def cobol_decimal(value: Any, scale: int = 0, default: Decimal | None = None) -> Decimal:
+    """Convert a display numeric value without using binary floating point."""
+    if value is None or (isinstance(value, str) and not value.strip()):
+        if default is not None:
+            return default
+        raise DataFormatError("empty COBOL numeric value")
+    text = str(value).strip().replace(",", ".")
+    if len(text) > 1 and text[-1] in "+-" and text[0] not in "+-":
+        text = text[-1] + text[:-1]
+    try:
+        number = Decimal(text)
+    except InvalidOperation as exc:
+        raise DataFormatError(f"invalid COBOL numeric value: {value!r}") from exc
+    if scale and "." not in text:
+        number = number.scaleb(-scale)
+    return number
+
+
+def quantize_cobol(
+    value: Any,
+    scale: int,
+    *,
+    rounded: bool = False,
+) -> Decimal:
+    mode = ROUND_HALF_UP if rounded else ROUND_DOWN
+    quantum = Decimal(1).scaleb(-scale)
+    return cobol_decimal(value).quantize(quantum, rounding=mode)
+
+
+def is_numeric(value: Any, *, signed: bool = True, decimal: bool = True) -> bool:
+    if value is None:
+        return False
+    text = str(value).strip()
+    if not text:
+        return False
+    if signed and text[:1] in "+-":
+        text = text[1:]
+    if decimal and text.count(".") <= 1:
+        text = text.replace(".", "", 1)
+    return bool(text) and text.isdigit()
+
+
+def decode_overpunch(value: str, scale: int = 0) -> Decimal:
+    text = value.strip()
+    if not text:
+        raise DataFormatError("empty overpunch value")
+    last = text[-1].upper()
+    if last in _POSITIVE_OVERPUNCH:
+        digit = _POSITIVE_OVERPUNCH.index(last)
+        sign = 1
+    elif last in _NEGATIVE_OVERPUNCH:
+        digit = _NEGATIVE_OVERPUNCH.index(last)
+        sign = -1
+    elif last.isdigit():
+        digit = int(last)
+        sign = 1
+    else:
+        raise DataFormatError(f"invalid overpunch sign: {last!r}")
+    digits = f"{text[:-1]}{digit}"
+    if not digits.isdigit():
+        raise DataFormatError(f"invalid overpunch digits: {value!r}")
+    return Decimal(sign * int(digits)).scaleb(-scale)
+
+
+def decode_packed_decimal(data: bytes, scale: int = 0) -> Decimal:
+    if not data:
+        raise DataFormatError("empty packed decimal")
+    nibbles: list[int] = []
+    for byte in data:
+        nibbles.extend((byte >> 4, byte & 0x0F))
+    sign_nibble = nibbles.pop()
+    if sign_nibble in (0x0B, 0x0D):
+        sign = -1
+    elif sign_nibble in (0x0A, 0x0C, 0x0E, 0x0F):
+        sign = 1
+    else:
+        raise DataFormatError(f"invalid packed decimal sign: {sign_nibble:X}")
+    if any(digit > 9 for digit in nibbles):
+        raise DataFormatError("invalid digit in packed decimal")
+    integer = int("".join(str(digit) for digit in nibbles) or "0")
+    return Decimal(sign * integer).scaleb(-scale)
+
+
+def encode_packed_decimal(value: Any, digits: int, scale: int = 0) -> bytes:
+    number = quantize_cobol(value, scale, rounded=False)
+    negative = number < 0
+    scaled = abs(int(number.scaleb(scale)))
+    digit_text = f"{scaled:0{digits}d}"
+    if len(digit_text) > digits:
+        raise DataFormatError(f"value {value!r} exceeds PIC 9({digits})")
+    nibbles = [int(char) for char in digit_text] + [0x0D if negative else 0x0C]
+    if len(nibbles) % 2:
+        nibbles.insert(0, 0)
+    return bytes((nibbles[index] << 4) | nibbles[index + 1] for index in range(0, len(nibbles), 2))