mainframe-migration-toolkit 0.2.0__py3-none-any.whl

mainframe_toolkit/copybook.py

@@ -0,0 +1,558 @@
+from __future__ import annotations
+
+import json
+import re
+from dataclasses import asdict, dataclass, field
+from pathlib import Path
+from typing import Any
+
+from .sequential import FieldSpec, RecordLayout
+
+
+class CopybookParseError(ValueError):
+    pass
+
+
+@dataclass
+class _Entry:
+    level: int
+    name: str
+    body: str
+    picture: str | None
+    usage: str | None
+    occurs: int
+    redefines: str | None
+    separate_sign: bool
+    children: list["_Entry"] = field(default_factory=list)
+
+
+@dataclass(frozen=True)
+class _Picture:
+    digits: int
+    characters: int
+    scale: int
+    signed: bool
+    textual: bool
+
+
+_ENTRY_RE = re.compile(
+    r"(?ims)^[ \t]*(?P<level>\d{1,2})[ \t]+"
+    r"(?P<name>[A-Z0-9][A-Z0-9_-]*)\b(?P<body>.*?)"
+    r"(?=^[ \t]*\d{1,2}[ \t]+[A-Z0-9]|\Z)"
+)
+_PICTURE_RE = re.compile(
+    r"\bPIC(?:TURE)?(?:\s+IS)?\s+"
+    r"(?P<picture>(?:[AX9SVP](?:\(\s*\d+\s*\))?)+)",
+    re.IGNORECASE,
+)
+_PICTURE_TOKEN_RE = re.compile(r"([AX9SVP])(?:\(\s*(\d+)\s*\))?", re.IGNORECASE)
+_USAGE_RE = re.compile(
+    r"\b(?:USAGE\s+(?:IS\s+)?)?"
+    r"(PACKED-DECIMAL|COMPUTATIONAL-[1-5]|COMP-[1-5]|"
+    r"COMPUTATIONAL|COMP|BINARY|DISPLAY)\b",
+    re.IGNORECASE,
+)
+
+
+def parse_copybook(
+    source: str | Path,
+    *,
+    source_format: str = "auto",
+    record_name: str | None = None,
+    encoding: str = "utf-8",
+    source_encoding: str = "utf-8",
+    include_fillers: bool = True,
+) -> RecordLayout:
+    text = _read_source(source, source_encoding)
+    return parse_copybook_text(
+        text,
+        source_format=source_format,
+        record_name=record_name,
+        encoding=encoding,
+        include_fillers=include_fillers,
+    )
+
+
+def parse_copybook_file(
+    path: str | Path,
+    **options: Any,
+) -> RecordLayout:
+    return parse_copybook(Path(path), **options)
+
+
+def parse_copybook_text(
+    text: str,
+    *,
+    source_format: str = "auto",
+    record_name: str | None = None,
+    encoding: str = "utf-8",
+    include_fillers: bool = True,
+) -> RecordLayout:
+    entries = _parse_entries(_normalise_source(text, source_format))
+    root = _select_root(entries, record_name)
+    fields, record_length = _place_node(root, 0, None, ())
+    fields = _unique_fields(fields)
+    if not include_fillers:
+        fields = [item for item in fields if not _is_filler(item.name)]
+    return RecordLayout(tuple(fields), record_length=record_length, encoding=encoding)
+
+
+def layout_to_dict(layout: RecordLayout) -> dict[str, Any]:
+    return {
+        "fields": [asdict(item) for item in layout.fields],
+        "record_length": layout.record_length,
+        "encoding": layout.encoding,
+    }
+
+
+def layout_to_json(
+    layout: RecordLayout,
+    path: str | Path | None = None,
+    *,
+    indent: int | None = 2,
+) -> str:
+    rendered = json.dumps(layout_to_dict(layout), ensure_ascii=False, indent=indent) + "\n"
+    if path is not None:
+        destination = Path(path)
+        destination.parent.mkdir(parents=True, exist_ok=True)
+        destination.write_text(rendered, encoding="utf-8")
+    return rendered
+
+
+def write_layout_json(layout: RecordLayout, path: str | Path) -> None:
+    layout_to_json(layout, path)
+
+
+def _read_source(source: str | Path, encoding: str) -> str:
+    if isinstance(source, Path):
+        return source.read_text(encoding=encoding)
+    if "\n" in source or "\r" in source:
+        return source
+    try:
+        path = Path(source)
+        if path.is_file():
+            return path.read_text(encoding=encoding)
+    except OSError:
+        pass
+    return source
+
+
+def _normalise_source(text: str, source_format: str) -> str:
+    selected = source_format.lower().replace("-", "_")
+    if selected not in {"auto", "fixed", "free"}:
+        raise ValueError("source_format must be auto, fixed, or free")
+    lines = text.expandtabs(8).splitlines()
+    directive = _source_directive(lines)
+    if selected == "auto":
+        selected = directive or _detect_source_format(lines)
+
+    cleaned: list[str] = []
+    for raw in lines:
+        if re.search(r">>\s*SOURCE\s+FORMAT", raw, re.IGNORECASE):
+            continue
+        if selected == "fixed":
+            padded = raw.ljust(7)
+            indicator = padded[6]
+            if indicator in {"*", "/", "D", "d"}:
+                continue
+            content = padded[7:72]
+            content = _strip_inline_comment(content)
+            if indicator == "-" and cleaned:
+                cleaned[-1] = f"{cleaned[-1].rstrip()} {content.lstrip()}"
+                continue
+        else:
+            content = _strip_inline_comment(raw)
+            if content.lstrip().startswith(("*", "/")):
+                continue
+        if content.strip():
+            cleaned.append(content.rstrip())
+    return _break_inline_entries("\n".join(cleaned))
+
+
+def _source_directive(lines: list[str]) -> str | None:
+    for line in lines:
+        match = re.search(
+            r">>\s*SOURCE\s+FORMAT(?:\s+IS)?\s+(FREE|FIXED)",
+            line,
+            re.IGNORECASE,
+        )
+        if match:
+            return match.group(1).lower()
+    return None
+
+
+def _detect_source_format(lines: list[str]) -> str:
+    if any(re.match(r"^\d{6}.", line) for line in lines):
+        return "fixed"
+    early_entry = re.compile(r"^\s{0,6}\d{1,2}\s+[A-Z0-9]", re.IGNORECASE)
+    if any(early_entry.match(line) for line in lines):
+        return "free"
+    if any(len(line) > 6 and line[6] in {"*", "/", "-"} for line in lines):
+        return "fixed"
+    if any(len(line) > 7 and not line[:7].strip() for line in lines):
+        return "fixed"
+    return "free"
+
+
+def _strip_inline_comment(line: str) -> str:
+    quote: str | None = None
+    index = 0
+    while index < len(line) - 1:
+        character = line[index]
+        if character in {"'", '"'}:
+            if quote == character and index + 1 < len(line) and line[index + 1] == character:
+                index += 2
+                continue
+            quote = None if quote == character else character if quote is None else quote
+        if quote is None and line[index : index + 2] == "*>":
+            return line[:index]
+        index += 1
+    return line
+
+
+def _break_inline_entries(source: str) -> str:
+    next_entry = re.compile(
+        r"\s+(?:0?[1-9]|[1-4]\d|66|77|78|88)\s+[A-Z0-9]",
+        re.IGNORECASE,
+    )
+    result: list[str] = []
+    quote: str | None = None
+    index = 0
+    while index < len(source):
+        character = source[index]
+        if character in {"'", '"'}:
+            if quote == character and index + 1 < len(source) and source[index + 1] == character:
+                result.extend((character, character))
+                index += 2
+                continue
+            quote = None if quote == character else character if quote is None else quote
+        result.append(character)
+        if character == "." and quote is None:
+            match = next_entry.match(source, index + 1)
+            if match:
+                result.append("\n")
+        index += 1
+    return "".join(result)
+
+
+def _parse_entries(source: str) -> list[_Entry]:
+    roots: list[_Entry] = []
+    stack: list[_Entry] = []
+    filler_count = 0
+    for match in _ENTRY_RE.finditer(source.upper()):
+        level = int(match.group("level"))
+        if level in {66, 78, 88}:
+            continue
+        if level == 0 or level > 49 and level != 77:
+            continue
+        name = match.group("name")
+        if name == "FILLER":
+            filler_count += 1
+            name = "FILLER" if filler_count == 1 else f"FILLER-{filler_count}"
+        body = " ".join(match.group("body").replace(".", " ").split())
+        picture_match = _PICTURE_RE.search(body)
+        picture = None
+        if picture_match:
+            picture = re.sub(r"\s+", "", picture_match.group("picture"))
+        usage_match = _USAGE_RE.search(body)
+        usage = _normalise_usage(usage_match.group(1)) if usage_match else None
+        occurs = _fixed_occurs(body)
+        redefines_match = re.search(r"\bREDEFINES\s+([A-Z0-9][A-Z0-9_-]*)\b", body)
+        entry = _Entry(
+            level=level,
+            name=name,
+            body=body,
+            picture=picture,
+            usage=usage,
+            occurs=occurs,
+            redefines=redefines_match.group(1) if redefines_match else None,
+            separate_sign=bool(re.search(r"\bSIGN\b.*?\bSEPARATE\b", body)),
+        )
+
+        if level == 77:
+            stack.clear()
+        else:
+            while stack and level <= stack[-1].level:
+                stack.pop()
+        if stack:
+            if stack[-1].picture or stack[-1].usage in {"comp_1", "comp_2"}:
+                raise CopybookParseError(f"elementary item {stack[-1].name} cannot contain {name}")
+            stack[-1].children.append(entry)
+        else:
+            roots.append(entry)
+        stack.append(entry)
+
+    if not roots:
+        raise CopybookParseError("copybook contains no record entries")
+    return roots
+
+
+def _normalise_usage(value: str) -> str:
+    usage = value.upper().replace("COMPUTATIONAL", "COMP")
+    return {
+        "DISPLAY": "display",
+        "PACKED-DECIMAL": "packed_decimal",
+        "COMP-3": "packed_decimal",
+        "BINARY": "binary",
+        "COMP": "binary",
+        "COMP-4": "binary",
+        "COMP-5": "binary",
+        "COMP-1": "comp_1",
+        "COMP-2": "comp_2",
+    }[usage]
+
+
+def _fixed_occurs(body: str) -> int:
+    match = re.search(r"\bOCCURS\s+(\d+)\b", body)
+    if not match:
+        return 1
+    tail = body[match.end() :]
+    if re.match(r"\s+TO\s+\d+", tail) or re.search(r"\bDEPENDING\s+ON\b", tail):
+        raise CopybookParseError("variable OCCURS is not supported")
+    count = int(match.group(1))
+    if count < 1:
+        raise CopybookParseError("OCCURS must be greater than zero")
+    return count
+
+
+def _select_root(entries: list[_Entry], record_name: str | None) -> _Entry:
+    if record_name is None:
+        return entries[0]
+    wanted = record_name.upper().replace("_", "-")
+    pending = list(entries)
+    while pending:
+        entry = pending.pop(0)
+        if entry.name.replace("_", "-") == wanted:
+            return entry
+        pending[0:0] = entry.children
+    raise CopybookParseError(f"record {record_name!r} was not found")
+
+
+def _place_node(
+    entry: _Entry,
+    start: int,
+    inherited_usage: str | None,
+    occurrence_path: tuple[int, ...],
+) -> tuple[list[FieldSpec], int]:
+    usage = entry.usage or inherited_usage
+    if entry.children:
+        if entry.picture is not None or usage in {"comp_1", "comp_2"}:
+            raise CopybookParseError(f"invalid group item {entry.name}")
+        all_fields: list[FieldSpec] = []
+        one_length: int | None = None
+        for occurrence in range(1, entry.occurs + 1):
+            suffix = occurrence_path + ((occurrence,) if entry.occurs > 1 else ())
+            base = start + (one_length or 0) * (occurrence - 1)
+            fields, length = _place_children(entry.children, base, usage, suffix)
+            if one_length is None:
+                one_length = length
+            elif length != one_length:
+                raise CopybookParseError(f"non-deterministic length for {entry.name}")
+            all_fields.extend(fields)
+        return all_fields, (one_length or 0) * entry.occurs
+
+    if usage in {"comp_1", "comp_2"} and entry.picture is None:
+        length = 4 if usage == "comp_1" else 8
+        fields = []
+        for occurrence in range(1, entry.occurs + 1):
+            suffix = occurrence_path + ((occurrence,) if entry.occurs > 1 else ())
+            fields.append(
+                _field(
+                    entry,
+                    start + (occurrence - 1) * length,
+                    length,
+                    "bytes",
+                    0,
+                    True,
+                    suffix,
+                    "latin-1",
+                )
+            )
+        return fields, length * entry.occurs
+    if entry.picture is None:
+        raise CopybookParseError(f"group item {entry.name} has no subordinate entries")
+
+    picture = _parse_picture(entry.picture)
+    selected_usage = usage or "display"
+    length, kind, field_encoding = _physical_type(picture, selected_usage, entry.separate_sign)
+    all_fields = []
+    for occurrence in range(1, entry.occurs + 1):
+        suffix = occurrence_path + ((occurrence,) if entry.occurs > 1 else ())
+        all_fields.append(
+            _field(
+                entry,
+                start + (occurrence - 1) * length,
+                length,
+                kind,
+                picture.scale,
+                picture.signed,
+                suffix,
+                field_encoding,
+            )
+        )
+    return all_fields, length * entry.occurs
+
+
+def _place_children(
+    children: list[_Entry],
+    start: int,
+    inherited_usage: str | None,
+    occurrence_path: tuple[int, ...],
+) -> tuple[list[FieldSpec], int]:
+    cursor = start
+    extent = start
+    positions: dict[str, tuple[int, int]] = {}
+    fields: list[FieldSpec] = []
+    for child in children:
+        child_start = cursor
+        if child.redefines:
+            target = positions.get(child.redefines)
+            if target is None:
+                raise CopybookParseError(
+                    f"{child.name} redefines unknown item {child.redefines}"
+                )
+            child_start = target[0]
+        child_fields, child_length = _place_node(
+            child, child_start, inherited_usage, occurrence_path
+        )
+        fields.extend(child_fields)
+        positions[child.name] = (child_start, child_length)
+        extent = max(extent, child_start + child_length)
+        cursor = max(cursor, child_start + child_length)
+    return fields, extent - start
+
+
+def _parse_picture(value: str) -> _Picture:
+    position = 0
+    tokens: list[tuple[str, int]] = []
+    for match in _PICTURE_TOKEN_RE.finditer(value):
+        if match.start() != position:
+            raise CopybookParseError(f"unsupported PICTURE {value!r}")
+        count = int(match.group(2) or 1)
+        if count < 1:
+            raise CopybookParseError(f"invalid repetition in PICTURE {value!r}")
+        tokens.append((match.group(1).upper(), count))
+        position = match.end()
+    if position != len(value) or not tokens:
+        raise CopybookParseError(f"unsupported PICTURE {value!r}")
+
+    digits = sum(count for symbol, count in tokens if symbol == "9")
+    characters = sum(count for symbol, count in tokens if symbol in {"A", "X", "9"})
+    textual = any(symbol in {"A", "X"} for symbol, _ in tokens)
+    signed = any(symbol == "S" for symbol, _ in tokens)
+    if textual:
+        if any(symbol in {"S", "V", "P"} for symbol, _ in tokens):
+            raise CopybookParseError(f"mixed numeric/alphanumeric PICTURE {value!r}")
+        return _Picture(digits, characters, 0, False, True)
+    if digits == 0:
+        raise CopybookParseError(f"numeric PICTURE {value!r} contains no digits")
+
+    symbols = [symbol for symbol, _ in tokens]
+    first_digit = next(index for index, item in enumerate(tokens) if item[0] == "9")
+    last_digit = len(tokens) - 1 - next(
+        index for index, item in enumerate(reversed(tokens)) if item[0] == "9"
+    )
+    leading_p = sum(count for symbol, count in tokens[:first_digit] if symbol == "P")
+    trailing_p = sum(count for symbol, count in tokens[last_digit + 1 :] if symbol == "P")
+    if leading_p:
+        scale = digits + leading_p
+    elif trailing_p:
+        scale = -trailing_p
+    elif "V" in symbols:
+        v_index = symbols.index("V")
+        scale = sum(
+            count for symbol, count in tokens[v_index + 1 :] if symbol in {"9", "P"}
+        )
+    else:
+        scale = 0
+    return _Picture(digits, characters, scale, signed, False)
+
+
+def _physical_type(
+    picture: _Picture,
+    usage: str,
+    separate_sign: bool,
+) -> tuple[int, str, str | None]:
+    if picture.textual:
+        if usage != "display":
+            raise CopybookParseError(f"usage {usage} is invalid for an alphanumeric item")
+        return picture.characters, "text", None
+    if usage == "display":
+        length = picture.digits + (1 if picture.signed and separate_sign else 0)
+        if picture.signed and not separate_sign:
+            return length, "zoned_decimal", None
+        return length, "decimal" if picture.scale else "integer", None
+    if usage == "packed_decimal":
+        return (picture.digits + 2) // 2, "packed_decimal", None
+    if usage == "binary":
+        if picture.digits <= 4:
+            length = 2
+        elif picture.digits <= 9:
+            length = 4
+        elif picture.digits <= 18:
+            length = 8
+        else:
+            raise CopybookParseError("binary PICTURE exceeds 18 stored digits")
+        return length, "binary", None
+    if usage == "comp_1":
+        return 4, "bytes", "latin-1"
+    if usage == "comp_2":
+        return 8, "bytes", "latin-1"
+    raise CopybookParseError(f"unsupported usage {usage!r}")
+
+
+def _field(
+    entry: _Entry,
+    start: int,
+    length: int,
+    kind: str,
+    scale: int,
+    signed: bool,
+    occurrence_path: tuple[int, ...],
+    encoding: str | None,
+) -> FieldSpec:
+    suffix = "".join(f"[{index}]" for index in occurrence_path)
+    return FieldSpec(
+        name=f"{entry.name}{suffix}",
+        start=start + 1,
+        length=length,
+        kind=kind,
+        scale=scale,
+        encoding=encoding,
+        signed=signed,
+    )
+
+
+def _unique_fields(fields: list[FieldSpec]) -> list[FieldSpec]:
+    counts: dict[str, int] = {}
+    used: set[str] = set()
+    result: list[FieldSpec] = []
+    for item in fields:
+        base = item.name
+        counts[base] = counts.get(base, 0) + 1
+        name = base if counts[base] == 1 else f"{base}#{counts[base]}"
+        while name in used:
+            counts[base] += 1
+            name = f"{base}#{counts[base]}"
+        used.add(name)
+        result.append(
+            item
+            if name == item.name
+            else FieldSpec(**{**asdict(item), "name": name})
+        )
+    return result
+
+
+def _is_filler(name: str) -> bool:
+    return bool(re.match(r"^FILLER(?:-\d+)?(?:\[|#|$)", name))
+
+
+__all__ = [
+    "CopybookParseError",
+    "layout_to_dict",
+    "layout_to_json",
+    "parse_copybook",
+    "parse_copybook_file",
+    "parse_copybook_text",
+    "write_layout_json",
+]

mainframe_toolkit/errors.py

@@ -0,0 +1,15 @@
+class ToolkitError(Exception):
+    """Base error raised by the toolkit."""
+
+
+class OptionalDependencyError(ToolkitError):
+    """Raised when an operation needs an optional package."""
+
+
+class DataFormatError(ToolkitError):
+    """Raised when a source record cannot be decoded safely."""
+
+
+class SpecificationError(ToolkitError):
+    """Raised when the migration project contract is invalid."""
+

mainframe_toolkit/external.py

@@ -0,0 +1,48 @@
+from __future__ import annotations
+
+import json
+from pathlib import Path
+from typing import Any, Callable, Mapping
+
+
+ExternalFunction = Callable[..., Any]
+
+
+class ExternalProgramRegistry:
+    def __init__(self, specifications: Mapping[str, str] | None = None):
+        self.specifications = {name.upper(): description for name, description in (specifications or {}).items()}
+        self._implementations: dict[str, ExternalFunction] = {}
+
+    @classmethod
+    def from_directory(cls, directory: str | Path) -> "ExternalProgramRegistry":
+        root = Path(directory)
+        combined: dict[str, str] = {}
+        for path in sorted(root.glob("*.json")):
+            with path.open(encoding="utf-8") as stream:
+                value = json.load(stream)
+            if not isinstance(value, dict) or not all(isinstance(item, str) for item in value.values()):
+                raise ValueError(f"{path} must contain a name-to-description object")
+            combined.update({str(name): description for name, description in value.items()})
+        return cls(combined)
+
+    def register(self, name: str, function: ExternalFunction) -> None:
+        self._implementations[name.upper()] = function
+
+    def implementation(self, name: str) -> ExternalFunction:
+        key = name.upper()
+        if key not in self._implementations:
+            description = self.specifications.get(key, "specification unavailable")
+            raise LookupError(f"external program {name} has no adapter: {description}")
+        return self._implementations[key]
+
+    def call(self, name: str, *args: Any, **kwargs: Any) -> Any:
+        return self.implementation(name)(*args, **kwargs)
+
+
+def external_program(name: str, registry: ExternalProgramRegistry) -> Callable[[ExternalFunction], ExternalFunction]:
+    def decorator(function: ExternalFunction) -> ExternalFunction:
+        registry.register(name, function)
+        return function
+
+    return decorator
+