jcl-parser 2.2.3__tar.gz → 2.3.0__tar.gz

{jcl_parser-2.2.3 → jcl_parser-2.3.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: jcl-parser
-Version: 2.2.3
+Version: 2.3.0
 Summary: Parse IBM JCL (Job Control Language) scripts into Python objects
 Author-email: Samuel Dion <sam94dion@gmail.com>
 License-Expression: AGPL-3.0-or-later

{jcl_parser-2.2.3 → jcl_parser-2.3.0}/jcl_parser.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: jcl-parser
-Version: 2.2.3
+Version: 2.3.0
 Summary: Parse IBM JCL (Job Control Language) scripts into Python objects
 Author-email: Samuel Dion <sam94dion@gmail.com>
 License-Expression: AGPL-3.0-or-later

{jcl_parser-2.2.3 → jcl_parser-2.3.0}/jclparser/__init__.py

@@ -20,7 +20,7 @@ application, for example::
 import logging as _logging
 
 # Package metadata
-__version__ = "2.2.3"  # x-release-please-version
+__version__ = "2.3.0"  # x-release-please-version
 __author__ = "JCL Parser Team"
 __description__ = "Parse IBM JCL scripts into Python objects"
 

{jcl_parser-2.2.3 → jcl_parser-2.3.0}/jclparser/cli.py

@@ -119,9 +119,15 @@ def _main(argv: list[str] | None = None) -> int:
     # ("FILE --explode steps"); plain parse_args() hands the whole positional
     # run to FILE/query before the flag and then rejects "steps".
     arg_parser = _build_arg_parser()
+    if not _argv:
+        # A bare `jcl-parser` used to print only "the following arguments
+        # are required: FILE"; show the queries and examples instead.
+        arg_parser.print_help()
+        return 2
     args = arg_parser.parse_intermixed_args(argv)
     if args.profile_output and not args.profile:
         arg_parser.error("--profile-output requires --profile")  # exits 2
+    _apply_format(args)
 
     source: Path | str
     if args.file == "-":
@@ -160,6 +166,14 @@ def _main(argv: list[str] | None = None) -> int:
     return _run_query_with_profile(job, args, prof, source)
 
 
+def _apply_format(args: argparse.Namespace) -> None:
+    """``--format json|sarif`` sets the older ``--json`` / ``--sarif`` flags."""
+    if args.format == "json":
+        args.json_output = True
+    elif args.format == "sarif":
+        args.sarif = True
+
+
 def _read_stdin() -> str:
     """The JCL piped on stdin, decoded like a file (``FILE`` is ``-``)."""
     return read_source(getattr(sys.stdin, "buffer", sys.stdin))
@@ -196,10 +210,70 @@ def _is_step_or_dd_name(job: Job, token: str) -> bool:
     )
 
 
+#: The query words ``jcl-parser FILE <word>`` understands, for typo hints.
+_QUERY_WORDS = (
+    "steps",
+    "procs",
+    "detect",
+    "metrics",
+    "lint",
+    "datasets",
+    "sql",
+    "analyze",
+    "paths",
+    "lineage",
+)
+
+_HELP_EPILOG = """\
+queries (the words after FILE):
+  (none)              job summary; with --json the whole parse as JSON
+  steps               every step, the program or PROC it runs, its DDs
+  procs [PROCNAME]    inline/external PROCs; --explode shows the expansion
+  STEP                one step in detail        e.g. STEP010
+  STEP.DDNAME         one DD statement          e.g. STEP010.SYSIN
+  DDNAME              that DD in every step     e.g. SYSOUT
+  detect              is the file a JOB, a PROC, or neither
+  metrics             step, DD, PROC and dataset counts, complexity
+  lint                lint rules (--lint-list lists them; --sarif for CI)
+  datasets            dataset lifecycle: created, read, deleted, passed
+  sql                 DB2 / SQL usage by step
+  analyze [STEP]      what each utility step (SORT, IDCAMS, ...) does
+  paths               which steps can run, from the COND / IF logic
+
+  FILE may be a directory: 'lineage [dot|mermaid]' (the default) maps
+  datasets across its jobs, 'lint' and --validate check every file
+  (--glob picks them). FILE '-' reads the JCL from stdin.
+
+examples:
+  jcl-parser PAYROLL.jcl
+  jcl-parser PAYROLL.jcl steps --explode --proc-path ./proclib
+  jcl-parser PAYROLL.jcl STEP010.SYSIN
+  jcl-parser PAYROLL.jcl lint --fail-on warning
+  jcl-parser PAYROLL.jcl --validate --format json
+  jcl-parser jobs/ lineage mermaid > lineage.mmd
+  jcl-lint jobs/*.jcl --format sarif -o jcl.sarif   (several files, for CI)
+
+exit codes:
+  0  success
+  1  the file could not be read or parsed, a name was not found, or a check
+     reported something (--validate errors, lint findings at --fail-on or
+     above, dataset / SQL findings)
+  2  the command line was rejected
+
+defaults:
+  lint reads [tool.jcl-parser.lint] from the nearest pyproject.toml (or
+  --config). External PROCs are loaded only with --proc-path, except by
+  --validate, which falls back to the JCL_PROCLIB_PATH variable.
+"""
+
+
 def _build_arg_parser() -> argparse.ArgumentParser:
     parser = argparse.ArgumentParser(
         prog="jcl-parser",
-        description="Parse and inspect IBM JCL files.",
+        usage="jcl-parser FILE [QUERY ...] [options]",
+        description="Parse and inspect IBM z/OS JCL: jobs, steps, DDs and PROCs.",
+        epilog=_HELP_EPILOG,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
     )
     parser.add_argument(
         "file",
@@ -210,15 +284,22 @@ def _build_arg_parser() -> argparse.ArgumentParser:
         "query",
         nargs="*",
         default=[],
-        help=(
-            "Query: 'steps', 'procs [PROCNAME]', a step name, STEP.DDNAME, "
-            "or a DD name to find across all steps; for a directory, "
-            "'lineage [dot|mermaid]'"
-        ),
+        metavar="QUERY",
+        help="What to show (see 'queries' below); none prints the job summary",
     )
     parser.add_argument(
         "--json", action="store_true", dest="json_output", help="Output as JSON"
     )
+    parser.add_argument(
+        "--format",
+        choices=("text", "json", "sarif"),
+        default=None,
+        help=(
+            "Output format: text (default), json (same as --json) or sarif "
+            "(same as --sarif; lint and --validate only). The spelling "
+            "cobol-parser and jcl-lint use"
+        ),
+    )
     parser.add_argument(
         "--validate", action="store_true", help="Validate the JCL and report errors"
     )
@@ -469,7 +550,44 @@ def _cmd_step_or_dd(job: Job, query: str, json_out: bool) -> int:
         return 0
 
     # Fall back to DD name across all steps
-    return _print_dd_across_steps(job, query.upper(), json_out)
+    status = _print_dd_across_steps(job, query.upper(), json_out)
+    if status != 0:
+        hint = _not_found_hint(job, query)
+        if hint:
+            print(hint, file=sys.stderr)
+    return status
+
+
+def _not_found_hint(job: Job, query: str) -> str | None:
+    """A "did you mean" line for a query that named no step or DD.
+
+    A mistyped query word (``stpes``) used to read only as "No step or DD
+    named 'STPES' found", which sends a new user looking for a step.
+    """
+    import difflib
+
+    words = difflib.get_close_matches(query.lower(), _QUERY_WORDS, n=1, cutoff=0.6)
+    names = sorted(
+        {s.name for s in iter_execution_steps(job.steps) if s.name}
+        | {
+            dd.name
+            for s in iter_execution_steps(job.steps)
+            for dd in s.dd_statements
+            if dd.name
+        }
+    )
+    close = difflib.get_close_matches(query.upper(), names, n=3, cutoff=0.6)
+    # JCL names are uppercase: an uppercase query is a misspelt name first,
+    # a lowercase one a misspelt query word first.
+    name_hint = "Did you mean: " + ", ".join(close) + "?" if close else None
+    word_hint = (
+        f"Did you mean the query '{words[0]}'? (see 'jcl-parser --help')"
+        if words
+        else None
+    )
+    if query == query.upper():
+        return name_hint or word_hint
+    return word_hint or name_hint
 
 
 def _cmd_directory(path: Path, args: argparse.Namespace) -> int:

{jcl_parser-2.2.3 → jcl_parser-2.3.0}/jclparser/explosion.py

@@ -1219,6 +1219,104 @@ def is_exec_override_keyword(key: str) -> bool:
     return key.upper().partition(".")[0] in _EXEC_OVERRIDE_KEYWORDS
 
 
+_SYMBOL_REFERENCE = re.compile(r"&&|&([A-Z@#$][A-Z0-9@#$]{0,7})", re.IGNORECASE)
+_INCLUDE_STATEMENT = re.compile(r"^//\S*\s+INCLUDE\s", re.IGNORECASE | re.MULTILINE)
+
+
+def _referenced_symbols(value: Any, found: set[str], seen: set[int]) -> None:
+    """Add every ``&NAME`` written anywhere in *value* to *found*.
+
+    Walks strings, dicts, lists and the model dataclasses (steps, DDs,
+    conditional blocks) a procedure body is made of. ``&&`` is an escaped
+    ampersand, not a reference.
+    """
+    if isinstance(value, str):
+        for match in _SYMBOL_REFERENCE.finditer(value):
+            if match.group(1):
+                found.add(match.group(1).upper())
+        return
+    if id(value) in seen:
+        return
+    seen.add(id(value))
+    if isinstance(value, dict):
+        for item in value.values():
+            _referenced_symbols(item, found, seen)
+    elif isinstance(value, (list, tuple)):
+        for item in value:
+            _referenced_symbols(item, found, seen)
+    elif isinstance(value, (Step, DDStatement, ConditionalBlock)):
+        for name, item in vars(value).items():
+            if name not in ("comments", "inline_data", "original_proc_call"):
+                _referenced_symbols(item, found, seen)
+
+
+def call_override_symbols(call: Step) -> set[str]:
+    """Every ``&NAME`` in the overrides coded with PROC call *call*.
+
+    The overriding and added DDs after the call (and their concatenations)
+    and the values of its keyword overrides (``PARM.PS1='&X'``) are merged
+    into the procedure, so the call's symbolics are substituted there too
+    (issue #176).
+    """
+    found: set[str] = set()
+    _referenced_symbols(
+        [
+            call.dd_statements,
+            list(call.dd_overrides.values()),
+            list(call.dd_override_concatenations.values()),
+            [
+                value
+                for key, value in call.parameters.items()
+                if is_exec_override_keyword(key)
+            ],
+        ],
+        found,
+        set(),
+    )
+    return found
+
+
+def unused_proc_symbols(
+    call: Step, procedure: Procedure, call_references: set[str] | None = None
+) -> list[str]:
+    """The symbolics PROC call *call* assigns that *procedure* never uses.
+
+    A symbolic coded on ``EXEC procname`` that is neither declared on the
+    PROC statement nor referenced as ``&NAME`` in the procedure body is a
+    JCL error on z/OS (``IEFC657I THE SYMBOL x WAS NOT USED``), and the
+    expansion silently runs with the PROC's defaults (issue #172). EXEC
+    keyword overrides (``PARM``, ``COND.PS1``, ...) are not symbolics. A
+    reference in the call's overriding or added DDs, or in a keyword
+    override's value, counts as a use (issue #176); *call_references* gives
+    those references when the call's overrides were already substituted
+    (default: :func:`call_override_symbols` of *call*).
+
+    Returns an empty list when the body cannot be fully checked: a procedure
+    with no steps (unloaded), or one whose text has an ``INCLUDE`` (its
+    members' references are not known before explosion).
+    """
+    if not procedure.steps:
+        return []
+    if procedure.source_text and _INCLUDE_STATEMENT.search(procedure.source_text):
+        return []
+    known = {name.upper() for name in procedure.parameters}
+    _referenced_symbols(
+        [procedure.steps, procedure.set_statements, procedure.parameters],
+        known,
+        set(),
+    )
+    if procedure.source_text:
+        _referenced_symbols(procedure.source_text, known, set())
+    known |= call_override_symbols(call) if call_references is None else call_references
+    return [
+        key
+        for key in call.parameters
+        if not is_exec_override_keyword(key)
+        and key.upper() != "PROC"
+        and key.upper() not in known
+    ]
+
+
 def _substitute_proc_call_parameters(
     parameters: dict[str, Any],
     scope: SymbolScope,
@@ -2099,6 +2197,8 @@ def _explode_step(
     # against the job's library order. Pass an immutable tuple: loaders key their
     # caches on the library list, so a hashable value keeps that contract safe.
     _libs = tuple(jcllib_order) if jcllib_order else None
+    # Read before loading resolves the call's overrides (issue #176).
+    call_references = call_override_symbols(step)
     procedure = _load_called_procedure(
         step, proc_loader, _libs, base_scope, call_scope, strict_symbols, warnings
     )
@@ -2121,6 +2221,12 @@ def _explode_step(
     )
 
     _apply_call_overrides(expanded, step, procedure.name or proc_name, warnings)
+    unused = unused_proc_symbols(step, procedure, call_references)
+    if unused and warnings is not None:
+        warnings.append(
+            f"EXEC {step.name or 'PROC'}: symbol(s) {', '.join(unused)} not used "
+            f"by PROC {procedure.name or proc_name} (IEFC657I); its defaults apply"
+        )
     _record_expansion(
         step, procedure, overrides, expanded, depth, audit_records, expansion_map
     )

{jcl_parser-2.2.3 → jcl_parser-2.3.0}/jclparser/program_profiles/dfsrrc00.py

@@ -13,12 +13,29 @@ comma-delimited sub-parameter of PARM:
 The profile extracts the application program and PSB name from PARM so that
 ``analyze_step``/``analyze_job`` can report what application is actually
 running, not just the region controller.
+
+That layout holds only for the batch regions (BMP, DLI, DBB). A message
+processing region codes ``PARM=(MSG,&CL1&CL2&CL3&CL4,...)``: PARM[1] is the
+four three-digit transaction classes it serves and it names no program. For
+MSG, IFP and any other region type, only the region type (and, for MSG, the
+classes) is reported (issue #173).
 """
 
 from __future__ import annotations
 
+from ..model import Step
 from .base import ProgramAnalysis, ProgramProfile
 
+#: Region types whose PARM[1] / PARM[2] are the application program and PSB.
+_BATCH_REGION_TYPES = frozenset({"BMP", "DLI", "DBB"})
+
+
+def _transaction_classes(field: str) -> list[str]:
+    """Split an MPP's ``&CL1&CL2&CL3&CL4`` field into its three-digit classes."""
+    if not field.isdigit() or len(field) % 3:
+        return []
+    return [field[i : i + 3] for i in range(0, len(field), 3)]
+
 
 class DFSRRC00Profile(ProgramProfile):
     program_names = frozenset({"DFSRRC00", "DFSRRC02"})
@@ -42,6 +59,9 @@ class DFSRRC00Profile(ProgramProfile):
         parts = [p.strip() for p in parm.split(",")]
 
         region_type = parts[0].upper() if parts and parts[0] else ""
+        if region_type and region_type not in _BATCH_REGION_TYPES:
+            return _online_region_analysis(step, region_type, parts, parm)
+
         app_program = parts[1].upper() if len(parts) > 1 and parts[1] else ""
         psb_name = parts[2].upper() if len(parts) > 2 and parts[2] else ""
 
@@ -67,3 +87,31 @@ class DFSRRC00Profile(ProgramProfile):
             notes=notes,
             raw_control=parm or None,
         )
+
+
+def _online_region_analysis(
+    step: Step, region_type: str, parts: list[str], parm: str
+) -> ProgramAnalysis:
+    """A region whose PARM names no program (MSG, IFP, ...): issue #173."""
+    notes = [f"IMS region type: {region_type}"]
+    classes = (
+        _transaction_classes(parts[1])
+        if region_type == "MSG" and len(parts) > 1
+        else []
+    )
+    summary = f"IMS {region_type} region ({step.program})"
+    if classes:
+        summary += f", classes {' '.join(classes)}"
+        notes.append(f"transaction classes: {' '.join(classes)}")
+    notes.append(
+        f"an IMS {region_type} region runs the transactions scheduled to it; "
+        "its PARM names no application program"
+    )
+    return ProgramAnalysis(
+        program=step.program or "DFSRRC00",
+        step_name=step.name or "",
+        summary=summary,
+        dataset_events=[],
+        notes=notes,
+        raw_control=parm or None,
+    )

{jcl_parser-2.2.3 → jcl_parser-2.3.0}/jclparser/validators.py

@@ -321,7 +321,10 @@ def validate_backward_references(job: Job) -> list[ValidationError]:
 
 
 def validate_procedure_references(job: Job) -> list[ValidationError]:
-    """Warn on ``EXEC procname`` calls whose PROC was not found.
+    """Check ``EXEC procname`` calls against the PROCs they call.
+
+    A symbolic the call assigns that the PROC never declares or references
+    is an ERROR (``IEFC657I``, issue #172).
 
     :func:`jclparser.parse_jcl` adds every PROC it can load (in-stream,
     JCLLIB / system PROCLIB under the base path, built-in) to
@@ -330,9 +333,24 @@ def validate_procedure_references(job: Job) -> list[ValidationError]:
     a WARNING, not an ERROR (issue #116).  A strict parse of a job with a
     ``JCLLIB ORDER`` still fails on it before this runs.
     """
+    from .explosion import unused_proc_symbols
+
     errors: list[ValidationError] = []
     for step in iter_execution_steps(job.steps):
-        if step.procedure and not job.get_procedure(step.procedure):
+        procedure = job.get_procedure(step.procedure) if step.procedure else None
+        if procedure is not None:
+            # A symbolic the PROC never declares or references (IEFC657I).
+            for name in unused_proc_symbols(step, procedure):
+                errors.append(
+                    ValidationError(
+                        f"Step '{step.name or ''}': symbol {name} is not used by "
+                        f"procedure '{step.procedure}' (IEFC657I); the "
+                        "procedure's defaults apply",
+                        ErrorType.SEMANTIC_ERROR,
+                        line_number=step.line_number,
+                    )
+                )
+        if step.procedure and procedure is None:
             errors.append(
                 ValidationError(
                     f"Procedure '{step.procedure}' not found in-stream or in "

{jcl_parser-2.2.3 → jcl_parser-2.3.0}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "jcl-parser"
-version = "2.2.3"
+version = "2.3.0"
 description = "Parse IBM JCL (Job Control Language) scripts into Python objects"
 authors = [
     {name = "Samuel Dion", email = "sam94dion@gmail.com"}