jcl-parser 2.0.1__tar.gz → 2.0.3__tar.gz

{jcl_parser-2.0.1 → jcl_parser-2.0.3}/PKG-INFO

@@ -1,8 +1,8 @@
 Metadata-Version: 2.4
 Name: jcl-parser
-Version: 2.0.1
+Version: 2.0.3
 Summary: Parse IBM JCL (Job Control Language) scripts into Python objects
-Author-email: JCL Parser Team <team@jclparser.dev>
+Author-email: Samuel Dion <sam94dion@gmail.com>
 License: AGPL-3.0-or-later
 Project-URL: Homepage, https://github.com/landparty/jcl-parser
 Project-URL: Repository, https://github.com/landparty/jcl-parser

{jcl_parser-2.0.1 → jcl_parser-2.0.3}/jcl_parser.egg-info/PKG-INFO

@@ -1,8 +1,8 @@
 Metadata-Version: 2.4
 Name: jcl-parser
-Version: 2.0.1
+Version: 2.0.3
 Summary: Parse IBM JCL (Job Control Language) scripts into Python objects
-Author-email: JCL Parser Team <team@jclparser.dev>
+Author-email: Samuel Dion <sam94dion@gmail.com>
 License: AGPL-3.0-or-later
 Project-URL: Homepage, https://github.com/landparty/jcl-parser
 Project-URL: Repository, https://github.com/landparty/jcl-parser

{jcl_parser-2.0.1 → jcl_parser-2.0.3}/jclparser/__init__.py

@@ -20,7 +20,7 @@ application, for example::
 import logging as _logging
 
 # Package metadata
-__version__ = "2.0.1"  # x-release-please-version
+__version__ = "2.0.3"  # x-release-please-version
 __author__ = "JCL Parser Team"
 __description__ = "Parse IBM JCL scripts into Python objects"
 

{jcl_parser-2.0.1 → jcl_parser-2.0.3}/jclparser/_job_builder.py

@@ -131,10 +131,13 @@ class JobBuilder:
         "XMIT": "_on_xmit",
     }
 
-    def __init__(self, dialect: str = "standard") -> None:
+    def __init__(self, dialect: str = "standard", strict: bool = False) -> None:
         #: ``ParserConfig.dialect``: decides which JES control statements are
         #: interpreted (see ``jclparser.plugins.SUPPORTED_DIALECTS``).
         self.dialect = dialect
+        #: Strict parsing: a SET value over ``MAX_SYMBOL_VALUE_LENGTH`` raises
+        #: instead of being left unresolved with a warning.
+        self.strict = strict
         self.job: Job | None = None
         #: tuples: (ConditionalBlock, active_branch)
         self.conditional_stack: list[tuple[ConditionalBlock, str]] = []
@@ -171,6 +174,10 @@ class JobBuilder:
         #: only if some SET symbol is redefined after it was first set.
         self.job_level_steps: list[Step] = []
         self.set_redefined = False
+        #: Steps of in-stream / cataloged procedures with the procedure they
+        #: belong to; finish() prunes their ``set_symbols`` snapshot the same
+        #: way (against the procedure's own final SET table).
+        self.procedure_steps: list[tuple[Procedure, Step]] = []
         self.pending_exported: set[str] = set()
         self.pending_jcllib: list[str] = []
         self.pending_includes: list[str] = []
@@ -312,6 +319,11 @@ class JobBuilder:
                 **self.job.set_statements,
             }
             self.job_level_steps.append(self.current_step)
+        elif self.current_procedure is not None:
+            # A SET inside the procedure is in effect from where it is coded,
+            # so this step sees the procedure's SET values as they stand now.
+            self.current_step.set_symbols = dict(self.current_procedure.set_statements)
+            self.procedure_steps.append((self.current_procedure, self.current_step))
         self.in_joblib_concat = False  # JOBLIB concatenation ends at first EXEC
         self.concat_override_key = None
 
@@ -437,7 +449,14 @@ class JobBuilder:
             symbols = self.job.set_statements if self.job else self.pending_sets
 
         def _record(symbol: str, value: str) -> None:
-            resolved = _resolve_set_value(value, symbols)
+            resolved = _resolve_set_value(
+                value,
+                symbols,
+                symbol=symbol,
+                line_number=stmt.line_number,
+                strict=self.strict,
+                warnings=self._warnings_sink(),
+            )
             if procedure is not None:
                 procedure.set_statements[symbol] = resolved
             elif self.job:
@@ -810,6 +829,12 @@ class JobBuilder:
                 for k, v in (step.set_symbols or {}).items()
             ):
                 step.set_symbols = None
+        for procedure, step in self.procedure_steps:
+            if all(
+                procedure.set_statements.get(k) == v
+                for k, v in (step.set_symbols or {}).items()
+            ):
+                step.set_symbols = None
 
     def _warn_orphaned_dds(self, job: Job) -> None:
         # orphaned_dds is only ever non-empty here when the whole job had no

{jcl_parser-2.0.1 → jcl_parser-2.0.3}/jclparser/_model_factories.py

@@ -21,13 +21,22 @@ import re
 from collections.abc import Callable
 from typing import Any
 
+from .errors import JCLSyntaxError
 from .model import DDStatement, Job, Statement, Step
-from .utils import POSITIONAL_PARAM_PREFIX, is_positional_key
+from .utils import MAX_SYMBOL_VALUE_LENGTH, POSITIONAL_PARAM_PREFIX, is_positional_key
 
 _SET_SYMBOL_PATTERN = re.compile(r"&&|&([A-Z0-9@#$]+)\.?")
 
 
-def _resolve_set_value(value: str, known_symbols: dict[str, str]) -> str:
+def _resolve_set_value(
+    value: str,
+    known_symbols: dict[str, str],
+    *,
+    symbol: str = "",
+    line_number: int = 0,
+    strict: bool = False,
+    warnings: list[str] | None = None,
+) -> str:
     """Resolve &SYMBOL references in a SET value using already-accumulated SET symbols.
 
     Mirrors the mainframe rule: SET values are evaluated left-to-right so a
@@ -36,15 +45,56 @@ def _resolve_set_value(value: str, known_symbols: dict[str, str]) -> str:
     itself holds a "&&NAME" temporary-dataset marker (e.g. ``SET
     MYTEMP=&&WORK``) before it ever reaches a DD statement. That collapse is
     explode_job()'s job (Tier 2), not this Tier-1 SET-value resolution.
+
+    A resolved value longer than :data:`~jclparser.utils.MAX_SYMBOL_VALUE_LENGTH`
+    is never built (#70): its length is computed first, and the value is
+    returned unresolved with a message on *warnings* -- or, when *strict*,
+    :class:`~jclparser.errors.JCLSyntaxError` is raised. *symbol* and
+    *line_number* only label that message.
     """
+    matches = list(_SET_SYMBOL_PATTERN.finditer(value))
+    if not matches:
+        return value
 
-    def _replacer(m: re.Match[str]) -> str:
+    def _replacement(m: re.Match[str]) -> str:
         if m.group(0) == "&&":
             return "&&"
-        sym = m.group(1).upper()
-        return known_symbols.get(sym, m.group(0))
+        return known_symbols.get(m.group(1).upper(), m.group(0))
 
-    return _SET_SYMBOL_PATTERN.sub(_replacer, value)
+    replacements = [_replacement(m) for m in matches]
+    length = len(value) + sum(
+        len(r) - len(m.group(0)) for m, r in zip(matches, replacements)
+    )
+    if length > MAX_SYMBOL_VALUE_LENGTH:
+        _report_set_overflow(symbol, length, line_number, strict, warnings)
+        return value
+    parts: list[str] = []
+    pos = 0
+    for m, r in zip(matches, replacements):
+        parts.append(value[pos : m.start()])
+        parts.append(r)
+        pos = m.end()
+    parts.append(value[pos:])
+    return "".join(parts)
+
+
+def _report_set_overflow(
+    symbol: str,
+    length: int,
+    line_number: int,
+    strict: bool,
+    warnings: list[str] | None,
+) -> None:
+    """Raise (strict) or warn that SET *symbol* would resolve to *length* chars."""
+    message = (
+        f"SET symbol &{symbol or '?'} would resolve to {length:,} characters, "
+        f"more than the {MAX_SYMBOL_VALUE_LENGTH:,}-character limit"
+    )
+    if strict:
+        raise JCLSyntaxError(message, line_number)
+    if warnings is not None:
+        prefix = f"Line {line_number}: " if line_number else ""
+        warnings.append(f"{prefix}{message}; value left unresolved")
 
 
 def _has_unresolved_symbol_ref(

{jcl_parser-2.0.1 → jcl_parser-2.0.3}/jclparser/_post_parse.py

@@ -246,7 +246,7 @@ def enforce_strict_dd_structure(content: str) -> None:
                 )
 
 
-def resolve_set_symbols(job: Job) -> None:
+def resolve_set_symbols(job: Job, strict: bool = False) -> None:
     """Substitute fully-resolved SET symbols into DD and EXEC-step parameters.
 
     SET statements may appear before or after the DD statements that reference
@@ -274,11 +274,17 @@ def resolve_set_symbols(job: Job) -> None:
     pass.  It also covers EXEC step parameters (e.g. ``PARM=&SYM``), not just
     DD dataset names/dispositions, so a job-level SET symbol resolves the same
     way regardless of which kind of statement references it.
+
+    A SET value whose expansion would exceed ``utils.MAX_SYMBOL_VALUE_LENGTH``
+    characters is left unresolved with one warning on ``job.warnings`` (#70);
+    a statement referencing it keeps the reference as written (again with a
+    warning). With *strict*, :class:`~jclparser.errors.JCLSyntaxError` is
+    raised instead.
     """
     from .explosion import substitute_dd_statement
     from .system_symbols import is_system_symbol_name
 
-    _set_scope = _scope_for_table(job.set_statements)
+    _set_scope = _scope_for_table(job.set_statements, job.warnings, strict)
     # A step read before a SET redefined a symbol sees the value in effect at
     # that point (Step.set_symbols); symbols it has no snapshot for (forward
     # references) still fall back to the job's final table.
@@ -290,7 +296,7 @@ def resolve_set_symbols(job: Job) -> None:
     for _owner, _dd in _dds_with_owners(_flat_steps):
         substitute_dd_statement(
             _dd,
-            _scope_for_step(_owner, job, _set_scope, _step_scopes),
+            _scope_for_step(_owner, job, _set_scope, _step_scopes, strict),
             strict=False,
             warnings=job.warnings,
             collapse_double_amp=False,
@@ -298,14 +304,18 @@ def resolve_set_symbols(job: Job) -> None:
         )
 
     for _step in _flat_steps:
-        _step_scope = _scope_for_step(_step, job, _set_scope, _step_scopes)
+        _step_scope = _scope_for_step(_step, job, _set_scope, _step_scopes, strict)
         _substitute_step_parameters(_step, _step_scope, job.warnings)
         _substitute_step_names(_step, _step_scope)
 
 
-def _preresolve_set_table(table: dict[str, str]) -> dict[str, str]:
-    """Return a copy of SET *table* with symbol references between values expanded."""
-    from .explosion import SymbolScope, substitute_symbols
+def _preresolve_set_table(table: dict[str, str]) -> tuple[dict[str, str], list[str]]:
+    """Return a copy of SET *table* with symbol references between values expanded.
+
+    Also returns the names of the symbols whose expansion would exceed
+    ``utils.MAX_SYMBOL_VALUE_LENGTH``; those keep their unexpanded value.
+    """
+    from .explosion import SymbolScope, _substitute_symbols_bounded, _SymbolOverflow
 
     # Pre-resolve the symbol table so every value is fully expanded before
     # we apply it to DD dataset names.  On each pass we substitute symbols
@@ -313,36 +323,67 @@ def _preresolve_set_table(table: dict[str, str]) -> dict[str, str]:
     # (excluding the symbol itself prevents direct self-reference loops).
     # We repeat until no value changes or we reach the pass limit.
     _raw: dict[str, str] = dict(table)
+    _overflowed: list[str] = []
     for _pass in range(10):
         _changed = False
         for _name, _val in list(_raw.items()):
-            if "&" not in _val:
+            if "&" not in _val or _name in _overflowed:
                 continue
             # Build a temporary scope that excludes this symbol's own name
             # so "&A." inside SET A's value does not expand to itself.
             _scope_for_val = SymbolScope(
                 set_symbols={k: v for k, v in _raw.items() if k != _name}
             )
-            _new_val = substitute_symbols(
-                _val,
-                _scope_for_val,
-                allow_undefined=True,
-                collapse_double_amp=False,
-            )
+            try:
+                _new_val = _substitute_symbols_bounded(
+                    _val,
+                    _scope_for_val,
+                    allow_undefined=True,
+                    collapse_double_amp=False,
+                )
+            except _SymbolOverflow:
+                # Values only grow from pass to pass: never retry it.
+                _overflowed.append(_name)
+                continue
             if _new_val != _val:
                 _raw[_name] = _new_val
                 _changed = True
         if not _changed:
             break  # table has stabilised – no need for more passes
-    return _raw
+    return _raw, _overflowed
 
 
-def _scope_for_table(table: dict[str, str]) -> SymbolScope:
-    """Build the substitution scope for SET symbol *table* (see resolve_set_symbols)."""
+def _report_table_overflow(
+    overflowed: list[str], warnings: list[str], strict: bool
+) -> None:
+    """Raise (strict) or warn once that the *overflowed* SET symbols stay unresolved."""
+    from .utils import MAX_SYMBOL_VALUE_LENGTH
+
+    names = ", ".join(f"&{n}" for n in overflowed)
+    message = (
+        f"SET symbol(s) {names} expand to more than "
+        f"{MAX_SYMBOL_VALUE_LENGTH:,} characters"
+    )
+    if strict:
+        raise JCLSyntaxError(message)
+    message += "; left unresolved"
+    if message not in warnings:
+        warnings.append(message)
+
+
+def _scope_for_table(
+    table: dict[str, str], warnings: list[str], strict: bool = False
+) -> SymbolScope:
+    """Build the substitution scope for SET symbol *table* (see resolve_set_symbols).
+
+    Overflow of a SET value is reported on *warnings* (or raised, *strict*).
+    """
     from .explosion import SymbolScope
     from .system_symbols import is_system_symbol_name
 
-    _raw = _preresolve_set_table(table)
+    _raw, _overflowed = _preresolve_set_table(table)
+    if _overflowed:
+        _report_table_overflow(_overflowed, warnings, strict)
 
     # Symbols whose values still contain a genuinely unresolved
     # &-reference after all passes are excluded from the final scope.
@@ -355,10 +396,15 @@ def _scope_for_table(table: dict[str, str]) -> SymbolScope:
     # ``DSN=&SYSUID..WORK`` before that pass runs (keep_undefined below). A value that legitimately contains "&&" (e.g.
     # SET MYTEMP=&&WORK) is NOT excluded -- it has nothing left to
     # resolve, it just happens to contain an escaped literal ampersand.
+    #
+    # An overflowed symbol keeps its unexpanded value in the scope, so a
+    # statement referencing it overflows the substitution budget in turn and
+    # keeps the reference as written instead of having it blanked (#70).
     _resolved_symbols = {
         k: v
         for k, v in _raw.items()
-        if not _has_unresolved_symbol_ref(v, ignore=is_system_symbol_name)
+        if k in _overflowed
+        or not _has_unresolved_symbol_ref(v, ignore=is_system_symbol_name)
     }
     return SymbolScope(set_symbols=_resolved_symbols)
 
@@ -368,6 +414,7 @@ def _scope_for_step(
     job: Job,
     set_scope: SymbolScope,
     step_scopes: dict[int, SymbolScope],
+    strict: bool = False,
 ) -> SymbolScope:
     """Return the SET scope in effect for *step*, memoised in *step_scopes*.
 
@@ -378,7 +425,7 @@ def _scope_for_step(
         return set_scope
     if id(step) not in step_scopes:
         step_scopes[id(step)] = _scope_for_table(
-            {**job.set_statements, **step.set_symbols}
+            {**job.set_statements, **step.set_symbols}, job.warnings, strict
         )
     return step_scopes[id(step)]
 

{jcl_parser-2.0.1 → jcl_parser-2.0.3}/jclparser/_source_scanner.py

@@ -31,7 +31,7 @@ from .errors import JCLSyntaxError
 from .utils import (
     instream_delimiter,
     is_continued_card_image,
-    normalize_jcl_line,
+    normalize_jcl_lines,
     split_source_lines,
 )
 
@@ -60,7 +60,7 @@ def _merge_source_lines(content: str) -> ScannedSource:
     or garbled records).
     """
     raw_lines = split_source_lines(content)
-    lines = [normalize_jcl_line(line) for line in raw_lines]
+    lines = normalize_jcl_lines(raw_lines)
 
     classifier = _RecordClassifier(lines)
     jcl_indices = classifier.run()
@@ -116,7 +116,10 @@ def merge_continuations(
         card_images={
             n
             for n, orig in enumerate(indices)
-            if is_continued_card_image(raw_lines[orig])
+            if is_continued_card_image(
+                raw_lines[orig],
+                raw_lines[orig + 1] if orig + 1 < len(raw_lines) else None,
+            )
         },
     )
 

{jcl_parser-2.0.1 → jcl_parser-2.0.3}/jclparser/explosion.py

@@ -36,7 +36,7 @@ from .proc_loader import (
     create_default_loader,
 )
 from .system_symbols import SystemSymbolProvider, get_default_provider
-from .utils import jcl_value_text
+from .utils import MAX_SYMBOL_VALUE_LENGTH, jcl_value_text
 
 logger = logging.getLogger(__name__)
 
@@ -53,10 +53,23 @@ _SYMBOL_SUB_PATTERN = re.compile(r"&&|&([A-Z0-9@#$]+)\.?")
 # how deep those nested references are followed.
 _MAX_NESTED_SUBSTITUTION = 8
 
-# Total characters nested symbol values may expand to within one substitution.
-# Fan-out over distinct symbols (``A='&B&B&B'``, ``B='&C&C&C'`` ...) grows
-# exponentially with depth even without a cycle, so depth alone is no bound.
-_MAX_NESTED_EXPANSION_CHARS = 1_000_000
+# Total characters the symbol values spliced in by one substitution may add
+# up to (``utils.MAX_SYMBOL_VALUE_LENGTH``). Fan-out over distinct symbols
+# (``A='&B&B&B'``, ``B='&C&C&C'`` ...) grows exponentially with depth even
+# without a cycle, so depth alone is no bound (#70).
+_MAX_NESTED_EXPANSION_CHARS = MAX_SYMBOL_VALUE_LENGTH
+
+
+class _SymbolOverflow(Exception):
+    """Internal: a substitution would splice in more than the character budget."""
+
+
+def _overflow_message(source_text: str) -> str:
+    return (
+        f"Symbols in '{source_text[:80]}' expand to more than "
+        f"{_MAX_NESTED_EXPANSION_CHARS:,} characters"
+    )
+
 
 # A value ending in a bare ``&NAME`` reference (not the ``&&NAME`` literal).
 _TRAILING_SYMBOL_REF = re.compile(r"(?<!&)&[A-Z0-9@#$]+$")
@@ -319,10 +332,13 @@ def _make_replacer(
     *_resolving* holds the symbols whose values are being expanded above this
     level; a reference back to one of them is a cycle and is left as written.
     *_budget* is a one-element list shared by every nested level, counting
-    down the characters nested expansion may still produce.
+    down the characters the substitution may still splice in; running out
+    raises :class:`_SymbolOverflow`, so no level builds a string much longer
+    than the budget.
     """
     if _budget is None:
         _budget = [_MAX_NESTED_EXPANSION_CHARS]
+    budget = _budget
 
     def _replacer(m: re.Match[str]) -> str:
         if m.group(0) == "&&":
@@ -361,17 +377,13 @@ def _make_replacer(
                     collapse_double_amp=False,
                     _depth=_depth + 1,
                     _resolving=_resolving | {symbol_name},
-                    _budget=_budget,
+                    _budget=budget,
                 ),
                 value,
             )
-            _budget[0] -= len(value)
-            if _budget[0] < 0:
-                raise JCLStructureError(
-                    f"Symbol &{symbol_name} expands to more than "
-                    f"{_MAX_NESTED_EXPANSION_CHARS:,} characters in "
-                    f"'{source_text[:80]}'"
-                )
+        budget[0] -= len(value)
+        if budget[0] < 0:
+            raise _SymbolOverflow
         if _TRAILING_SYMBOL_REF.search(value):
             # Splicing a bare ``&SYSUID`` in front of ``.DATA`` would make that
             # period the reference's delimiter; give it its own.
@@ -381,6 +393,30 @@ def _make_replacer(
     return _replacer
 
 
+def _substitute_symbols_bounded(
+    text: str,
+    scope: "SymbolScope",
+    strict: bool = False,
+    allow_undefined: bool = False,
+    warnings: list[str] | None = None,
+    collapse_double_amp: bool = True,
+    keep_undefined: Callable[[str], bool] | None = None,
+) -> str:
+    """:func:`substitute_symbols` that raises :class:`_SymbolOverflow` on overflow."""
+    return _SYMBOL_SUB_PATTERN.sub(
+        _make_replacer(
+            scope,
+            strict,
+            allow_undefined,
+            source_text=text,
+            warnings=warnings,
+            collapse_double_amp=collapse_double_amp,
+            keep_undefined=keep_undefined,
+        ),
+        text,
+    )
+
+
 def substitute_symbols(
     text: str,
     scope: "SymbolScope",
@@ -415,25 +451,35 @@ def substitute_symbols(
             symbols) can still resolve it.
 
     Returns:
-        Text with symbols substituted
+        Text with symbols substituted. When the substituted values would add
+        up to more than ``utils.MAX_SYMBOL_VALUE_LENGTH`` characters (chained
+        symbols that double at each step), *text* is returned unchanged and a
+        message goes to *warnings* (#70).
 
     Raises:
         UndefinedSymbolError: If strict=True and symbol undefined
+        JCLStructureError: If strict=True and the substitution would exceed
+            ``utils.MAX_SYMBOL_VALUE_LENGTH`` characters
     """
     if not text or not isinstance(text, str):
         return text
-    return _SYMBOL_SUB_PATTERN.sub(
-        _make_replacer(
+    try:
+        return _substitute_symbols_bounded(
+            text,
             scope,
-            strict,
-            allow_undefined,
-            source_text=text,
+            strict=strict,
+            allow_undefined=allow_undefined,
             warnings=warnings,
             collapse_double_amp=collapse_double_amp,
             keep_undefined=keep_undefined,
-        ),
-        text,
-    )
+        )
+    except _SymbolOverflow:
+        message = _overflow_message(text)
+        if strict:
+            raise JCLStructureError(message) from None
+        if warnings is not None:
+            warnings.append(f"{message}; left unresolved")
+        return text
 
 
 def detect_circular_refs(
@@ -523,6 +569,8 @@ def expand_proc_invocation(
             else:
                 # Clone the step
                 new_step = _clone_step_for_explosion(proc_step)
+                step_scope = _proc_step_scope(expansion_scope, proc_step)
+                new_step.set_symbols = None
 
                 # Generate qualified step name
                 if new_step.name:
@@ -538,18 +586,28 @@ def expand_proc_invocation(
                 if new_step.program:
                     new_step.program = substitute_symbols(
                         new_step.program,
-                        expansion_scope,
+                        step_scope,
                         strict=strict,
                         warnings=warnings,
                     )
 
-                # Substitute symbols in step parameters
-                new_step.parameters = substitute_parameters_dict(
-                    new_step.parameters,
-                    expansion_scope,
-                    strict=strict,
-                    warnings=warnings,
-                )
+                # Substitute symbols in step parameters. A nested
+                # ``EXEC procname`` keeps "&&" in its symbolic values, as a
+                # top-level one does (TMP='&&WORK' is a temporary dataset).
+                if new_step.is_procedure:
+                    new_step.parameters = _substitute_proc_call_parameters(
+                        new_step.parameters,
+                        step_scope,
+                        strict=strict,
+                        warnings=warnings,
+                    )
+                else:
+                    new_step.parameters = substitute_parameters_dict(
+                        new_step.parameters,
+                        step_scope,
+                        strict=strict,
+                        warnings=warnings,
+                    )
                 if "COND" in new_step.parameters:
                     new_step.parameters["COND"] = _qualify_cond_parameter(
                         new_step.parameters["COND"], proc_steps, prefix
@@ -559,7 +617,7 @@ def expand_proc_invocation(
                 # for a nested ``EXEC procname``, its PROCSTEP.DDNAME overrides
                 for dd in _step_dds_and_overrides(new_step):
                     substitute_dd_statement(
-                        dd, expansion_scope, strict=strict, warnings=warnings
+                        dd, step_scope, strict=strict, warnings=warnings
                     )
 
                 result.append(new_step)
@@ -570,6 +628,24 @@ def expand_proc_invocation(
     return expanded_steps
 
 
+def _proc_step_scope(expansion_scope: SymbolScope, proc_step: Step) -> SymbolScope:
+    """Return *expansion_scope* with the PROC SETs in effect at *proc_step*.
+
+    A SET inside a procedure is in effect from where it is coded: a step read
+    before a later SET redefined a symbol keeps the earlier value
+    (``Step.set_symbols``). Symbols it has no snapshot for still resolve
+    against the procedure's final SET table.
+    """
+    if proc_step.set_symbols is None:
+        return expansion_scope
+    return SymbolScope(
+        set_symbols={**expansion_scope.set_symbols, **proc_step.set_symbols},
+        proc_params=expansion_scope.proc_params,
+        exec_overrides=expansion_scope.exec_overrides,
+        system_provider=expansion_scope.system_provider,
+    )
+
+
 _WHOLE_SYMBOL_RE = re.compile(r"&([A-Z0-9@#$]+)\.?")
 
 
@@ -968,6 +1044,33 @@ _EXEC_OVERRIDE_KEYWORDS = frozenset(
 _PARM_KEYWORDS = frozenset({"PARM", "PARMDD"})
 
 
+def _substitute_proc_call_parameters(
+    parameters: dict[str, Any],
+    scope: SymbolScope,
+    strict: bool = False,
+    allow_undefined: bool = False,
+    warnings: list[str] | None = None,
+) -> dict[str, Any]:
+    """Substitute symbols in the parameters of an ``EXEC procname`` statement.
+
+    EXEC keyword overrides (``PARM.PS1='A&&B'``) collapse "&&" to "&" like any
+    other step parameter; a symbolic value keeps it, because it may carry a
+    temporary dataset name (``TMP='&&WORK'``) into the procedure's DDs.
+    """
+    return {
+        key: substitute_parameters_dict(
+            {key: value},
+            scope,
+            strict=strict,
+            allow_undefined=allow_undefined,
+            warnings=warnings,
+            collapse_double_amp=key.upper().partition(".")[0]
+            in _EXEC_OVERRIDE_KEYWORDS,
+        )[key]
+        for key, value in parameters.items()
+    }
+
+
 def _set_exec_keyword(step: Step, keyword: str, value: Any) -> None:
     """Override (or, for an empty value, nullify) *keyword* on *step*."""
     if keyword in _PARM_KEYWORDS:
@@ -1341,16 +1444,9 @@ def explode_job(  # noqa: C901 -- complexity baseline, split when touched
                 # So are its EXEC keyword overrides (PARM.PS1='&SYSUID') and
                 # symbolic values (HLQ=&SYSUID). A symbolic value keeps "&&":
                 # it may carry a temporary dataset name (TMP='&&WORK').
-                step.parameters = {
-                    key: substitute_parameters_dict(
-                        {key: value},
-                        step_scope,
-                        allow_undefined=True,
-                        collapse_double_amp=key.upper().partition(".")[0]
-                        in _EXEC_OVERRIDE_KEYWORDS,
-                    )[key]
-                    for key, value in step.parameters.items()
-                }
+                step.parameters = _substitute_proc_call_parameters(
+                    step.parameters, step_scope, allow_undefined=True
+                )
                 # Expand procedure
                 expanded = _explode_step(
                     step=step,

{jcl_parser-2.0.1 → jcl_parser-2.0.3}/jclparser/model.py

@@ -542,10 +542,11 @@ class Step:
     dd_override_concatenations: dict[str, list[DDStatement]] = field(
         default_factory=dict
     )
-    #: The job's SET symbols as they stood when this step's EXEC was read,
-    #: kept only when a later ``SET`` redefines one of them (``None``
-    #: otherwise): a SET value is in effect from the SET statement onward, so
-    #: this step's ``&SYM`` references must not see the redefinition.
+    #: The job's SET symbols as they stood when this step's EXEC was read
+    #: (for a step inside a PROC body, the procedure's own SETs), kept only
+    #: when a later ``SET`` redefines one of them (``None`` otherwise): a SET
+    #: value is in effect from the SET statement onward, so this step's
+    #: ``&SYM`` references must not see the redefinition.
     set_symbols: dict[str, str] | None = field(default=None, repr=False, compare=False)
 
     @property

{jcl_parser-2.0.1 → jcl_parser-2.0.3}/jclparser/parser.py

@@ -137,7 +137,7 @@ def _parse_jcl_core(
     )
 
     scanned = _merge_source_lines(content)
-    job = _build_job(scanned, config)
+    job = _build_job(scanned, config, strict=strict_mode)
 
     # Anything the line-classification pass discarded or repaired (a comment
     # field that looked like operands, an unterminated CNTL block) is reported
@@ -156,7 +156,7 @@ def _parse_jcl_core(
     if strict_mode and not job.steps:
         enforce_strict_dd_structure(content)
 
-    _resolve_references(job, expanded_includes)
+    _resolve_references(job, expanded_includes, strict=strict_mode)
 
     if strict_mode and validate_job:
         _raise_on_fatal_findings(job)
@@ -208,13 +208,17 @@ def _expand_includes(
     return content, expanded
 
 
-def _build_job(scanned: ScannedSource, config: ParserConfig | None) -> Job:
+def _build_job(
+    scanned: ScannedSource, config: ParserConfig | None, *, strict: bool = False
+) -> Job:
     """Run the parse loop over the scanned source lines.
 
     The builder holds all loop state and has one handler per statement type
     (see jclparser/_job_builder.py).
     """
-    builder = JobBuilder(dialect=config.dialect if config is not None else "standard")
+    builder = JobBuilder(
+        dialect=config.dialect if config is not None else "standard", strict=strict
+    )
     for line, line_num in scanned.merged_lines:
         # Handle inline data continuation
         if builder.consume_inline_data(line):
@@ -250,7 +254,9 @@ def _build_job(scanned: ScannedSource, config: ParserConfig | None) -> Job:
     return builder.finish()
 
 
-def _resolve_references(job: Job, expanded_includes: list[str]) -> None:
+def _resolve_references(
+    job: Job, expanded_includes: list[str], *, strict: bool = False
+) -> None:
     """Post-parse passes that need the whole job: INCLUDEs, OUTPUT=, SET."""
     # Record any INCLUDE members that were expanded during pre-processing.
     # Members resolved before the main parse loop never reach the INCLUDE
@@ -271,7 +277,7 @@ def _resolve_references(job: Job, expanded_includes: list[str]) -> None:
             "resolving %d SET symbol(s) across DD statements",
             len(job.set_statements),
         )
-        resolve_set_symbols(job)
+        resolve_set_symbols(job, strict=strict)
 
 
 def _raise_on_fatal_findings(job: Job) -> None:

{jcl_parser-2.0.1 → jcl_parser-2.0.3}/jclparser/sql_detector.py

@@ -78,6 +78,15 @@ NON_SQL_CONTROL_PROGRAMS: frozenset[str] = frozenset(
     }
 )
 
+# DB2 online-utility programs.  Their in-stream data is utility control
+# statements (``RUNSTATS ... UPDATE ALL``, ``MODIFY RECOVERY ... DELETE AGE(n)``)
+# whose options reuse SQL verbs; only the SQL inside an ``EXEC SQL ...
+# ENDEXEC`` utility statement is SQL.
+DB2_UTILITY_CONTROL_PROGRAMS: frozenset[str] = frozenset({"DSNUTILB"})
+
+_EXEC_SQL_RE = re.compile(r"\bEXEC\s+SQL\b", re.IGNORECASE)
+_ENDEXEC_RE = re.compile(r"\bENDEXEC\b", re.IGNORECASE)
+
 # SYSTSIN holds TSO commands (``DELETE``, ``RENAME`` and ``ALTER`` are TSO
 # commands too); only the DB2 DSN subcommands in it are SQL-related.
 _TSO_COMMAND_DD = "SYSTSIN"
@@ -178,6 +187,39 @@ def detect_sql_in_text(text: str) -> list[tuple[str, str]]:
     return results
 
 
+def detect_sql_in_utility_statements(text: str) -> list[tuple[str, str]]:
+    """Scan DB2 utility control statements for SQL.
+
+    Only ``EXEC SQL ... ENDEXEC`` utility statements contribute: the
+    ``EXEC SQL`` marker itself (``EMBEDDED``) and the SQL statement keywords
+    between it and ``ENDEXEC``, which may share a line with either.  Every
+    other utility statement (``RUNSTATS``, ``REORG``, ``MODIFY RECOVERY``, ...)
+    is skipped, so options such as ``UPDATE ALL`` or ``DELETE AGE(90)`` are
+    not reported as DML.
+    """
+    results: list[tuple[str, str]] = []
+    in_exec_sql = False
+    for raw_line in split_source_lines(text):
+        rest = raw_line.strip()
+        while rest:
+            if not in_exec_sql:
+                start = _EXEC_SQL_RE.search(rest)
+                if start is None:
+                    break
+                results.append(("EXEC SQL", _KW_CATEGORY["EXEC SQL"]))
+                in_exec_sql = True
+                rest = rest[start.end() :]
+                continue
+            end = _ENDEXEC_RE.search(rest)
+            body = rest if end is None else rest[: end.start()]
+            results.extend(detect_sql_in_text(body))
+            if end is None:
+                break
+            in_exec_sql = False
+            rest = rest[end.end() :]
+    return results
+
+
 def detect_sql_in_dd(dd: DDStatement) -> list[tuple[str, str]]:
     """Scan a single DD statement's inline data for SQL.
 
@@ -196,7 +238,10 @@ def detect_sql_in_step(step: Step) -> SQLDetectionResult:
     the inline data of every DD statement.
     In-stream data of a program in :data:`NON_SQL_CONTROL_PROGRAMS` is not
     scanned, and in a ``SYSTSIN`` DD (TSO commands) only DB2 subcommands such
-    as ``BIND`` count.
+    as ``BIND`` count.  For a program in
+    :data:`DB2_UTILITY_CONTROL_PROGRAMS` only ``EXEC SQL ... ENDEXEC``
+    utility statements are scanned
+    (:func:`detect_sql_in_utility_statements`).
     """
     result = SQLDetectionResult(
         step_name=step.name,
@@ -204,15 +249,21 @@ def detect_sql_in_step(step: Step) -> SQLDetectionResult:
         is_db2_program=is_db2_program(step.program)
         or bool(step.procedure and step.procedure.upper() in DB2_PROCEDURES),
     )
-    if step.program and step.program.upper() in NON_SQL_CONTROL_PROGRAMS:
+    program = step.program.upper() if step.program else ""
+    if program in NON_SQL_CONTROL_PROGRAMS:
         return result
+    utility_control = program in DB2_UTILITY_CONTROL_PROGRAMS
     is_tso_input = False
     for dd in step.dd_statements:
         # An unnamed DD is a concatenation member of the last named one, so
         # ``//  DD *`` after ``//SYSTSIN DD ...`` is TSO input too.
         if dd.name:
             is_tso_input = dd.name.upper() == _TSO_COMMAND_DD
-        for kw, cat in detect_sql_in_dd(dd):
+        if utility_control:
+            found = detect_sql_in_utility_statements(dd.inline_data or "")
+        else:
+            found = detect_sql_in_dd(dd)
+        for kw, cat in found:
             if is_tso_input and cat not in _TSO_SQL_CATEGORIES:
                 continue
             result.sql_statements.append((kw, cat, dd.name))

{jcl_parser-2.0.1 → jcl_parser-2.0.3}/jclparser/utils.py

@@ -10,6 +10,7 @@ from __future__ import annotations
 import builtins
 import errno
 import re
+from collections.abc import Sequence
 from pathlib import Path
 from typing import Any
 
@@ -94,21 +95,56 @@ def jcl_value_text(value: Any) -> str:
     return str(value)
 
 
-def is_continued_card_image(line: str) -> bool:
+def _continues_on_next_record(line: str) -> bool:
+    """True if a 72-character ``//`` statement record leaves its operand open
+    at column 71: a trailing comma, or an unclosed quoted string."""
+    if not line.startswith("//") or line.startswith("//*"):
+        return False
+    return line[70] == "," or line[:71].count("'") % 2 == 1
+
+
+def _is_continuation_record(line: str) -> bool:
+    """True if *line* is a ``//`` record with a blank name field and operands."""
+    return line.startswith("// ") and bool(line[3:].strip())
+
+
+def _has_column_72_indicator(line: str, next_line: str | None) -> bool:
+    """True if column 72 of a 72-character record is a continuation indicator.
+
+    Such a record is a fixed-format card whose sequence field was trimmed.
+    A blank column 71 says so on its own. With column 71 non-blank, the
+    record could equally be an editor line whose last character is content,
+    so column 72 counts as the indicator only when the operand is left open
+    at column 71 (trailing comma or unclosed quote) *and* the next record,
+    *next_line*, is a continuation (issue #80).
+    """
+    if len(line) != 72 or line[71].isspace():
+        return False
+    if line[70] == " ":
+        return True
+    return (
+        next_line is not None
+        and _continues_on_next_record(line)
+        and _is_continuation_record(next_line)
+    )
+
+
+def is_continued_card_image(line: str, next_line: str | None = None) -> bool:
     """Return ``True`` if raw record *line* is a card image continued in column 72.
 
     Exactly the records :func:`normalize_jcl_line` takes a non-blank column-72
-    continuation indicator off. For such a record the statement text fills
-    columns 1-71, so blanks up to column 71 are known to be coded -- which
-    matters when a quoted string is continued: it runs through column 71
-    and its trailing blanks are part of the value (issue #65).
+    continuation indicator off (*next_line* is the raw record after *line*,
+    as there). For such a record the statement text fills columns 1-71, so
+    blanks up to column 71 are known to be coded -- which matters when a
+    quoted string is continued: it runs through column 71 and its trailing
+    blanks are part of the value (issue #65).
     """
-    if len(line) >= 80 or (len(line) == 72 and line[70] == " "):
+    if len(line) >= 80:
         return not line[71].isspace()
-    return False
+    return _has_column_72_indicator(line, next_line)
 
 
-def normalize_jcl_line(line: str) -> str:
+def normalize_jcl_line(line: str, next_line: str | None = None) -> str:
     """
     Normalize JCL line by removing extra whitespace and handling continuations.
 
@@ -123,13 +159,18 @@ def normalize_jcl_line(line: str) -> str:
     indicating a true mainframe fixed-width record.  Lines shorter than 80
     characters are assumed to come from a modern text editor where all
     characters are actual JCL content and must not be truncated — with one
-    exception: a record of exactly 72 characters whose column 71 is blank and
-    whose column 72 is not is a fixed-format record that had its trailing
-    sequence field trimmed, so its column-72 character is a continuation
-    indicator rather than content.
+    exception: a record of exactly 72 characters whose column 72 is a
+    continuation indicator is a fixed-format record that had its trailing
+    sequence field trimmed. That is the case when column 71 is blank, or when
+    column 71 leaves the operand open (a trailing comma or an unclosed quoted
+    string) and *next_line*, the raw record after *line*, is a continuation
+    record (``//`` and a blank name field). Without *next_line* only the
+    blank-column-71 case applies. Use :func:`normalize_jcl_lines` for a whole
+    source.
 
     Args:
         line: Raw JCL line
+        next_line: The raw record after *line*, if any
 
     Returns:
         Normalized line
@@ -138,10 +179,7 @@ def normalize_jcl_line(line: str) -> str:
     # column-72 continuation indicator (0-indexed 71).  Only do this for full
     # 80-column mainframe records; shorter lines are modern text-editor JCL
     # where every character is content.
-    if len(line) >= 80:
-        line = line[:71]
-    elif len(line) == 72 and line[70] == " " and not line[71].isspace():
-        # Sequence field trimmed away, continuation indicator left behind.
+    if len(line) >= 80 or _has_column_72_indicator(line, next_line):
         line = line[:71]
 
     # Remove trailing whitespace but preserve leading spaces for continuation detection
@@ -153,6 +191,23 @@ def normalize_jcl_line(line: str) -> str:
     return line
 
 
+def normalize_jcl_lines(raw_lines: Sequence[str]) -> list[str]:
+    """:func:`normalize_jcl_line` every record of *raw_lines*, with lookahead."""
+    return [
+        normalize_jcl_line(line, raw_lines[i + 1] if i + 1 < len(raw_lines) else None)
+        for i, line in enumerate(raw_lines)
+    ]
+
+
+#: Longest value a symbol substitution may produce: a resolved SET value, a
+#: pre-resolved SET table entry, or the characters one ``substitute_symbols()``
+#: call splices in. A safety limit against chained symbols that double at each
+#: step (``SET A2='&A1&A1'`` ...), deliberately far above the z/OS 255-character
+#: symbol limit so every currently accepted job still parses. Past it the text
+#: is left unresolved with a warning, or the strict paths raise.
+MAX_SYMBOL_VALUE_LENGTH: int = 65536
+
+
 #: Largest PROC / INCLUDE library member the loaders will read. Members are
 #: named by the JCL being parsed, so an oversized file is refused (read as
 #: missing, with a warning) rather than pulled into memory whole.

{jcl_parser-2.0.1 → jcl_parser-2.0.3}/pyproject.toml

@@ -4,10 +4,10 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "jcl-parser"
-version = "2.0.1"
+version = "2.0.3"
 description = "Parse IBM JCL (Job Control Language) scripts into Python objects"
 authors = [
-    {name = "JCL Parser Team", email = "team@jclparser.dev"}
+    {name = "Samuel Dion", email = "sam94dion@gmail.com"}
 ]
 readme = "README.md"
 license = {text = "AGPL-3.0-or-later"}