jcl-parser 2.0.2__tar.gz → 2.0.3__tar.gz

{jcl_parser-2.0.2 → jcl_parser-2.0.3}/PKG-INFO

@@ -1,8 +1,8 @@
 Metadata-Version: 2.4
 Name: jcl-parser
-Version: 2.0.2
+Version: 2.0.3
 Summary: Parse IBM JCL (Job Control Language) scripts into Python objects
-Author-email: JCL Parser Team <team@jclparser.dev>
+Author-email: Samuel Dion <sam94dion@gmail.com>
 License: AGPL-3.0-or-later
 Project-URL: Homepage, https://github.com/landparty/jcl-parser
 Project-URL: Repository, https://github.com/landparty/jcl-parser

{jcl_parser-2.0.2 → jcl_parser-2.0.3}/jcl_parser.egg-info/PKG-INFO

@@ -1,8 +1,8 @@
 Metadata-Version: 2.4
 Name: jcl-parser
-Version: 2.0.2
+Version: 2.0.3
 Summary: Parse IBM JCL (Job Control Language) scripts into Python objects
-Author-email: JCL Parser Team <team@jclparser.dev>
+Author-email: Samuel Dion <sam94dion@gmail.com>
 License: AGPL-3.0-or-later
 Project-URL: Homepage, https://github.com/landparty/jcl-parser
 Project-URL: Repository, https://github.com/landparty/jcl-parser

{jcl_parser-2.0.2 → jcl_parser-2.0.3}/jclparser/__init__.py

@@ -20,7 +20,7 @@ application, for example::
 import logging as _logging
 
 # Package metadata
-__version__ = "2.0.2"  # x-release-please-version
+__version__ = "2.0.3"  # x-release-please-version
 __author__ = "JCL Parser Team"
 __description__ = "Parse IBM JCL scripts into Python objects"
 

{jcl_parser-2.0.2 → jcl_parser-2.0.3}/jclparser/_job_builder.py

@@ -174,6 +174,10 @@ class JobBuilder:
         #: only if some SET symbol is redefined after it was first set.
         self.job_level_steps: list[Step] = []
         self.set_redefined = False
+        #: Steps of in-stream / cataloged procedures with the procedure they
+        #: belong to; finish() prunes their ``set_symbols`` snapshot the same
+        #: way (against the procedure's own final SET table).
+        self.procedure_steps: list[tuple[Procedure, Step]] = []
         self.pending_exported: set[str] = set()
         self.pending_jcllib: list[str] = []
         self.pending_includes: list[str] = []
@@ -315,6 +319,11 @@ class JobBuilder:
                 **self.job.set_statements,
             }
             self.job_level_steps.append(self.current_step)
+        elif self.current_procedure is not None:
+            # A SET inside the procedure is in effect from where it is coded,
+            # so this step sees the procedure's SET values as they stand now.
+            self.current_step.set_symbols = dict(self.current_procedure.set_statements)
+            self.procedure_steps.append((self.current_procedure, self.current_step))
         self.in_joblib_concat = False  # JOBLIB concatenation ends at first EXEC
         self.concat_override_key = None
 
@@ -820,6 +829,12 @@ class JobBuilder:
                 for k, v in (step.set_symbols or {}).items()
             ):
                 step.set_symbols = None
+        for procedure, step in self.procedure_steps:
+            if all(
+                procedure.set_statements.get(k) == v
+                for k, v in (step.set_symbols or {}).items()
+            ):
+                step.set_symbols = None
 
     def _warn_orphaned_dds(self, job: Job) -> None:
         # orphaned_dds is only ever non-empty here when the whole job had no

{jcl_parser-2.0.2 → jcl_parser-2.0.3}/jclparser/explosion.py

@@ -569,6 +569,8 @@ def expand_proc_invocation(
             else:
                 # Clone the step
                 new_step = _clone_step_for_explosion(proc_step)
+                step_scope = _proc_step_scope(expansion_scope, proc_step)
+                new_step.set_symbols = None
 
                 # Generate qualified step name
                 if new_step.name:
@@ -584,18 +586,28 @@ def expand_proc_invocation(
                 if new_step.program:
                     new_step.program = substitute_symbols(
                         new_step.program,
-                        expansion_scope,
+                        step_scope,
                         strict=strict,
                         warnings=warnings,
                     )
 
-                # Substitute symbols in step parameters
-                new_step.parameters = substitute_parameters_dict(
-                    new_step.parameters,
-                    expansion_scope,
-                    strict=strict,
-                    warnings=warnings,
-                )
+                # Substitute symbols in step parameters. A nested
+                # ``EXEC procname`` keeps "&&" in its symbolic values, as a
+                # top-level one does (TMP='&&WORK' is a temporary dataset).
+                if new_step.is_procedure:
+                    new_step.parameters = _substitute_proc_call_parameters(
+                        new_step.parameters,
+                        step_scope,
+                        strict=strict,
+                        warnings=warnings,
+                    )
+                else:
+                    new_step.parameters = substitute_parameters_dict(
+                        new_step.parameters,
+                        step_scope,
+                        strict=strict,
+                        warnings=warnings,
+                    )
                 if "COND" in new_step.parameters:
                     new_step.parameters["COND"] = _qualify_cond_parameter(
                         new_step.parameters["COND"], proc_steps, prefix
@@ -605,7 +617,7 @@ def expand_proc_invocation(
                 # for a nested ``EXEC procname``, its PROCSTEP.DDNAME overrides
                 for dd in _step_dds_and_overrides(new_step):
                     substitute_dd_statement(
-                        dd, expansion_scope, strict=strict, warnings=warnings
+                        dd, step_scope, strict=strict, warnings=warnings
                     )
 
                 result.append(new_step)
@@ -616,6 +628,24 @@ def expand_proc_invocation(
     return expanded_steps
 
 
+def _proc_step_scope(expansion_scope: SymbolScope, proc_step: Step) -> SymbolScope:
+    """Return *expansion_scope* with the PROC SETs in effect at *proc_step*.
+
+    A SET inside a procedure is in effect from where it is coded: a step read
+    before a later SET redefined a symbol keeps the earlier value
+    (``Step.set_symbols``). Symbols it has no snapshot for still resolve
+    against the procedure's final SET table.
+    """
+    if proc_step.set_symbols is None:
+        return expansion_scope
+    return SymbolScope(
+        set_symbols={**expansion_scope.set_symbols, **proc_step.set_symbols},
+        proc_params=expansion_scope.proc_params,
+        exec_overrides=expansion_scope.exec_overrides,
+        system_provider=expansion_scope.system_provider,
+    )
+
+
 _WHOLE_SYMBOL_RE = re.compile(r"&([A-Z0-9@#$]+)\.?")
 
 
@@ -1014,6 +1044,33 @@ _EXEC_OVERRIDE_KEYWORDS = frozenset(
 _PARM_KEYWORDS = frozenset({"PARM", "PARMDD"})
 
 
+def _substitute_proc_call_parameters(
+    parameters: dict[str, Any],
+    scope: SymbolScope,
+    strict: bool = False,
+    allow_undefined: bool = False,
+    warnings: list[str] | None = None,
+) -> dict[str, Any]:
+    """Substitute symbols in the parameters of an ``EXEC procname`` statement.
+
+    EXEC keyword overrides (``PARM.PS1='A&&B'``) collapse "&&" to "&" like any
+    other step parameter; a symbolic value keeps it, because it may carry a
+    temporary dataset name (``TMP='&&WORK'``) into the procedure's DDs.
+    """
+    return {
+        key: substitute_parameters_dict(
+            {key: value},
+            scope,
+            strict=strict,
+            allow_undefined=allow_undefined,
+            warnings=warnings,
+            collapse_double_amp=key.upper().partition(".")[0]
+            in _EXEC_OVERRIDE_KEYWORDS,
+        )[key]
+        for key, value in parameters.items()
+    }
+
+
 def _set_exec_keyword(step: Step, keyword: str, value: Any) -> None:
     """Override (or, for an empty value, nullify) *keyword* on *step*."""
     if keyword in _PARM_KEYWORDS:
@@ -1387,16 +1444,9 @@ def explode_job(  # noqa: C901 -- complexity baseline, split when touched
                 # So are its EXEC keyword overrides (PARM.PS1='&SYSUID') and
                 # symbolic values (HLQ=&SYSUID). A symbolic value keeps "&&":
                 # it may carry a temporary dataset name (TMP='&&WORK').
-                step.parameters = {
-                    key: substitute_parameters_dict(
-                        {key: value},
-                        step_scope,
-                        allow_undefined=True,
-                        collapse_double_amp=key.upper().partition(".")[0]
-                        in _EXEC_OVERRIDE_KEYWORDS,
-                    )[key]
-                    for key, value in step.parameters.items()
-                }
+                step.parameters = _substitute_proc_call_parameters(
+                    step.parameters, step_scope, allow_undefined=True
+                )
                 # Expand procedure
                 expanded = _explode_step(
                     step=step,

{jcl_parser-2.0.2 → jcl_parser-2.0.3}/jclparser/model.py

@@ -542,10 +542,11 @@ class Step:
     dd_override_concatenations: dict[str, list[DDStatement]] = field(
         default_factory=dict
     )
-    #: The job's SET symbols as they stood when this step's EXEC was read,
-    #: kept only when a later ``SET`` redefines one of them (``None``
-    #: otherwise): a SET value is in effect from the SET statement onward, so
-    #: this step's ``&SYM`` references must not see the redefinition.
+    #: The job's SET symbols as they stood when this step's EXEC was read
+    #: (for a step inside a PROC body, the procedure's own SETs), kept only
+    #: when a later ``SET`` redefines one of them (``None`` otherwise): a SET
+    #: value is in effect from the SET statement onward, so this step's
+    #: ``&SYM`` references must not see the redefinition.
     set_symbols: dict[str, str] | None = field(default=None, repr=False, compare=False)
 
     @property

{jcl_parser-2.0.2 → jcl_parser-2.0.3}/jclparser/sql_detector.py

@@ -78,6 +78,15 @@ NON_SQL_CONTROL_PROGRAMS: frozenset[str] = frozenset(
     }
 )
 
+# DB2 online-utility programs.  Their in-stream data is utility control
+# statements (``RUNSTATS ... UPDATE ALL``, ``MODIFY RECOVERY ... DELETE AGE(n)``)
+# whose options reuse SQL verbs; only the SQL inside an ``EXEC SQL ...
+# ENDEXEC`` utility statement is SQL.
+DB2_UTILITY_CONTROL_PROGRAMS: frozenset[str] = frozenset({"DSNUTILB"})
+
+_EXEC_SQL_RE = re.compile(r"\bEXEC\s+SQL\b", re.IGNORECASE)
+_ENDEXEC_RE = re.compile(r"\bENDEXEC\b", re.IGNORECASE)
+
 # SYSTSIN holds TSO commands (``DELETE``, ``RENAME`` and ``ALTER`` are TSO
 # commands too); only the DB2 DSN subcommands in it are SQL-related.
 _TSO_COMMAND_DD = "SYSTSIN"
@@ -178,6 +187,39 @@ def detect_sql_in_text(text: str) -> list[tuple[str, str]]:
     return results
 
 
+def detect_sql_in_utility_statements(text: str) -> list[tuple[str, str]]:
+    """Scan DB2 utility control statements for SQL.
+
+    Only ``EXEC SQL ... ENDEXEC`` utility statements contribute: the
+    ``EXEC SQL`` marker itself (``EMBEDDED``) and the SQL statement keywords
+    between it and ``ENDEXEC``, which may share a line with either.  Every
+    other utility statement (``RUNSTATS``, ``REORG``, ``MODIFY RECOVERY``, ...)
+    is skipped, so options such as ``UPDATE ALL`` or ``DELETE AGE(90)`` are
+    not reported as DML.
+    """
+    results: list[tuple[str, str]] = []
+    in_exec_sql = False
+    for raw_line in split_source_lines(text):
+        rest = raw_line.strip()
+        while rest:
+            if not in_exec_sql:
+                start = _EXEC_SQL_RE.search(rest)
+                if start is None:
+                    break
+                results.append(("EXEC SQL", _KW_CATEGORY["EXEC SQL"]))
+                in_exec_sql = True
+                rest = rest[start.end() :]
+                continue
+            end = _ENDEXEC_RE.search(rest)
+            body = rest if end is None else rest[: end.start()]
+            results.extend(detect_sql_in_text(body))
+            if end is None:
+                break
+            in_exec_sql = False
+            rest = rest[end.end() :]
+    return results
+
+
 def detect_sql_in_dd(dd: DDStatement) -> list[tuple[str, str]]:
     """Scan a single DD statement's inline data for SQL.
 
@@ -196,7 +238,10 @@ def detect_sql_in_step(step: Step) -> SQLDetectionResult:
     the inline data of every DD statement.
     In-stream data of a program in :data:`NON_SQL_CONTROL_PROGRAMS` is not
     scanned, and in a ``SYSTSIN`` DD (TSO commands) only DB2 subcommands such
-    as ``BIND`` count.
+    as ``BIND`` count.  For a program in
+    :data:`DB2_UTILITY_CONTROL_PROGRAMS` only ``EXEC SQL ... ENDEXEC``
+    utility statements are scanned
+    (:func:`detect_sql_in_utility_statements`).
     """
     result = SQLDetectionResult(
         step_name=step.name,
@@ -204,15 +249,21 @@ def detect_sql_in_step(step: Step) -> SQLDetectionResult:
         is_db2_program=is_db2_program(step.program)
         or bool(step.procedure and step.procedure.upper() in DB2_PROCEDURES),
     )
-    if step.program and step.program.upper() in NON_SQL_CONTROL_PROGRAMS:
+    program = step.program.upper() if step.program else ""
+    if program in NON_SQL_CONTROL_PROGRAMS:
         return result
+    utility_control = program in DB2_UTILITY_CONTROL_PROGRAMS
     is_tso_input = False
     for dd in step.dd_statements:
         # An unnamed DD is a concatenation member of the last named one, so
         # ``//  DD *`` after ``//SYSTSIN DD ...`` is TSO input too.
         if dd.name:
             is_tso_input = dd.name.upper() == _TSO_COMMAND_DD
-        for kw, cat in detect_sql_in_dd(dd):
+        if utility_control:
+            found = detect_sql_in_utility_statements(dd.inline_data or "")
+        else:
+            found = detect_sql_in_dd(dd)
+        for kw, cat in found:
             if is_tso_input and cat not in _TSO_SQL_CATEGORIES:
                 continue
             result.sql_statements.append((kw, cat, dd.name))

{jcl_parser-2.0.2 → jcl_parser-2.0.3}/pyproject.toml

@@ -4,10 +4,10 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "jcl-parser"
-version = "2.0.2"
+version = "2.0.3"
 description = "Parse IBM JCL (Job Control Language) scripts into Python objects"
 authors = [
-    {name = "JCL Parser Team", email = "team@jclparser.dev"}
+    {name = "Samuel Dion", email = "sam94dion@gmail.com"}
 ]
 readme = "README.md"
 license = {text = "AGPL-3.0-or-later"}