jcl-parser 2.3.0__tar.gz → 2.3.1__tar.gz

{jcl_parser-2.3.0 → jcl_parser-2.3.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: jcl-parser
-Version: 2.3.0
+Version: 2.3.1
 Summary: Parse IBM JCL (Job Control Language) scripts into Python objects
 Author-email: Samuel Dion <sam94dion@gmail.com>
 License-Expression: AGPL-3.0-or-later

{jcl_parser-2.3.0 → jcl_parser-2.3.1}/jcl_parser.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: jcl-parser
-Version: 2.3.0
+Version: 2.3.1
 Summary: Parse IBM JCL (Job Control Language) scripts into Python objects
 Author-email: Samuel Dion <sam94dion@gmail.com>
 License-Expression: AGPL-3.0-or-later

{jcl_parser-2.3.0 → jcl_parser-2.3.1}/jclparser/__init__.py

@@ -20,7 +20,7 @@ application, for example::
 import logging as _logging
 
 # Package metadata
-__version__ = "2.3.0"  # x-release-please-version
+__version__ = "2.3.1"  # x-release-please-version
 __author__ = "JCL Parser Team"
 __description__ = "Parse IBM JCL scripts into Python objects"
 

{jcl_parser-2.3.0 → jcl_parser-2.3.1}/jclparser/_job_builder.py

@@ -621,11 +621,31 @@ class JobBuilder:
             sink.append(member)
             # Found members are expanded before the parse loop runs, so an
             # INCLUDE that reaches here could not be loaded.
-            self._warnings_sink().append(
-                f"Line {stmt.line_number}: INCLUDE member {member} not found in "
-                "the JCLLIB / procedure libraries; its statements are missing "
-                "from the job"
+            self._warnings_sink().append(self._include_warning(stmt, str(member)))
+
+    def _include_warning(self, stmt: Statement, member: str) -> str:
+        """The warning for an INCLUDE whose member could not be loaded."""
+        line = f"Line {stmt.line_number}: INCLUDE member"
+        missing = " its statements are missing from the job"
+        if "&" in member:
+            from .include_loader import _substitute_member_symbols
+
+            symbols = (
+                {**self.pending_sets, **self.job.set_statements}
+                if self.job
+                else self.pending_sets
             )
+            resolved = _substitute_member_symbols(member, symbols)
+            if "&" in resolved:
+                # Not a missing member: its name could not be built (#180).
+                return (
+                    f"{line} {member} uses a symbol with no SET value before "
+                    f"it;{missing}"
+                )
+            member = f"{resolved} (from {member})"
+        return (
+            f"{line} {member} not found in the JCLLIB / procedure libraries;{missing}"
+        )
 
     def _on_conditional(self, stmt: Statement, line_num: int) -> None:
         # Conditional logic - track conditionals properly

{jcl_parser-2.3.0 → jcl_parser-2.3.1}/jclparser/_statement_parser.py

@@ -803,7 +803,7 @@ def _extract_jcllib_order(content: str) -> list[str]:
     symbols: dict[str, str] = {}
     lines = split_source_lines(content)
     for index, line in enumerate(lines):
-        _record_set_symbols(line.lstrip(), symbols)
+        _record_set_symbols(line.lstrip(), symbols, lines, index + 1)
         params_part = _jcllib_operands(line)
         if params_part is None:
             continue

{jcl_parser-2.3.0 → jcl_parser-2.3.1}/jclparser/explosion.py

@@ -1898,8 +1898,9 @@ class _IncludeExpandingLoader(ProcedureLoader):
         from .utils import split_source_lines
 
         seen = dict(symbols) if symbols is not None else {}
-        for line in split_source_lines(expanded_text):
-            _record_set_symbols(line.rstrip(), seen)
+        lines = split_source_lines(expanded_text)
+        for index, line in enumerate(lines):
+            _record_set_symbols(line.rstrip(), seen, lines, index + 1)
             missing = _extract_include_member(line.rstrip())
             if not missing:
                 continue

{jcl_parser-2.3.0 → jcl_parser-2.3.1}/jclparser/include_loader.py

@@ -26,6 +26,7 @@ import errno
 import logging
 import os
 from abc import ABC, abstractmethod
+from collections.abc import Sequence
 from pathlib import Path
 from typing import NamedTuple
 
@@ -335,21 +336,22 @@ class _IncludeExpander:
         for index, line in enumerate(lines):
             expanded = None
             if index not in data_indices:
-                expanded = self._expand_statement(line, index + 1, depth, active)
+                expanded = self._expand_statement(lines, index, depth, active)
             if expanded is None:
                 expanded = [(line, LineOrigin(None, index + 1))]
             result.extend(expanded)
         return result
 
     def _expand_statement(
-        self, line: str, line_number: int, depth: int, active: frozenset[str]
+        self, lines: list[str], index: int, depth: int, active: frozenset[str]
     ) -> list[_Entry] | None:
-        """The expansion of *line* if it is a resolvable INCLUDE, else None."""
+        """The expansion of ``lines[index]`` if it is a resolvable INCLUDE."""
+        line_number = index + 1
         # Preserve leading whitespace: only strip the line terminator so that
         # in-stream data lines (which may contain '//' after leading spaces)
         # are never mistaken for JCL statements.
-        statement_line = line.rstrip("\r\n")
-        _record_set_symbols(statement_line, self.symbols)
+        statement_line = lines[index].rstrip("\r\n")
+        _record_set_symbols(statement_line, self.symbols, lines, index + 1)
         member = _extract_include_member(statement_line)
         if member is None:
             return None
@@ -413,12 +415,19 @@ def _member_origin(origin: LineOrigin, member: str, include_line: int) -> LineOr
     return LineOrigin(origin.member, include_line, origin.member_line)
 
 
-def _record_set_symbols(line: str, symbols: dict[str, str]) -> None:
+def _record_set_symbols(
+    line: str,
+    symbols: dict[str, str],
+    lines: Sequence[str] = (),
+    follow: int = 0,
+) -> None:
     """Record the assignments of a ``// SET`` statement *line* into *symbols*.
 
     Values are resolved against the symbols already recorded, mirroring the
     parser's left-to-right SET evaluation. Lines that are not a SET statement
-    are ignored.
+    are ignored. A SET whose operand field ends in a comma is continued on the
+    following ``//`` records: *lines* from index *follow* on (the record after
+    *line*) are read for its remaining assignments (#180).
     """
     if not line.startswith("//") or line.startswith("//*"):
         return
@@ -436,6 +445,7 @@ def _record_set_symbols(line: str, symbols: dict[str, str]) -> None:
     from .utils import is_positional_key, jcl_value_text
 
     statement, _ = split_comment_field(line)
+    statement = _continue_operands(statement, lines, follow)
     stmt = parse_statement(statement)
     if stmt is None or stmt.type != "SET":
         return
@@ -445,6 +455,28 @@ def _record_set_symbols(line: str, symbols: dict[str, str]) -> None:
         symbols[key.upper()] = _resolve_set_value(jcl_value_text(value), symbols)
 
 
+def _continue_operands(statement: str, lines: Sequence[str], follow: int) -> str:
+    """Append the operands of the continuation records of *statement*.
+
+    A statement whose operand field ends in a comma continues on the next
+    ``//`` record that has a blank in column 3; its operand field (up to the
+    first blank) is appended. ``//*`` comment records in between are skipped.
+    """
+    from ._statement_parser import split_comment_field
+
+    statement = statement.rstrip()
+    while statement.endswith(",") and follow < len(lines):
+        cont = lines[follow].rstrip("\r\n")
+        follow += 1
+        if cont.startswith("//*"):
+            continue
+        if not cont.startswith("// ") or not cont[2:].strip():
+            break
+        operands, _ = split_comment_field("//  " + cont[2:].lstrip())
+        statement += operands[2:].strip()
+    return statement
+
+
 def _substitute_member_symbols(member: str, symbols: dict[str, str]) -> str:
     """Resolve ``&SYMBOL`` references in an INCLUDE member name.
 

{jcl_parser-2.3.0 → jcl_parser-2.3.1}/jclparser/lineage.py

@@ -22,7 +22,8 @@ share a load library or DBRMLIB do not depend on one another (issue #145).  The
 A DD's role comes from its ``DISP`` status: ``NEW`` / ``MOD`` write the
 dataset, ``SHR`` / ``OLD`` read it (``OLD`` may also update it in place, which
 DISP alone can't tell).  A normal disposition of ``DELETE`` makes any status
-but ``NEW`` a read (a delete runs after the dataset's producer), and
+but ``NEW`` a read (a delete runs after the dataset's producer and after
+every other job that reads or writes it), and
 ``(NEW,DELETE)`` -- a scratch dataset -- neither.  Temporary datasets (``&&TEMP``), ``NULLFILE``,
 ``SYSOUT`` and DDs without a ``DISP`` are left out; a ``DSN=*.STEP.DD``
 backward reference counts as the dataset it points at.
@@ -93,6 +94,9 @@ class DatasetUse:
     dataset: str
     disp: str
     line_number: int = 0
+    #: The DD deletes the dataset (normal disposition ``DELETE``, any status
+    #: but ``NEW``); it is a read that must also follow every other use.
+    deletes: bool = False
 
     def to_dict(self) -> dict[str, Any]:
         return {
@@ -102,6 +106,7 @@ class DatasetUse:
             "dataset": self.dataset,
             "disp": self.disp,
             "line_number": self.line_number,
+            "deletes": self.deletes,
         }
 
 
@@ -131,7 +136,8 @@ class DatasetLineage:
 
 @dataclass(frozen=True)
 class JobDependency:
-    """*consumer* reads datasets that *producer* writes."""
+    """*consumer* reads datasets that *producer* writes, or deletes datasets
+    that *producer* also reads or writes (it must run after that job)."""
 
     producer: str
     consumer: str
@@ -257,6 +263,7 @@ def _record_uses(job_id: str, job: Job, datasets: dict[str, DatasetLineage]) ->
                 dataset=dsn,
                 disp=evt.disp_status,
                 line_number=evt.line_number,
+                deletes=evt.disp_normal == "DELETE" and evt.disp_status != "NEW",
             )
             role = _role(evt.disp_status, evt.disp_normal)
             if role == "write":
@@ -313,12 +320,41 @@ def _dependencies(datasets: Mapping[str, DatasetLineage]) -> list[JobDependency]
                         labels = edges.setdefault((producer, consumer), [])
                         if label not in labels:
                             labels.append(label)
+    _add_delete_edges(datasets, edges)
     return [
         JobDependency(producer, consumer, tuple(sorted(keys)))
         for (producer, consumer), keys in sorted(edges.items())
     ]
 
 
+def _add_delete_edges(
+    datasets: Mapping[str, DatasetLineage], edges: dict[tuple[str, str], list[str]]
+) -> None:
+    """A job that deletes a dataset runs after every other job that uses it.
+
+    The producer edge is already there (a delete is a read); this adds one
+    from each other reader of the dataset -- or of its library or a member
+    of it -- so the deleter cannot be ordered before a job that still needs
+    the dataset (#179). Two jobs that each delete a dataset the other reads
+    depend on each other, a cycle.
+    """
+    for deleted, lineage in datasets.items():
+        deleters = _unique(u.job for u in lineage.readers if u.deletes)
+        if not deleters:
+            continue
+        for used, other in datasets.items():
+            if used != deleted and not _feeds(used, deleted):
+                continue
+            # Label the edge with the more specific name (the member).
+            label = deleted if deleted != library_of(deleted) else used
+            for user in _unique([*other.reader_jobs, *other.writer_jobs]):
+                for deleter in deleters:
+                    if user != deleter:
+                        labels = edges.setdefault((user, deleter), [])
+                        if label not in labels:
+                            labels.append(label)
+
+
 def _run_order(
     jobs: list[str], dependencies: list[JobDependency]
 ) -> tuple[list[str], list[str]]:

{jcl_parser-2.3.0 → jcl_parser-2.3.1}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "jcl-parser"
-version = "2.3.0"
+version = "2.3.1"
 description = "Parse IBM JCL (Job Control Language) scripts into Python objects"
 authors = [
     {name = "Samuel Dion", email = "sam94dion@gmail.com"}