jcl-parser 2.2.0__tar.gz → 2.2.1__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (82) hide show
  1. {jcl_parser-2.2.0 → jcl_parser-2.2.1}/PKG-INFO +1 -1
  2. {jcl_parser-2.2.0 → jcl_parser-2.2.1}/jcl_parser.egg-info/PKG-INFO +1 -1
  3. {jcl_parser-2.2.0 → jcl_parser-2.2.1}/jclparser/__init__.py +1 -1
  4. {jcl_parser-2.2.0 → jcl_parser-2.2.1}/jclparser/_job_builder.py +18 -0
  5. {jcl_parser-2.2.0 → jcl_parser-2.2.1}/jclparser/_source_scanner.py +95 -6
  6. {jcl_parser-2.2.0 → jcl_parser-2.2.1}/jclparser/checks.py +40 -3
  7. {jcl_parser-2.2.0 → jcl_parser-2.2.1}/jclparser/explosion.py +198 -11
  8. {jcl_parser-2.2.0 → jcl_parser-2.2.1}/jclparser/include_loader.py +6 -2
  9. {jcl_parser-2.2.0 → jcl_parser-2.2.1}/jclparser/jes_statements.py +3 -0
  10. {jcl_parser-2.2.0 → jcl_parser-2.2.1}/jclparser/lineage.py +32 -3
  11. {jcl_parser-2.2.0 → jcl_parser-2.2.1}/jclparser/lint/runner.py +49 -11
  12. {jcl_parser-2.2.0 → jcl_parser-2.2.1}/jclparser/lint/suppress.py +34 -6
  13. {jcl_parser-2.2.0 → jcl_parser-2.2.1}/jclparser/model.py +3 -0
  14. {jcl_parser-2.2.0 → jcl_parser-2.2.1}/jclparser/parser.py +24 -3
  15. {jcl_parser-2.2.0 → jcl_parser-2.2.1}/jclparser/proc_loader.py +3 -0
  16. jcl_parser-2.2.1/jclparser/program_profiles/bpxbatch.py +116 -0
  17. jcl_parser-2.2.1/jclparser/program_profiles/dsnutilb.py +163 -0
  18. {jcl_parser-2.2.0 → jcl_parser-2.2.1}/jclparser/program_profiles/icetool.py +27 -17
  19. {jcl_parser-2.2.0 → jcl_parser-2.2.1}/pyproject.toml +1 -1
  20. jcl_parser-2.2.0/jclparser/program_profiles/bpxbatch.py +0 -68
  21. jcl_parser-2.2.0/jclparser/program_profiles/dsnutilb.py +0 -79
  22. {jcl_parser-2.2.0 → jcl_parser-2.2.1}/LICENSE +0 -0
  23. {jcl_parser-2.2.0 → jcl_parser-2.2.1}/README.md +0 -0
  24. {jcl_parser-2.2.0 → jcl_parser-2.2.1}/jcl_parser.egg-info/SOURCES.txt +0 -0
  25. {jcl_parser-2.2.0 → jcl_parser-2.2.1}/jcl_parser.egg-info/dependency_links.txt +0 -0
  26. {jcl_parser-2.2.0 → jcl_parser-2.2.1}/jcl_parser.egg-info/entry_points.txt +0 -0
  27. {jcl_parser-2.2.0 → jcl_parser-2.2.1}/jcl_parser.egg-info/requires.txt +0 -0
  28. {jcl_parser-2.2.0 → jcl_parser-2.2.1}/jcl_parser.egg-info/top_level.txt +0 -0
  29. {jcl_parser-2.2.0 → jcl_parser-2.2.1}/jclparser/__main__.py +0 -0
  30. {jcl_parser-2.2.0 → jcl_parser-2.2.1}/jclparser/_include_origins.py +0 -0
  31. {jcl_parser-2.2.0 → jcl_parser-2.2.1}/jclparser/_model_factories.py +0 -0
  32. {jcl_parser-2.2.0 → jcl_parser-2.2.1}/jclparser/_post_parse.py +0 -0
  33. {jcl_parser-2.2.0 → jcl_parser-2.2.1}/jclparser/_statement_parser.py +0 -0
  34. {jcl_parser-2.2.0 → jcl_parser-2.2.1}/jclparser/builtin_procs.py +0 -0
  35. {jcl_parser-2.2.0 → jcl_parser-2.2.1}/jclparser/cli.py +0 -0
  36. {jcl_parser-2.2.0 → jcl_parser-2.2.1}/jclparser/condition_parser.py +0 -0
  37. {jcl_parser-2.2.0 → jcl_parser-2.2.1}/jclparser/continuation_handler.py +0 -0
  38. {jcl_parser-2.2.0 → jcl_parser-2.2.1}/jclparser/dataset_lifecycle.py +0 -0
  39. {jcl_parser-2.2.0 → jcl_parser-2.2.1}/jclparser/errors.py +0 -0
  40. {jcl_parser-2.2.0 → jcl_parser-2.2.1}/jclparser/jcl_file_type.py +0 -0
  41. {jcl_parser-2.2.0 → jcl_parser-2.2.1}/jclparser/lint/__init__.py +0 -0
  42. {jcl_parser-2.2.0 → jcl_parser-2.2.1}/jclparser/lint/base.py +0 -0
  43. {jcl_parser-2.2.0 → jcl_parser-2.2.1}/jclparser/lint/config.py +0 -0
  44. {jcl_parser-2.2.0 → jcl_parser-2.2.1}/jclparser/lint/rules.py +0 -0
  45. {jcl_parser-2.2.0 → jcl_parser-2.2.1}/jclparser/metrics.py +0 -0
  46. {jcl_parser-2.2.0 → jcl_parser-2.2.1}/jclparser/parameter_ast.py +0 -0
  47. {jcl_parser-2.2.0 → jcl_parser-2.2.1}/jclparser/parameter_validators.py +0 -0
  48. {jcl_parser-2.2.0 → jcl_parser-2.2.1}/jclparser/plugins/__init__.py +0 -0
  49. {jcl_parser-2.2.0 → jcl_parser-2.2.1}/jclparser/plugins/base.py +0 -0
  50. {jcl_parser-2.2.0 → jcl_parser-2.2.1}/jclparser/profiling.py +0 -0
  51. {jcl_parser-2.2.0 → jcl_parser-2.2.1}/jclparser/program_profiles/__init__.py +0 -0
  52. {jcl_parser-2.2.0 → jcl_parser-2.2.1}/jclparser/program_profiles/_control.py +0 -0
  53. {jcl_parser-2.2.0 → jcl_parser-2.2.1}/jclparser/program_profiles/_invoked.py +0 -0
  54. {jcl_parser-2.2.0 → jcl_parser-2.2.1}/jclparser/program_profiles/adrdssu.py +0 -0
  55. {jcl_parser-2.2.0 → jcl_parser-2.2.1}/jclparser/program_profiles/base.py +0 -0
  56. {jcl_parser-2.2.0 → jcl_parser-2.2.1}/jclparser/program_profiles/binder.py +0 -0
  57. {jcl_parser-2.2.0 → jcl_parser-2.2.1}/jclparser/program_profiles/compilers.py +0 -0
  58. {jcl_parser-2.2.0 → jcl_parser-2.2.1}/jclparser/program_profiles/db2_batch.py +0 -0
  59. {jcl_parser-2.2.0 → jcl_parser-2.2.1}/jclparser/program_profiles/dfsrrc00.py +0 -0
  60. {jcl_parser-2.2.0 → jcl_parser-2.2.1}/jclparser/program_profiles/ftp.py +0 -0
  61. {jcl_parser-2.2.0 → jcl_parser-2.2.1}/jclparser/program_profiles/idcams.py +0 -0
  62. {jcl_parser-2.2.0 → jcl_parser-2.2.1}/jclparser/program_profiles/iebcompr.py +0 -0
  63. {jcl_parser-2.2.0 → jcl_parser-2.2.1}/jclparser/program_profiles/iebdg.py +0 -0
  64. {jcl_parser-2.2.0 → jcl_parser-2.2.1}/jclparser/program_profiles/iebgener.py +0 -0
  65. {jcl_parser-2.2.0 → jcl_parser-2.2.1}/jclparser/program_profiles/iebptpch.py +0 -0
  66. {jcl_parser-2.2.0 → jcl_parser-2.2.1}/jclparser/program_profiles/iebupdte.py +0 -0
  67. {jcl_parser-2.2.0 → jcl_parser-2.2.1}/jclparser/program_profiles/iefbr14.py +0 -0
  68. {jcl_parser-2.2.0 → jcl_parser-2.2.1}/jclparser/program_profiles/iehinitt.py +0 -0
  69. {jcl_parser-2.2.0 → jcl_parser-2.2.1}/jclparser/program_profiles/iehlist.py +0 -0
  70. {jcl_parser-2.2.0 → jcl_parser-2.2.1}/jclparser/program_profiles/iehprogm.py +0 -0
  71. {jcl_parser-2.2.0 → jcl_parser-2.2.1}/jclparser/program_profiles/ikjeft01.py +0 -0
  72. {jcl_parser-2.2.0 → jcl_parser-2.2.1}/jclparser/program_profiles/irxjcl.py +0 -0
  73. {jcl_parser-2.2.0 → jcl_parser-2.2.1}/jclparser/program_profiles/registry.py +0 -0
  74. {jcl_parser-2.2.0 → jcl_parser-2.2.1}/jclparser/program_profiles/sort.py +0 -0
  75. {jcl_parser-2.2.0 → jcl_parser-2.2.1}/jclparser/py.typed +0 -0
  76. {jcl_parser-2.2.0 → jcl_parser-2.2.1}/jclparser/rc_analysis.py +0 -0
  77. {jcl_parser-2.2.0 → jcl_parser-2.2.1}/jclparser/serialization.py +0 -0
  78. {jcl_parser-2.2.0 → jcl_parser-2.2.1}/jclparser/sql_detector.py +0 -0
  79. {jcl_parser-2.2.0 → jcl_parser-2.2.1}/jclparser/system_symbols.py +0 -0
  80. {jcl_parser-2.2.0 → jcl_parser-2.2.1}/jclparser/utils.py +0 -0
  81. {jcl_parser-2.2.0 → jcl_parser-2.2.1}/jclparser/validators.py +0 -0
  82. {jcl_parser-2.2.0 → jcl_parser-2.2.1}/setup.cfg +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: jcl-parser
3
- Version: 2.2.0
3
+ Version: 2.2.1
4
4
  Summary: Parse IBM JCL (Job Control Language) scripts into Python objects
5
5
  Author-email: Samuel Dion <sam94dion@gmail.com>
6
6
  License-Expression: AGPL-3.0-or-later
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: jcl-parser
3
- Version: 2.2.0
3
+ Version: 2.2.1
4
4
  Summary: Parse IBM JCL (Job Control Language) scripts into Python objects
5
5
  Author-email: Samuel Dion <sam94dion@gmail.com>
6
6
  License-Expression: AGPL-3.0-or-later
@@ -20,7 +20,7 @@ application, for example::
20
20
  import logging as _logging
21
21
 
22
22
  # Package metadata
23
- __version__ = "2.2.0" # x-release-please-version
23
+ __version__ = "2.2.1" # x-release-please-version
24
24
  __author__ = "JCL Parser Team"
25
25
  __description__ = "Parse IBM JCL scripts into Python objects"
26
26
 
@@ -304,6 +304,24 @@ class JobBuilder:
304
304
  self.pending_jes_typed.append(jes3)
305
305
  self.pending_jes_statements.append(raw_text)
306
306
 
307
+ def handle_jes3_data(self, line: str) -> None:
308
+ """Keep a record of a JES3 ``//*DATASET`` with that statement.
309
+
310
+ The ``END=`` terminator record ends the data and is not kept.
311
+ """
312
+ from .jes_statements import Jes3ControlStatement
313
+
314
+ typed = self.job.jes_typed_statements if self.job else self.pending_jes_typed
315
+ for stmt in reversed(typed):
316
+ if (
317
+ isinstance(stmt, Jes3ControlStatement)
318
+ and stmt.name.upper() == "DATASET"
319
+ ):
320
+ terminator = stmt.parameters.get("END", "").strip("'")
321
+ if not (terminator and line.startswith(terminator)):
322
+ stmt.data.append(line)
323
+ return
324
+
307
325
  # ------------------------------------------------------------------ #
308
326
  # Statement dispatch
309
327
  # ------------------------------------------------------------------ #
@@ -52,11 +52,19 @@ class ScannedSource(NamedTuple):
52
52
  control_data_indices: set[int]
53
53
  #: Findings about records that were discarded or repaired.
54
54
  warnings: list[str]
55
+ #: 0-based indices of the records of a JES3 ``//*DATASET`` input data
56
+ #: set (between ``//*DATASET`` and ``//*ENDDATASET`` or its ``END=``
57
+ #: terminator, the terminator included when it is not ENDDATASET).
58
+ jes3_data_indices: frozenset[int] = frozenset()
55
59
 
56
60
 
57
- def _merge_source_lines(content: str) -> ScannedSource:
61
+ def _merge_source_lines(content: str, dialect: str = "standard") -> ScannedSource:
58
62
  """Normalize, mark in-stream data, and merge continuation lines.
59
63
 
64
+ Under every dialect but ``jes2`` (where ``//*`` is only ever a comment),
65
+ the records of a JES3 ``//*DATASET`` ... ``//*ENDDATASET`` input data set
66
+ are data too (issue #162).
67
+
60
68
  Neither in-stream data nor CNTL control records may be parsed as JCL
61
69
  statements; the parse loop uses the returned index sets to route them.
62
70
  The warnings report an unterminated CNTL block and comment fields that
@@ -66,7 +74,7 @@ def _merge_source_lines(content: str) -> ScannedSource:
66
74
  raw_lines = split_source_lines(content)
67
75
  lines = normalize_jcl_lines(raw_lines)
68
76
 
69
- classifier = _RecordClassifier(lines)
77
+ classifier = _RecordClassifier(lines, jes3_datasets=dialect != "jes2")
70
78
  jcl_indices = classifier.run()
71
79
 
72
80
  handler = ContinuationLineHandler()
@@ -84,11 +92,16 @@ def _merge_source_lines(content: str) -> ScannedSource:
84
92
  # raw record is woven in (trailing blanks aside) rather than the
85
93
  # normalized one (issue #115). CNTL control records are statements
86
94
  # with JCL's column rules and stay normalized.
87
- data_lines = {i: _data_record(raw_lines[i]) for i in classifier.inline_data_indices}
95
+ data_lines = {
96
+ i: _data_record(raw_lines[i])
97
+ for i in classifier.inline_data_indices | classifier.jes3_data_indices
98
+ }
88
99
  merged_lines = _weave_lines(
89
100
  lines,
90
101
  merged_start,
91
- classifier.inline_data_indices | classifier.control_data_indices,
102
+ classifier.inline_data_indices
103
+ | classifier.control_data_indices
104
+ | classifier.jes3_data_indices,
92
105
  data_lines,
93
106
  )
94
107
 
@@ -113,6 +126,7 @@ def _merge_source_lines(content: str) -> ScannedSource:
113
126
  classifier.inline_data_indices,
114
127
  classifier.control_data_indices,
115
128
  warnings,
129
+ frozenset(classifier.jes3_data_indices),
116
130
  )
117
131
 
118
132
 
@@ -200,12 +214,19 @@ class _RecordClassifier:
200
214
  :attr:`control_data_indices` and :attr:`warnings`.
201
215
  """
202
216
 
203
- def __init__(self, lines: list[str]) -> None:
217
+ def __init__(self, lines: list[str], *, jes3_datasets: bool = True) -> None:
204
218
  self.lines = lines
205
219
  self.inline_data_indices: set[int] = set()
206
220
  self.control_data_indices: set[int] = set()
221
+ self.jes3_data_indices: set[int] = set()
207
222
  self.warnings: list[str] = []
208
223
 
224
+ # JES3 ``//*DATASET``: the records up to ``//*ENDDATASET`` (or the
225
+ # ``END=`` terminator) are the content of an input data set.
226
+ self.jes3_datasets = jes3_datasets
227
+ self.jes3_terminator: str | None = None
228
+ self.jes3_start_line = 0
229
+
209
230
  self.in_inline_data = False
210
231
  self.delimiter = "/*"
211
232
  # When the current in-stream dataset was opened with ``DD *`` (and
@@ -228,10 +249,67 @@ class _RecordClassifier:
228
249
  self.pending: tuple[str, str] | None = None
229
250
 
230
251
  def run(self) -> list[int]:
231
- return [i for i, line in enumerate(self.lines) if self._is_jcl(i, line)]
252
+ jcl = [i for i, line in enumerate(self.lines) if self._is_jcl(i, line)]
253
+ if self.jes3_terminator is not None:
254
+ self.warnings.append(
255
+ f"Line {self.jes3_start_line}: //*DATASET has no matching "
256
+ f"{self._jes3_terminator_name()}; its data runs to the end "
257
+ f"of the source"
258
+ )
259
+ return jcl
260
+
261
+ def _jes3_terminator_name(self) -> str:
262
+ return (
263
+ "//*ENDDATASET"
264
+ if self.jes3_terminator == _ENDDATASET
265
+ else f"END={self.jes3_terminator} terminator"
266
+ )
267
+
268
+ def _consume_jes3_data(self, i: int, line: str) -> bool:
269
+ """Handle a record inside a JES3 ``//*DATASET``; True if it was data.
270
+
271
+ The ``//*ENDDATASET`` record itself stays a JCL (JES3 control)
272
+ record; an ``END=`` terminator record is part of the data block.
273
+ """
274
+ if _jes3_command(line) == "ENDDATASET":
275
+ self.jes3_terminator = None
276
+ return False
277
+ if self.jes3_terminator != _ENDDATASET and line.startswith(
278
+ str(self.jes3_terminator)
279
+ ):
280
+ self.jes3_terminator = None
281
+ self.jes3_data_indices.add(i)
282
+ return True
283
+
284
+ def _opens_jes3_dataset(self, i: int, line: str) -> None:
285
+ """Note a ``//*DATASET`` record: data follows until its terminator.
286
+
287
+ A JES3 ``//*DATASET`` statement always codes ``DDNAME=``; without it
288
+ the record is an ordinary comment that happens to start with that
289
+ word (``//*DATASET DEFINITIONS FOR ...``) and opens no data (#164).
290
+ """
291
+ if not self.jes3_datasets or _jes3_command(line) != "DATASET":
292
+ return
293
+ terminator = _ENDDATASET
294
+ has_ddname = False
295
+ for param in line.strip()[3:].split(None, 1)[1:2]:
296
+ for item in param.split(None, 1)[0].split(","):
297
+ key, _, value = item.partition("=")
298
+ if key.upper() == "DDNAME" and value:
299
+ has_ddname = True
300
+ if key.upper() == "END" and value.strip("'"):
301
+ terminator = value.strip("'")
302
+ if not has_ddname:
303
+ return
304
+ self.jes3_terminator = terminator
305
+ self.jes3_start_line = i + 1
232
306
 
233
307
  def _is_jcl(self, i: int, line: str) -> bool:
234
308
  """Classify record *i*; return True if it is a JCL record."""
309
+ if self.jes3_terminator is not None and self._consume_jes3_data(i, line):
310
+ return False
311
+ if line.startswith("//*") and not self.in_inline_data:
312
+ self._opens_jes3_dataset(i, line)
235
313
  if self.in_inline_data and self._consume_inline_data(i, line):
236
314
  return False
237
315
  if self.in_cntl_block and self._consume_control_data(i, line):
@@ -371,6 +449,17 @@ class _RecordClassifier:
371
449
  return operation, operands
372
450
 
373
451
 
452
+ #: Marks a ``//*DATASET`` whose data ends at ``//*ENDDATASET``.
453
+ _ENDDATASET = "\0ENDDATASET"
454
+
455
+
456
+ def _jes3_command(line: str) -> str | None:
457
+ """The command of a JES3 ``//*COMMAND`` record (no blank after ``//*``)."""
458
+ if not line.startswith("//*") or len(line) < 4 or line[3].isspace():
459
+ return None
460
+ return line[3:].split(None, 1)[0].upper()
461
+
462
+
374
463
  #: Statements whose operands can open a data block in the pre-scan.
375
464
  _DATA_OPENING_OPERATIONS = frozenset({"DD", "XMIT"})
376
465
 
@@ -118,7 +118,41 @@ def _lint_file(
118
118
  if explode:
119
119
  job = job.explode()
120
120
  except _UNREADABLE as exc:
121
- return FileReport(_uri(path), [_parse_failure(exc)])
121
+ bare = _bare_member_job(path)
122
+ if bare is None:
123
+ return FileReport(_uri(path), [_parse_failure(exc)])
124
+ job = bare
125
+ return _lint_one(path, job, config)
126
+
127
+
128
+ def _bare_member_job(path: Path) -> Job | None:
129
+ """A PROC member with no PROC statement, as a job to lint, or None."""
130
+ from .lineage import _bare_procedure_member
131
+ from .lint.runner import procedure_body_job
132
+ from .model import iter_execution_steps
133
+
134
+ proc = _bare_procedure_member(path)
135
+ if proc is None:
136
+ return None
137
+ job = procedure_body_job(proc)
138
+ # The member was read behind a one-line JOB wrapper; give the findings
139
+ # the member's own line numbers.
140
+ for step in iter_execution_steps(job.steps):
141
+ step.line_number = max(step.line_number - 1, 0)
142
+ for dd in step.dd_statements:
143
+ dd.line_number = max(dd.line_number - 1, 0)
144
+ return job
145
+
146
+
147
+ def _lint_member(path: Path, config: LintConfig) -> FileReport:
148
+ """Lint a PROC member found in a directory: its PROC bodies' steps."""
149
+ try:
150
+ job = parse_jcl(path, enable_external_procs=False)
151
+ except _UNREADABLE:
152
+ bare = _bare_member_job(path)
153
+ if bare is None:
154
+ raise
155
+ job = bare
122
156
  return _lint_one(path, job, config)
123
157
 
124
158
 
@@ -129,8 +163,8 @@ def _lint_directory(
129
163
  explode: bool,
130
164
  proc_base_path: str | None,
131
165
  ) -> list[FileReport]:
132
- # load_job_stream explodes with the directory's own PROC members; the
133
- # members themselves are not jobs and are not linted.
166
+ # load_job_stream explodes with the directory's own PROC members; each
167
+ # member is linted on its own too, for the steps of its PROC body.
134
168
  stream = load_job_stream(
135
169
  directory, pattern, explode=explode, proc_base_path=proc_base_path
136
170
  )
@@ -138,6 +172,9 @@ def _lint_directory(
138
172
  _lint_one(directory / job_id, job, config)
139
173
  for job_id, job in stream.jobs.items()
140
174
  ]
175
+ reports += [
176
+ _lint_member(directory / member_id, config) for member_id in stream.procedures
177
+ ]
141
178
  for job_id, message in stream.errors.items():
142
179
  failure = _parse_failure(ValueError(message))
143
180
  reports.append(FileReport(_uri(directory / job_id), [failure]))
@@ -21,6 +21,7 @@ from .errors import (
21
21
  ProcedureNotFoundError,
22
22
  UndefinedSymbolError,
23
23
  )
24
+ from .include_loader import IncludeMemberLoader
24
25
  from .model import (
25
26
  ConditionalBlock,
26
27
  DDStatement,
@@ -1514,6 +1515,7 @@ def explode_job(
1514
1515
  audit: bool = True,
1515
1516
  system_provider: SystemSymbolProvider | None = None,
1516
1517
  strict_symbols: bool = False,
1518
+ include_loader: IncludeMemberLoader | None = None,
1517
1519
  ) -> Job:
1518
1520
  """
1519
1521
  Explode all procedure invocations in a job.
@@ -1532,6 +1534,11 @@ def explode_job(
1532
1534
  audit: Whether to generate audit logs
1533
1535
  system_provider: System symbol provider (defaults to mock provider)
1534
1536
  strict_symbols: If True, raise errors for undefined symbols
1537
+ include_loader: Loader for the ``// INCLUDE MEMBER=`` statements of
1538
+ a cataloged (external) PROC, searched in the job's JCLLIB
1539
+ order; defaults to the loader :func:`parse_jcl` read the
1540
+ job's own INCLUDEs with. A member it cannot find is a
1541
+ job warning.
1535
1542
 
1536
1543
  Returns:
1537
1544
  New Job instance with exploded procedures
@@ -1547,8 +1554,11 @@ def explode_job(
1547
1554
  # the job's JCLLIB / proc_base_path (so an external PROC called from inside
1548
1555
  # another PROC or an IF block is found where the top-level ones were),
1549
1556
  # else the default chain. Inline procedures always win.
1550
- if proc_loader is None:
1551
- proc_loader = _job_proc_loader(job)
1557
+ include_expander = _IncludeExpandingLoader(
1558
+ proc_loader if proc_loader is not None else _job_proc_loader(job),
1559
+ include_loader or _job_include_loader(job),
1560
+ )
1561
+ proc_loader = include_expander
1552
1562
 
1553
1563
  # Get system provider
1554
1564
  system_provider = system_provider or get_default_provider()
@@ -1640,6 +1650,7 @@ def explode_job(
1640
1650
  # copy in _exploded_job) -- e.g. an undefined &SYMBOL only resolvable
1641
1651
  # inside a PROC body.
1642
1652
  exploded_job.warnings.extend(explosion_warnings)
1653
+ include_expander.record_on(exploded_job)
1643
1654
 
1644
1655
  # Attach audit metadata
1645
1656
  if audit:
@@ -1670,6 +1681,153 @@ def _job_step_scope(job: Job, step: Step, base_scope: SymbolScope) -> SymbolScop
1670
1681
  )
1671
1682
 
1672
1683
 
1684
+ def _job_include_loader(job: Job) -> IncludeMemberLoader:
1685
+ """The INCLUDE loader :func:`parse_jcl` attached to *job*, else the
1686
+ default filesystem one."""
1687
+ from .include_loader import JCLLIBMemberLoader
1688
+
1689
+ loader: IncludeMemberLoader | None = getattr(job, "_include_loader", None)
1690
+ return loader if loader is not None else JCLLIBMemberLoader()
1691
+
1692
+
1693
+ class _IncludeExpandingLoader(ProcedureLoader):
1694
+ """*inner*, with the INCLUDEs of each external PROC it returns expanded.
1695
+
1696
+ ``parse_jcl`` expands INCLUDEs in the job text, in-stream PROCs
1697
+ included, before parsing; a cataloged PROC is read later, so its
1698
+ INCLUDEs are expanded here, against the job's JCLLIB order (issue #153).
1699
+ :meth:`record_on` adds the members expanded to the exploded job's
1700
+ ``includes`` and a warning for each member not found.
1701
+ """
1702
+
1703
+ def __init__(
1704
+ self, inner: ProcedureLoader, include_loader: IncludeMemberLoader
1705
+ ) -> None:
1706
+ self._inner = inner
1707
+ self._include_loader = include_loader
1708
+ self._included: list[str] = []
1709
+ self._warnings: list[str] = []
1710
+ self._cache: dict[tuple[str, tuple[str, ...] | None], Procedure | None] = {}
1711
+ self._call_cache: dict[
1712
+ tuple[str, tuple[str, ...] | None, tuple[tuple[str, str], ...]], Procedure
1713
+ ] = {}
1714
+
1715
+ def load_procedure(
1716
+ self, name: str, libraries: Sequence[str] | None = None
1717
+ ) -> Procedure | None:
1718
+ key = (name.upper(), tuple(libraries) if libraries else None)
1719
+ if key not in self._cache:
1720
+ proc = self._inner.load_procedure(name, libraries)
1721
+ self._cache[key] = (
1722
+ self._expanded(proc, libraries) if proc is not None else None
1723
+ )
1724
+ return self._cache[key]
1725
+
1726
+ def for_call(
1727
+ self,
1728
+ proc: Procedure,
1729
+ libraries: Sequence[str] | None,
1730
+ scope: SymbolScope,
1731
+ ) -> Procedure:
1732
+ """*proc* as one call of it sees it: INCLUDE member names written as
1733
+ symbols (``MEMBER=&MEM``) resolved in that call's *scope* -- EXEC
1734
+ values, PROC defaults, SETs (issue #160).
1735
+
1736
+ *proc* is returned as is when no INCLUDE of its text names a symbol.
1737
+ """
1738
+ text = proc.source_text
1739
+ if not text or not _has_symbolic_include(text):
1740
+ return proc
1741
+ symbols = _flattened_caller_scope(scope).set_symbols
1742
+ key = (
1743
+ (proc.name or "").upper(),
1744
+ tuple(libraries) if libraries else None,
1745
+ tuple(sorted(symbols.items())),
1746
+ )
1747
+ if key not in self._call_cache:
1748
+ self._call_cache[key] = self._expanded(proc, libraries, symbols)
1749
+ return self._call_cache[key]
1750
+
1751
+ def record_on(self, job: Job) -> None:
1752
+ job.warnings.extend(self._warnings)
1753
+ job.includes.extend(m for m in self._included if m not in job.includes)
1754
+
1755
+ def _expanded(
1756
+ self,
1757
+ proc: Procedure,
1758
+ libraries: Sequence[str] | None,
1759
+ symbols: dict[str, str] | None = None,
1760
+ ) -> Procedure:
1761
+ """*proc* with the INCLUDEs of its text expanded.
1762
+
1763
+ Without *symbols* (the expansion at load time), an INCLUDE whose
1764
+ member is a symbol is left for :meth:`for_call`, without a warning.
1765
+ """
1766
+ from ._statement_parser import _has_include_statement
1767
+ from .include_loader import resolve_includes
1768
+ from .proc_loader import _build_procedure_from_content
1769
+
1770
+ text = proc.source_text
1771
+ if not proc.is_external or not text or not _has_include_statement(text):
1772
+ return proc
1773
+ members: list[str] = []
1774
+ expanded_text = resolve_includes(
1775
+ text,
1776
+ self._include_loader,
1777
+ list(libraries) if libraries else None,
1778
+ _collected=members,
1779
+ _symbols=dict(symbols) if symbols is not None else None,
1780
+ )
1781
+ self._warn_missing(proc, expanded_text, symbols)
1782
+ self._included.extend(m for m in members if m not in self._included)
1783
+ if not members:
1784
+ return proc
1785
+ rebuilt = _build_procedure_from_content(expanded_text, proc.name)
1786
+ if rebuilt is None:
1787
+ return proc
1788
+ rebuilt.source_text = text
1789
+ return rebuilt
1790
+
1791
+ def _warn_missing(
1792
+ self, proc: Procedure, expanded_text: str, symbols: dict[str, str] | None
1793
+ ) -> None:
1794
+ """Warn once for each INCLUDE left in *expanded_text* (not found)."""
1795
+ from .include_loader import (
1796
+ _extract_include_member,
1797
+ _record_set_symbols,
1798
+ _substitute_member_symbols,
1799
+ )
1800
+ from .utils import split_source_lines
1801
+
1802
+ seen = dict(symbols) if symbols is not None else {}
1803
+ for line in split_source_lines(expanded_text):
1804
+ _record_set_symbols(line.rstrip(), seen)
1805
+ missing = _extract_include_member(line.rstrip())
1806
+ if not missing:
1807
+ continue
1808
+ if "&" in missing:
1809
+ if symbols is None:
1810
+ continue
1811
+ missing = _substitute_member_symbols(missing, seen)
1812
+ warning = (
1813
+ f"INCLUDE member {missing} in PROC {proc.name} not found; "
1814
+ "its statements are missing from the exploded job"
1815
+ )
1816
+ if warning not in self._warnings:
1817
+ self._warnings.append(warning)
1818
+
1819
+
1820
+ def _has_symbolic_include(text: str) -> bool:
1821
+ """Whether an ``// INCLUDE`` statement of *text* names its member by symbol."""
1822
+ from .include_loader import _extract_include_member
1823
+ from .utils import split_source_lines
1824
+
1825
+ return any(
1826
+ "&" in (_extract_include_member(line.rstrip()) or "")
1827
+ for line in split_source_lines(text)
1828
+ )
1829
+
1830
+
1673
1831
  def _job_proc_loader(job: Job) -> ProcedureLoader:
1674
1832
  """The loader :func:`explode_job` uses when the caller passes none."""
1675
1833
  job_loader: ProcedureLoader | None = getattr(job, "_proc_loader", None)
@@ -1828,6 +1986,41 @@ def _record_expansion(
1828
1986
  )
1829
1987
 
1830
1988
 
1989
+ def _load_called_procedure(
1990
+ step: Step,
1991
+ proc_loader: ProcedureLoader,
1992
+ libraries: tuple[str, ...] | None,
1993
+ base_scope: SymbolScope,
1994
+ call_scope: SymbolScope | None,
1995
+ strict_symbols: bool,
1996
+ warnings: list[str] | None,
1997
+ ) -> Procedure:
1998
+ """The procedure PROC call *step* names, as that call sees it.
1999
+
2000
+ The call's still-unresolved overrides are resolved in the procedure's
2001
+ scope when *call_scope* is given (see :func:`_explode_step`), then the
2002
+ INCLUDEs naming their member by symbol are expanded in the call's scope.
2003
+
2004
+ Raises:
2005
+ ProcedureNotFoundError: If *proc_loader* cannot find the procedure.
2006
+ """
2007
+ proc_name = step.procedure or ""
2008
+ procedure = proc_loader.load_procedure(proc_name, libraries)
2009
+ if procedure is None:
2010
+ raise ProcedureNotFoundError(proc_name, step.line_number)
2011
+ if call_scope is not None:
2012
+ _resolve_call_in_proc_scope(
2013
+ step, procedure, call_scope, strict_symbols, warnings
2014
+ )
2015
+ if isinstance(proc_loader, _IncludeExpandingLoader):
2016
+ procedure = proc_loader.for_call(
2017
+ procedure,
2018
+ libraries,
2019
+ _expansion_scope(procedure, step.parameters, base_scope),
2020
+ )
2021
+ return procedure
2022
+
2023
+
1831
2024
  def _explode_step(
1832
2025
  step: Step,
1833
2026
  proc_loader: ProcedureLoader,
@@ -1906,15 +2099,9 @@ def _explode_step(
1906
2099
  # against the job's library order. Pass an immutable tuple: loaders key their
1907
2100
  # caches on the library list, so a hashable value keeps that contract safe.
1908
2101
  _libs = tuple(jcllib_order) if jcllib_order else None
1909
- procedure = proc_loader.load_procedure(proc_name, _libs)
1910
-
1911
- if procedure is None:
1912
- raise ProcedureNotFoundError(proc_name, step.line_number)
1913
-
1914
- if call_scope is not None:
1915
- _resolve_call_in_proc_scope(
1916
- step, procedure, call_scope, strict_symbols, warnings
1917
- )
2102
+ procedure = _load_called_procedure(
2103
+ step, proc_loader, _libs, base_scope, call_scope, strict_symbols, warnings
2104
+ )
1918
2105
 
1919
2106
  # Extract parameter overrides from step
1920
2107
  overrides = step.parameters.copy()
@@ -324,8 +324,12 @@ class _IncludeExpander:
324
324
  # stay as-is. They are located with the same scan the parser uses.
325
325
  from ._source_scanner import _merge_source_lines
326
326
 
327
- _, inline_data, control_data, _ = _merge_source_lines(content)
328
- data_indices = inline_data | control_data
327
+ scanned = _merge_source_lines(content)
328
+ data_indices = (
329
+ scanned.inline_data_indices
330
+ | scanned.control_data_indices
331
+ | scanned.jes3_data_indices
332
+ )
329
333
 
330
334
  result: list[_Entry] = []
331
335
  for index, line in enumerate(lines):
@@ -48,6 +48,9 @@ class Jes3ControlStatement(JESControlStatement):
48
48
  """
49
49
 
50
50
  parameters: dict[str, str] = field(default_factory=dict)
51
+ #: For ``//*DATASET``: the records of the input data set, as coded, up
52
+ #: to ``//*ENDDATASET`` (or the ``END=`` terminator, which is not kept).
53
+ data: list[str] = field(default_factory=list)
51
54
 
52
55
 
53
56
  @dataclass
@@ -444,6 +444,29 @@ def _member_procedures(path: Path, member: Job) -> dict[str, Procedure]:
444
444
  return procs
445
445
 
446
446
 
447
+ def _bare_procedure_member(path: Path) -> Procedure | None:
448
+ """*path* read as a cataloged PROC member with no ``PROC`` statement.
449
+
450
+ On z/OS the PROC statement of a cataloged procedure is optional (it only
451
+ gives symbolic parameters defaults), so a member may start at its first
452
+ ``EXEC``. :func:`parse_jcl` rejects such a JOB-less file ("EXEC
453
+ statement before JOB statement"); the file-based PROC loader reads it as
454
+ the body of a procedure named after the member, and so does this
455
+ (issue #147). None when *path* is not such a member.
456
+ """
457
+ from .jcl_file_type import FileType, detect_file_type_from_content
458
+ from .proc_loader import _build_procedure_from_content
459
+ from .utils import _read_file_with_fallback
460
+
461
+ try:
462
+ content = _read_file_with_fallback(path)
463
+ if detect_file_type_from_content(content) != FileType.PROC:
464
+ return None
465
+ return _build_procedure_from_content(content, path.stem.upper())
466
+ except (JCLParseError, OSError, UnicodeError, ValueError):
467
+ return None
468
+
469
+
447
470
  def load_job_stream(
448
471
  directory: str | Path,
449
472
  pattern: str = "*",
@@ -456,8 +479,9 @@ def load_job_stream(
456
479
  *pattern* is a :meth:`pathlib.Path.glob` pattern (``"**/*.jcl"`` recurses).
457
480
  Hidden files are skipped. A file that fails to parse is recorded in
458
481
  :attr:`JobStream.errors` rather than raised; a JOB-less PROC member is
459
- listed in :attr:`JobStream.procedures`. With *explode* (the default)
460
- each job's PROC calls are expanded: a PROC is looked up in the job
482
+ listed in :attr:`JobStream.procedures`, and so is a member with no
483
+ ``PROC`` statement that starts at its first ``EXEC`` (issue #147). With
484
+ *explode* (the default) each job's PROC calls are expanded: a PROC is looked up in the job
461
485
  itself, then among the PROC members of the directory (by member name,
462
486
  see :func:`_member_procedures`; issue #144), then in *proc_base_path* as
463
487
  :func:`parse_jcl` does, then among the built-in PROCs. A job whose
@@ -482,7 +506,12 @@ def load_job_stream(
482
506
  enable_external_procs=proc_base_path is not None,
483
507
  )
484
508
  except (JCLParseError, UnicodeError, ValueError) as exc:
485
- stream.errors[job_id] = _first_line(str(exc))
509
+ bare = _bare_procedure_member(path)
510
+ if bare is None:
511
+ stream.errors[job_id] = _first_line(str(exc))
512
+ else:
513
+ stream.procedures.append(job_id)
514
+ members.setdefault(path.stem.upper(), bare)
486
515
  continue
487
516
  if job.is_procedure_member:
488
517
  stream.procedures.append(job_id)