jcl-parser 2.2.2__tar.gz → 2.3.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {jcl_parser-2.2.2 → jcl_parser-2.3.0}/PKG-INFO +1 -1
- {jcl_parser-2.2.2 → jcl_parser-2.3.0}/jcl_parser.egg-info/PKG-INFO +1 -1
- {jcl_parser-2.2.2 → jcl_parser-2.3.0}/jclparser/__init__.py +1 -1
- {jcl_parser-2.2.2 → jcl_parser-2.3.0}/jclparser/cli.py +125 -7
- {jcl_parser-2.2.2 → jcl_parser-2.3.0}/jclparser/explosion.py +106 -0
- {jcl_parser-2.2.2 → jcl_parser-2.3.0}/jclparser/lint/rules.py +15 -10
- {jcl_parser-2.2.2 → jcl_parser-2.3.0}/jclparser/model.py +23 -0
- {jcl_parser-2.2.2 → jcl_parser-2.3.0}/jclparser/program_profiles/dfsrrc00.py +48 -0
- {jcl_parser-2.2.2 → jcl_parser-2.3.0}/jclparser/validators.py +34 -7
- {jcl_parser-2.2.2 → jcl_parser-2.3.0}/pyproject.toml +1 -1
- {jcl_parser-2.2.2 → jcl_parser-2.3.0}/LICENSE +0 -0
- {jcl_parser-2.2.2 → jcl_parser-2.3.0}/README.md +0 -0
- {jcl_parser-2.2.2 → jcl_parser-2.3.0}/jcl_parser.egg-info/SOURCES.txt +0 -0
- {jcl_parser-2.2.2 → jcl_parser-2.3.0}/jcl_parser.egg-info/dependency_links.txt +0 -0
- {jcl_parser-2.2.2 → jcl_parser-2.3.0}/jcl_parser.egg-info/entry_points.txt +0 -0
- {jcl_parser-2.2.2 → jcl_parser-2.3.0}/jcl_parser.egg-info/requires.txt +0 -0
- {jcl_parser-2.2.2 → jcl_parser-2.3.0}/jcl_parser.egg-info/top_level.txt +0 -0
- {jcl_parser-2.2.2 → jcl_parser-2.3.0}/jclparser/__main__.py +0 -0
- {jcl_parser-2.2.2 → jcl_parser-2.3.0}/jclparser/_include_origins.py +0 -0
- {jcl_parser-2.2.2 → jcl_parser-2.3.0}/jclparser/_job_builder.py +0 -0
- {jcl_parser-2.2.2 → jcl_parser-2.3.0}/jclparser/_model_factories.py +0 -0
- {jcl_parser-2.2.2 → jcl_parser-2.3.0}/jclparser/_post_parse.py +0 -0
- {jcl_parser-2.2.2 → jcl_parser-2.3.0}/jclparser/_source_scanner.py +0 -0
- {jcl_parser-2.2.2 → jcl_parser-2.3.0}/jclparser/_statement_parser.py +0 -0
- {jcl_parser-2.2.2 → jcl_parser-2.3.0}/jclparser/builtin_procs.py +0 -0
- {jcl_parser-2.2.2 → jcl_parser-2.3.0}/jclparser/checks.py +0 -0
- {jcl_parser-2.2.2 → jcl_parser-2.3.0}/jclparser/condition_parser.py +0 -0
- {jcl_parser-2.2.2 → jcl_parser-2.3.0}/jclparser/continuation_handler.py +0 -0
- {jcl_parser-2.2.2 → jcl_parser-2.3.0}/jclparser/dataset_lifecycle.py +0 -0
- {jcl_parser-2.2.2 → jcl_parser-2.3.0}/jclparser/errors.py +0 -0
- {jcl_parser-2.2.2 → jcl_parser-2.3.0}/jclparser/include_loader.py +0 -0
- {jcl_parser-2.2.2 → jcl_parser-2.3.0}/jclparser/jcl_file_type.py +0 -0
- {jcl_parser-2.2.2 → jcl_parser-2.3.0}/jclparser/jes_statements.py +0 -0
- {jcl_parser-2.2.2 → jcl_parser-2.3.0}/jclparser/lineage.py +0 -0
- {jcl_parser-2.2.2 → jcl_parser-2.3.0}/jclparser/lint/__init__.py +0 -0
- {jcl_parser-2.2.2 → jcl_parser-2.3.0}/jclparser/lint/base.py +0 -0
- {jcl_parser-2.2.2 → jcl_parser-2.3.0}/jclparser/lint/config.py +0 -0
- {jcl_parser-2.2.2 → jcl_parser-2.3.0}/jclparser/lint/runner.py +0 -0
- {jcl_parser-2.2.2 → jcl_parser-2.3.0}/jclparser/lint/suppress.py +0 -0
- {jcl_parser-2.2.2 → jcl_parser-2.3.0}/jclparser/metrics.py +0 -0
- {jcl_parser-2.2.2 → jcl_parser-2.3.0}/jclparser/parameter_ast.py +0 -0
- {jcl_parser-2.2.2 → jcl_parser-2.3.0}/jclparser/parameter_validators.py +0 -0
- {jcl_parser-2.2.2 → jcl_parser-2.3.0}/jclparser/parser.py +0 -0
- {jcl_parser-2.2.2 → jcl_parser-2.3.0}/jclparser/plugins/__init__.py +0 -0
- {jcl_parser-2.2.2 → jcl_parser-2.3.0}/jclparser/plugins/base.py +0 -0
- {jcl_parser-2.2.2 → jcl_parser-2.3.0}/jclparser/proc_loader.py +0 -0
- {jcl_parser-2.2.2 → jcl_parser-2.3.0}/jclparser/profiling.py +0 -0
- {jcl_parser-2.2.2 → jcl_parser-2.3.0}/jclparser/program_profiles/__init__.py +0 -0
- {jcl_parser-2.2.2 → jcl_parser-2.3.0}/jclparser/program_profiles/_control.py +0 -0
- {jcl_parser-2.2.2 → jcl_parser-2.3.0}/jclparser/program_profiles/_invoked.py +0 -0
- {jcl_parser-2.2.2 → jcl_parser-2.3.0}/jclparser/program_profiles/adrdssu.py +0 -0
- {jcl_parser-2.2.2 → jcl_parser-2.3.0}/jclparser/program_profiles/base.py +0 -0
- {jcl_parser-2.2.2 → jcl_parser-2.3.0}/jclparser/program_profiles/binder.py +0 -0
- {jcl_parser-2.2.2 → jcl_parser-2.3.0}/jclparser/program_profiles/bpxbatch.py +0 -0
- {jcl_parser-2.2.2 → jcl_parser-2.3.0}/jclparser/program_profiles/compilers.py +0 -0
- {jcl_parser-2.2.2 → jcl_parser-2.3.0}/jclparser/program_profiles/db2_batch.py +0 -0
- {jcl_parser-2.2.2 → jcl_parser-2.3.0}/jclparser/program_profiles/dsnutilb.py +0 -0
- {jcl_parser-2.2.2 → jcl_parser-2.3.0}/jclparser/program_profiles/ftp.py +0 -0
- {jcl_parser-2.2.2 → jcl_parser-2.3.0}/jclparser/program_profiles/icetool.py +0 -0
- {jcl_parser-2.2.2 → jcl_parser-2.3.0}/jclparser/program_profiles/idcams.py +0 -0
- {jcl_parser-2.2.2 → jcl_parser-2.3.0}/jclparser/program_profiles/iebcompr.py +0 -0
- {jcl_parser-2.2.2 → jcl_parser-2.3.0}/jclparser/program_profiles/iebdg.py +0 -0
- {jcl_parser-2.2.2 → jcl_parser-2.3.0}/jclparser/program_profiles/iebgener.py +0 -0
- {jcl_parser-2.2.2 → jcl_parser-2.3.0}/jclparser/program_profiles/iebptpch.py +0 -0
- {jcl_parser-2.2.2 → jcl_parser-2.3.0}/jclparser/program_profiles/iebupdte.py +0 -0
- {jcl_parser-2.2.2 → jcl_parser-2.3.0}/jclparser/program_profiles/iefbr14.py +0 -0
- {jcl_parser-2.2.2 → jcl_parser-2.3.0}/jclparser/program_profiles/iehinitt.py +0 -0
- {jcl_parser-2.2.2 → jcl_parser-2.3.0}/jclparser/program_profiles/iehlist.py +0 -0
- {jcl_parser-2.2.2 → jcl_parser-2.3.0}/jclparser/program_profiles/iehprogm.py +0 -0
- {jcl_parser-2.2.2 → jcl_parser-2.3.0}/jclparser/program_profiles/ikjeft01.py +0 -0
- {jcl_parser-2.2.2 → jcl_parser-2.3.0}/jclparser/program_profiles/irxjcl.py +0 -0
- {jcl_parser-2.2.2 → jcl_parser-2.3.0}/jclparser/program_profiles/registry.py +0 -0
- {jcl_parser-2.2.2 → jcl_parser-2.3.0}/jclparser/program_profiles/sort.py +0 -0
- {jcl_parser-2.2.2 → jcl_parser-2.3.0}/jclparser/py.typed +0 -0
- {jcl_parser-2.2.2 → jcl_parser-2.3.0}/jclparser/rc_analysis.py +0 -0
- {jcl_parser-2.2.2 → jcl_parser-2.3.0}/jclparser/serialization.py +0 -0
- {jcl_parser-2.2.2 → jcl_parser-2.3.0}/jclparser/sql_detector.py +0 -0
- {jcl_parser-2.2.2 → jcl_parser-2.3.0}/jclparser/system_symbols.py +0 -0
- {jcl_parser-2.2.2 → jcl_parser-2.3.0}/jclparser/utils.py +0 -0
- {jcl_parser-2.2.2 → jcl_parser-2.3.0}/setup.cfg +0 -0
|
@@ -20,7 +20,7 @@ application, for example::
|
|
|
20
20
|
import logging as _logging
|
|
21
21
|
|
|
22
22
|
# Package metadata
|
|
23
|
-
__version__ = "2.
|
|
23
|
+
__version__ = "2.3.0" # x-release-please-version
|
|
24
24
|
__author__ = "JCL Parser Team"
|
|
25
25
|
__description__ = "Parse IBM JCL scripts into Python objects"
|
|
26
26
|
|
|
@@ -119,9 +119,15 @@ def _main(argv: list[str] | None = None) -> int:
|
|
|
119
119
|
# ("FILE --explode steps"); plain parse_args() hands the whole positional
|
|
120
120
|
# run to FILE/query before the flag and then rejects "steps".
|
|
121
121
|
arg_parser = _build_arg_parser()
|
|
122
|
+
if not _argv:
|
|
123
|
+
# A bare `jcl-parser` used to print only "the following arguments
|
|
124
|
+
# are required: FILE"; show the queries and examples instead.
|
|
125
|
+
arg_parser.print_help()
|
|
126
|
+
return 2
|
|
122
127
|
args = arg_parser.parse_intermixed_args(argv)
|
|
123
128
|
if args.profile_output and not args.profile:
|
|
124
129
|
arg_parser.error("--profile-output requires --profile") # exits 2
|
|
130
|
+
_apply_format(args)
|
|
125
131
|
|
|
126
132
|
source: Path | str
|
|
127
133
|
if args.file == "-":
|
|
@@ -160,6 +166,14 @@ def _main(argv: list[str] | None = None) -> int:
|
|
|
160
166
|
return _run_query_with_profile(job, args, prof, source)
|
|
161
167
|
|
|
162
168
|
|
|
169
|
+
def _apply_format(args: argparse.Namespace) -> None:
|
|
170
|
+
"""``--format json|sarif`` sets the older ``--json`` / ``--sarif`` flags."""
|
|
171
|
+
if args.format == "json":
|
|
172
|
+
args.json_output = True
|
|
173
|
+
elif args.format == "sarif":
|
|
174
|
+
args.sarif = True
|
|
175
|
+
|
|
176
|
+
|
|
163
177
|
def _read_stdin() -> str:
|
|
164
178
|
"""The JCL piped on stdin, decoded like a file (``FILE`` is ``-``)."""
|
|
165
179
|
return read_source(getattr(sys.stdin, "buffer", sys.stdin))
|
|
@@ -196,10 +210,70 @@ def _is_step_or_dd_name(job: Job, token: str) -> bool:
|
|
|
196
210
|
)
|
|
197
211
|
|
|
198
212
|
|
|
213
|
+
#: The query words ``jcl-parser FILE <word>`` understands, for typo hints.
|
|
214
|
+
_QUERY_WORDS = (
|
|
215
|
+
"steps",
|
|
216
|
+
"procs",
|
|
217
|
+
"detect",
|
|
218
|
+
"metrics",
|
|
219
|
+
"lint",
|
|
220
|
+
"datasets",
|
|
221
|
+
"sql",
|
|
222
|
+
"analyze",
|
|
223
|
+
"paths",
|
|
224
|
+
"lineage",
|
|
225
|
+
)
|
|
226
|
+
|
|
227
|
+
_HELP_EPILOG = """\
|
|
228
|
+
queries (the words after FILE):
|
|
229
|
+
(none) job summary; with --json the whole parse as JSON
|
|
230
|
+
steps every step, the program or PROC it runs, its DDs
|
|
231
|
+
procs [PROCNAME] inline/external PROCs; --explode shows the expansion
|
|
232
|
+
STEP one step in detail e.g. STEP010
|
|
233
|
+
STEP.DDNAME one DD statement e.g. STEP010.SYSIN
|
|
234
|
+
DDNAME that DD in every step e.g. SYSOUT
|
|
235
|
+
detect is the file a JOB, a PROC, or neither
|
|
236
|
+
metrics step, DD, PROC and dataset counts, complexity
|
|
237
|
+
lint lint rules (--lint-list lists them; --sarif for CI)
|
|
238
|
+
datasets dataset lifecycle: created, read, deleted, passed
|
|
239
|
+
sql DB2 / SQL usage by step
|
|
240
|
+
analyze [STEP] what each utility step (SORT, IDCAMS, ...) does
|
|
241
|
+
paths which steps can run, from the COND / IF logic
|
|
242
|
+
|
|
243
|
+
FILE may be a directory: 'lineage [dot|mermaid]' (the default) maps
|
|
244
|
+
datasets across its jobs, 'lint' and --validate check every file
|
|
245
|
+
(--glob picks them). FILE '-' reads the JCL from stdin.
|
|
246
|
+
|
|
247
|
+
examples:
|
|
248
|
+
jcl-parser PAYROLL.jcl
|
|
249
|
+
jcl-parser PAYROLL.jcl steps --explode --proc-path ./proclib
|
|
250
|
+
jcl-parser PAYROLL.jcl STEP010.SYSIN
|
|
251
|
+
jcl-parser PAYROLL.jcl lint --fail-on warning
|
|
252
|
+
jcl-parser PAYROLL.jcl --validate --format json
|
|
253
|
+
jcl-parser jobs/ lineage mermaid > lineage.mmd
|
|
254
|
+
jcl-lint jobs/*.jcl --format sarif -o jcl.sarif (several files, for CI)
|
|
255
|
+
|
|
256
|
+
exit codes:
|
|
257
|
+
0 success
|
|
258
|
+
1 the file could not be read or parsed, a name was not found, or a check
|
|
259
|
+
reported something (--validate errors, lint findings at --fail-on or
|
|
260
|
+
above, dataset / SQL findings)
|
|
261
|
+
2 the command line was rejected
|
|
262
|
+
|
|
263
|
+
defaults:
|
|
264
|
+
lint reads [tool.jcl-parser.lint] from the nearest pyproject.toml (or
|
|
265
|
+
--config). External PROCs are loaded only with --proc-path, except by
|
|
266
|
+
--validate, which falls back to the JCL_PROCLIB_PATH variable.
|
|
267
|
+
"""
|
|
268
|
+
|
|
269
|
+
|
|
199
270
|
def _build_arg_parser() -> argparse.ArgumentParser:
|
|
200
271
|
parser = argparse.ArgumentParser(
|
|
201
272
|
prog="jcl-parser",
|
|
202
|
-
|
|
273
|
+
usage="jcl-parser FILE [QUERY ...] [options]",
|
|
274
|
+
description="Parse and inspect IBM z/OS JCL: jobs, steps, DDs and PROCs.",
|
|
275
|
+
epilog=_HELP_EPILOG,
|
|
276
|
+
formatter_class=argparse.RawDescriptionHelpFormatter,
|
|
203
277
|
)
|
|
204
278
|
parser.add_argument(
|
|
205
279
|
"file",
|
|
@@ -210,15 +284,22 @@ def _build_arg_parser() -> argparse.ArgumentParser:
|
|
|
210
284
|
"query",
|
|
211
285
|
nargs="*",
|
|
212
286
|
default=[],
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
"or a DD name to find across all steps; for a directory, "
|
|
216
|
-
"'lineage [dot|mermaid]'"
|
|
217
|
-
),
|
|
287
|
+
metavar="QUERY",
|
|
288
|
+
help="What to show (see 'queries' below); none prints the job summary",
|
|
218
289
|
)
|
|
219
290
|
parser.add_argument(
|
|
220
291
|
"--json", action="store_true", dest="json_output", help="Output as JSON"
|
|
221
292
|
)
|
|
293
|
+
parser.add_argument(
|
|
294
|
+
"--format",
|
|
295
|
+
choices=("text", "json", "sarif"),
|
|
296
|
+
default=None,
|
|
297
|
+
help=(
|
|
298
|
+
"Output format: text (default), json (same as --json) or sarif "
|
|
299
|
+
"(same as --sarif; lint and --validate only). The spelling "
|
|
300
|
+
"cobol-parser and jcl-lint use"
|
|
301
|
+
),
|
|
302
|
+
)
|
|
222
303
|
parser.add_argument(
|
|
223
304
|
"--validate", action="store_true", help="Validate the JCL and report errors"
|
|
224
305
|
)
|
|
@@ -469,7 +550,44 @@ def _cmd_step_or_dd(job: Job, query: str, json_out: bool) -> int:
|
|
|
469
550
|
return 0
|
|
470
551
|
|
|
471
552
|
# Fall back to DD name across all steps
|
|
472
|
-
|
|
553
|
+
status = _print_dd_across_steps(job, query.upper(), json_out)
|
|
554
|
+
if status != 0:
|
|
555
|
+
hint = _not_found_hint(job, query)
|
|
556
|
+
if hint:
|
|
557
|
+
print(hint, file=sys.stderr)
|
|
558
|
+
return status
|
|
559
|
+
|
|
560
|
+
|
|
561
|
+
def _not_found_hint(job: Job, query: str) -> str | None:
|
|
562
|
+
"""A "did you mean" line for a query that named no step or DD.
|
|
563
|
+
|
|
564
|
+
A mistyped query word (``stpes``) used to read only as "No step or DD
|
|
565
|
+
named 'STPES' found", which sends a new user looking for a step.
|
|
566
|
+
"""
|
|
567
|
+
import difflib
|
|
568
|
+
|
|
569
|
+
words = difflib.get_close_matches(query.lower(), _QUERY_WORDS, n=1, cutoff=0.6)
|
|
570
|
+
names = sorted(
|
|
571
|
+
{s.name for s in iter_execution_steps(job.steps) if s.name}
|
|
572
|
+
| {
|
|
573
|
+
dd.name
|
|
574
|
+
for s in iter_execution_steps(job.steps)
|
|
575
|
+
for dd in s.dd_statements
|
|
576
|
+
if dd.name
|
|
577
|
+
}
|
|
578
|
+
)
|
|
579
|
+
close = difflib.get_close_matches(query.upper(), names, n=3, cutoff=0.6)
|
|
580
|
+
# JCL names are uppercase: an uppercase query is a misspelt name first,
|
|
581
|
+
# a lowercase one a misspelt query word first.
|
|
582
|
+
name_hint = "Did you mean: " + ", ".join(close) + "?" if close else None
|
|
583
|
+
word_hint = (
|
|
584
|
+
f"Did you mean the query '{words[0]}'? (see 'jcl-parser --help')"
|
|
585
|
+
if words
|
|
586
|
+
else None
|
|
587
|
+
)
|
|
588
|
+
if query == query.upper():
|
|
589
|
+
return name_hint or word_hint
|
|
590
|
+
return word_hint or name_hint
|
|
473
591
|
|
|
474
592
|
|
|
475
593
|
def _cmd_directory(path: Path, args: argparse.Namespace) -> int:
|
|
@@ -1219,6 +1219,104 @@ def is_exec_override_keyword(key: str) -> bool:
|
|
|
1219
1219
|
return key.upper().partition(".")[0] in _EXEC_OVERRIDE_KEYWORDS
|
|
1220
1220
|
|
|
1221
1221
|
|
|
1222
|
+
_SYMBOL_REFERENCE = re.compile(r"&&|&([A-Z@#$][A-Z0-9@#$]{0,7})", re.IGNORECASE)
|
|
1223
|
+
_INCLUDE_STATEMENT = re.compile(r"^//\S*\s+INCLUDE\s", re.IGNORECASE | re.MULTILINE)
|
|
1224
|
+
|
|
1225
|
+
|
|
1226
|
+
def _referenced_symbols(value: Any, found: set[str], seen: set[int]) -> None:
|
|
1227
|
+
"""Add every ``&NAME`` written anywhere in *value* to *found*.
|
|
1228
|
+
|
|
1229
|
+
Walks strings, dicts, lists and the model dataclasses (steps, DDs,
|
|
1230
|
+
conditional blocks) a procedure body is made of. ``&&`` is an escaped
|
|
1231
|
+
ampersand, not a reference.
|
|
1232
|
+
"""
|
|
1233
|
+
if isinstance(value, str):
|
|
1234
|
+
for match in _SYMBOL_REFERENCE.finditer(value):
|
|
1235
|
+
if match.group(1):
|
|
1236
|
+
found.add(match.group(1).upper())
|
|
1237
|
+
return
|
|
1238
|
+
if id(value) in seen:
|
|
1239
|
+
return
|
|
1240
|
+
seen.add(id(value))
|
|
1241
|
+
if isinstance(value, dict):
|
|
1242
|
+
for item in value.values():
|
|
1243
|
+
_referenced_symbols(item, found, seen)
|
|
1244
|
+
elif isinstance(value, (list, tuple)):
|
|
1245
|
+
for item in value:
|
|
1246
|
+
_referenced_symbols(item, found, seen)
|
|
1247
|
+
elif isinstance(value, (Step, DDStatement, ConditionalBlock)):
|
|
1248
|
+
for name, item in vars(value).items():
|
|
1249
|
+
if name not in ("comments", "inline_data", "original_proc_call"):
|
|
1250
|
+
_referenced_symbols(item, found, seen)
|
|
1251
|
+
|
|
1252
|
+
|
|
1253
|
+
def call_override_symbols(call: Step) -> set[str]:
|
|
1254
|
+
"""Every ``&NAME`` in the overrides coded with PROC call *call*.
|
|
1255
|
+
|
|
1256
|
+
The overriding and added DDs after the call (and their concatenations)
|
|
1257
|
+
and the values of its keyword overrides (``PARM.PS1='&X'``) are merged
|
|
1258
|
+
into the procedure, so the call's symbolics are substituted there too
|
|
1259
|
+
(issue #176).
|
|
1260
|
+
"""
|
|
1261
|
+
found: set[str] = set()
|
|
1262
|
+
_referenced_symbols(
|
|
1263
|
+
[
|
|
1264
|
+
call.dd_statements,
|
|
1265
|
+
list(call.dd_overrides.values()),
|
|
1266
|
+
list(call.dd_override_concatenations.values()),
|
|
1267
|
+
[
|
|
1268
|
+
value
|
|
1269
|
+
for key, value in call.parameters.items()
|
|
1270
|
+
if is_exec_override_keyword(key)
|
|
1271
|
+
],
|
|
1272
|
+
],
|
|
1273
|
+
found,
|
|
1274
|
+
set(),
|
|
1275
|
+
)
|
|
1276
|
+
return found
|
|
1277
|
+
|
|
1278
|
+
|
|
1279
|
+
def unused_proc_symbols(
|
|
1280
|
+
call: Step, procedure: Procedure, call_references: set[str] | None = None
|
|
1281
|
+
) -> list[str]:
|
|
1282
|
+
"""The symbolics PROC call *call* assigns that *procedure* never uses.
|
|
1283
|
+
|
|
1284
|
+
A symbolic coded on ``EXEC procname`` that is neither declared on the
|
|
1285
|
+
PROC statement nor referenced as ``&NAME`` in the procedure body is a
|
|
1286
|
+
JCL error on z/OS (``IEFC657I THE SYMBOL x WAS NOT USED``), and the
|
|
1287
|
+
expansion silently runs with the PROC's defaults (issue #172). EXEC
|
|
1288
|
+
keyword overrides (``PARM``, ``COND.PS1``, ...) are not symbolics. A
|
|
1289
|
+
reference in the call's overriding or added DDs, or in a keyword
|
|
1290
|
+
override's value, counts as a use (issue #176); *call_references* gives
|
|
1291
|
+
those references when the call's overrides were already substituted
|
|
1292
|
+
(default: :func:`call_override_symbols` of *call*).
|
|
1293
|
+
|
|
1294
|
+
Returns an empty list when the body cannot be fully checked: a procedure
|
|
1295
|
+
with no steps (unloaded), or one whose text has an ``INCLUDE`` (its
|
|
1296
|
+
members' references are not known before explosion).
|
|
1297
|
+
"""
|
|
1298
|
+
if not procedure.steps:
|
|
1299
|
+
return []
|
|
1300
|
+
if procedure.source_text and _INCLUDE_STATEMENT.search(procedure.source_text):
|
|
1301
|
+
return []
|
|
1302
|
+
known = {name.upper() for name in procedure.parameters}
|
|
1303
|
+
_referenced_symbols(
|
|
1304
|
+
[procedure.steps, procedure.set_statements, procedure.parameters],
|
|
1305
|
+
known,
|
|
1306
|
+
set(),
|
|
1307
|
+
)
|
|
1308
|
+
if procedure.source_text:
|
|
1309
|
+
_referenced_symbols(procedure.source_text, known, set())
|
|
1310
|
+
known |= call_override_symbols(call) if call_references is None else call_references
|
|
1311
|
+
return [
|
|
1312
|
+
key
|
|
1313
|
+
for key in call.parameters
|
|
1314
|
+
if not is_exec_override_keyword(key)
|
|
1315
|
+
and key.upper() != "PROC"
|
|
1316
|
+
and key.upper() not in known
|
|
1317
|
+
]
|
|
1318
|
+
|
|
1319
|
+
|
|
1222
1320
|
def _substitute_proc_call_parameters(
|
|
1223
1321
|
parameters: dict[str, Any],
|
|
1224
1322
|
scope: SymbolScope,
|
|
@@ -2099,6 +2197,8 @@ def _explode_step(
|
|
|
2099
2197
|
# against the job's library order. Pass an immutable tuple: loaders key their
|
|
2100
2198
|
# caches on the library list, so a hashable value keeps that contract safe.
|
|
2101
2199
|
_libs = tuple(jcllib_order) if jcllib_order else None
|
|
2200
|
+
# Read before loading resolves the call's overrides (issue #176).
|
|
2201
|
+
call_references = call_override_symbols(step)
|
|
2102
2202
|
procedure = _load_called_procedure(
|
|
2103
2203
|
step, proc_loader, _libs, base_scope, call_scope, strict_symbols, warnings
|
|
2104
2204
|
)
|
|
@@ -2121,6 +2221,12 @@ def _explode_step(
|
|
|
2121
2221
|
)
|
|
2122
2222
|
|
|
2123
2223
|
_apply_call_overrides(expanded, step, procedure.name or proc_name, warnings)
|
|
2224
|
+
unused = unused_proc_symbols(step, procedure, call_references)
|
|
2225
|
+
if unused and warnings is not None:
|
|
2226
|
+
warnings.append(
|
|
2227
|
+
f"EXEC {step.name or 'PROC'}: symbol(s) {', '.join(unused)} not used "
|
|
2228
|
+
f"by PROC {procedure.name or proc_name} (IEFC657I); its defaults apply"
|
|
2229
|
+
)
|
|
2124
2230
|
_record_expansion(
|
|
2125
2231
|
step, procedure, overrides, expanded, depth, audit_records, expansion_map
|
|
2126
2232
|
)
|
|
@@ -12,7 +12,7 @@ from collections.abc import Iterable
|
|
|
12
12
|
from typing import TYPE_CHECKING
|
|
13
13
|
|
|
14
14
|
from ..errors import ErrorSeverity, ValidationError
|
|
15
|
-
from ..model import ConditionalBlock, _dataset_dds, iter_execution_steps
|
|
15
|
+
from ..model import ConditionalBlock, _dataset_dds, dd_label, iter_execution_steps
|
|
16
16
|
from .base import LintRule
|
|
17
17
|
|
|
18
18
|
if TYPE_CHECKING:
|
|
@@ -29,7 +29,8 @@ class HardcodedHLQRule(LintRule):
|
|
|
29
29
|
defer = config.should_run(LiteralSetValueHlqRule.code)
|
|
30
30
|
set_values = _qualifier_symbols(job)
|
|
31
31
|
for step in iter_execution_steps(job.steps):
|
|
32
|
-
|
|
32
|
+
dds = _dataset_dds(step)
|
|
33
|
+
for dd in dds:
|
|
33
34
|
dsn = dd.dataset_name
|
|
34
35
|
if not dsn or dsn.startswith("&&"):
|
|
35
36
|
continue
|
|
@@ -49,7 +50,7 @@ class HardcodedHLQRule(LintRule):
|
|
|
49
50
|
if defer and dsn.split(".", 1)[0].upper() in set_values:
|
|
50
51
|
continue
|
|
51
52
|
yield self._err(
|
|
52
|
-
f"DD '{dd
|
|
53
|
+
f"DD '{dd_label(dds, dd)}' in step '{step.name}' uses literal HLQ "
|
|
53
54
|
f"in dataset '{dsn}' — consider &HLQ",
|
|
54
55
|
line_number=dd.line_number,
|
|
55
56
|
config=config,
|
|
@@ -63,7 +64,8 @@ class LiteralSysdaUnitRule(LintRule):
|
|
|
63
64
|
|
|
64
65
|
def check(self, job: Job, config: LintConfig) -> Iterable[ValidationError]:
|
|
65
66
|
for step in iter_execution_steps(job.steps):
|
|
66
|
-
|
|
67
|
+
dds = _dataset_dds(step)
|
|
68
|
+
for dd in dds:
|
|
67
69
|
unit = dd.parameters.get("UNIT")
|
|
68
70
|
if not unit:
|
|
69
71
|
continue
|
|
@@ -72,7 +74,7 @@ class LiteralSysdaUnitRule(LintRule):
|
|
|
72
74
|
unit_val = unit if isinstance(unit, str) else str(unit[0])
|
|
73
75
|
if unit_val.strip("()").split(",")[0].strip().upper() == "SYSDA":
|
|
74
76
|
yield self._err(
|
|
75
|
-
f"DD '{dd
|
|
77
|
+
f"DD '{dd_label(dds, dd)}' in step '{step.name}' uses UNIT=SYSDA",
|
|
76
78
|
line_number=dd.line_number,
|
|
77
79
|
config=config,
|
|
78
80
|
)
|
|
@@ -97,7 +99,8 @@ class CriticalDatasetDeleteRule(LintRule):
|
|
|
97
99
|
if not patterns:
|
|
98
100
|
return
|
|
99
101
|
for step in iter_execution_steps(job.steps):
|
|
100
|
-
|
|
102
|
+
dds = _dataset_dds(step)
|
|
103
|
+
for dd in dds:
|
|
101
104
|
if not dd.dataset_name:
|
|
102
105
|
continue
|
|
103
106
|
upper = dd.dataset_name.upper()
|
|
@@ -156,7 +159,8 @@ class NewDatasetWithoutRlseRule(LintRule):
|
|
|
156
159
|
|
|
157
160
|
def check(self, job: Job, config: LintConfig) -> Iterable[ValidationError]:
|
|
158
161
|
for step in iter_execution_steps(job.steps):
|
|
159
|
-
|
|
162
|
+
dds = _dataset_dds(step)
|
|
163
|
+
for dd in dds:
|
|
160
164
|
dsn = dd.dataset_name
|
|
161
165
|
# A temporary dataset is deleted at the end of the job anyway.
|
|
162
166
|
if not dsn or dsn.startswith("&&"):
|
|
@@ -169,7 +173,7 @@ class NewDatasetWithoutRlseRule(LintRule):
|
|
|
169
173
|
if space is None or space.rlse:
|
|
170
174
|
continue
|
|
171
175
|
yield self._err(
|
|
172
|
-
f"DD '{dd
|
|
176
|
+
f"DD '{dd_label(dds, dd)}' in step '{step.name}' allocates new dataset "
|
|
173
177
|
f"'{dsn}' with SPACE but no RLSE",
|
|
174
178
|
line_number=dd.line_number,
|
|
175
179
|
config=config,
|
|
@@ -259,7 +263,8 @@ class LiteralSetValueHlqRule(LintRule):
|
|
|
259
263
|
if not symbols:
|
|
260
264
|
return
|
|
261
265
|
for step in iter_execution_steps(job.steps):
|
|
262
|
-
|
|
266
|
+
dds = _dataset_dds(step)
|
|
267
|
+
for dd in dds:
|
|
263
268
|
dsn = dd.dataset_name
|
|
264
269
|
# coded_dataset_name is set only when the name used symbols.
|
|
265
270
|
if not dsn or dd.coded_dataset_name is not None or "&" in dsn:
|
|
@@ -270,7 +275,7 @@ class LiteralSetValueHlqRule(LintRule):
|
|
|
270
275
|
if symbol is None:
|
|
271
276
|
continue
|
|
272
277
|
yield self._err(
|
|
273
|
-
f"DD '{dd
|
|
278
|
+
f"DD '{dd_label(dds, dd)}' in step '{step.name}' codes HLQ "
|
|
274
279
|
f"'{dsn.split('.', 1)[0]}' in '{dsn}' literally; "
|
|
275
280
|
f"SET symbol &{symbol} has that value",
|
|
276
281
|
line_number=dd.line_number,
|
|
@@ -1082,6 +1082,29 @@ def iter_execution_steps(steps: list[Step | ConditionalBlock]) -> Iterator[Step]
|
|
|
1082
1082
|
yield from iter_execution_steps(s.else_steps)
|
|
1083
1083
|
|
|
1084
1084
|
|
|
1085
|
+
def dd_label(dds: list[DDStatement], dd: DDStatement) -> str:
|
|
1086
|
+
"""*dd* as messages name it: ``IN``, or ``IN, concatenation member 2``
|
|
1087
|
+
for an unnamed concatenation member coded after ``//IN DD`` in *dds*
|
|
1088
|
+
(the named DD is member 1). The same label ``DatasetEvent.dd_label``
|
|
1089
|
+
gives. An unnamed DD with no named DD before it is ``(unnamed)``.
|
|
1090
|
+
"""
|
|
1091
|
+
if dd.name:
|
|
1092
|
+
return dd.name
|
|
1093
|
+
owner, member = "", 0
|
|
1094
|
+
for other in dds:
|
|
1095
|
+
if other.name:
|
|
1096
|
+
owner, member = other.name, 0
|
|
1097
|
+
elif owner:
|
|
1098
|
+
member = member + 1 if member else 2
|
|
1099
|
+
if other is dd:
|
|
1100
|
+
break
|
|
1101
|
+
else:
|
|
1102
|
+
return "(unnamed)"
|
|
1103
|
+
if not owner:
|
|
1104
|
+
return "(unnamed)"
|
|
1105
|
+
return f"{owner}, concatenation member {member}"
|
|
1106
|
+
|
|
1107
|
+
|
|
1085
1108
|
def _dataset_dds(step: Step) -> list[DDStatement]:
|
|
1086
1109
|
"""The DDs of *step* that can name a dataset, in source order.
|
|
1087
1110
|
|
|
@@ -13,12 +13,29 @@ comma-delimited sub-parameter of PARM:
|
|
|
13
13
|
The profile extracts the application program and PSB name from PARM so that
|
|
14
14
|
``analyze_step``/``analyze_job`` can report what application is actually
|
|
15
15
|
running, not just the region controller.
|
|
16
|
+
|
|
17
|
+
That layout holds only for the batch regions (BMP, DLI, DBB). A message
|
|
18
|
+
processing region codes ``PARM=(MSG,&CL1&CL2&CL3&CL4,...)``: PARM[1] is the
|
|
19
|
+
four three-digit transaction classes it serves and it names no program. For
|
|
20
|
+
MSG, IFP and any other region type, only the region type (and, for MSG, the
|
|
21
|
+
classes) is reported (issue #173).
|
|
16
22
|
"""
|
|
17
23
|
|
|
18
24
|
from __future__ import annotations
|
|
19
25
|
|
|
26
|
+
from ..model import Step
|
|
20
27
|
from .base import ProgramAnalysis, ProgramProfile
|
|
21
28
|
|
|
29
|
+
#: Region types whose PARM[1] / PARM[2] are the application program and PSB.
|
|
30
|
+
_BATCH_REGION_TYPES = frozenset({"BMP", "DLI", "DBB"})
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
def _transaction_classes(field: str) -> list[str]:
|
|
34
|
+
"""Split an MPP's ``&CL1&CL2&CL3&CL4`` field into its three-digit classes."""
|
|
35
|
+
if not field.isdigit() or len(field) % 3:
|
|
36
|
+
return []
|
|
37
|
+
return [field[i : i + 3] for i in range(0, len(field), 3)]
|
|
38
|
+
|
|
22
39
|
|
|
23
40
|
class DFSRRC00Profile(ProgramProfile):
|
|
24
41
|
program_names = frozenset({"DFSRRC00", "DFSRRC02"})
|
|
@@ -42,6 +59,9 @@ class DFSRRC00Profile(ProgramProfile):
|
|
|
42
59
|
parts = [p.strip() for p in parm.split(",")]
|
|
43
60
|
|
|
44
61
|
region_type = parts[0].upper() if parts and parts[0] else ""
|
|
62
|
+
if region_type and region_type not in _BATCH_REGION_TYPES:
|
|
63
|
+
return _online_region_analysis(step, region_type, parts, parm)
|
|
64
|
+
|
|
45
65
|
app_program = parts[1].upper() if len(parts) > 1 and parts[1] else ""
|
|
46
66
|
psb_name = parts[2].upper() if len(parts) > 2 and parts[2] else ""
|
|
47
67
|
|
|
@@ -67,3 +87,31 @@ class DFSRRC00Profile(ProgramProfile):
|
|
|
67
87
|
notes=notes,
|
|
68
88
|
raw_control=parm or None,
|
|
69
89
|
)
|
|
90
|
+
|
|
91
|
+
|
|
92
|
+
def _online_region_analysis(
|
|
93
|
+
step: Step, region_type: str, parts: list[str], parm: str
|
|
94
|
+
) -> ProgramAnalysis:
|
|
95
|
+
"""A region whose PARM names no program (MSG, IFP, ...): issue #173."""
|
|
96
|
+
notes = [f"IMS region type: {region_type}"]
|
|
97
|
+
classes = (
|
|
98
|
+
_transaction_classes(parts[1])
|
|
99
|
+
if region_type == "MSG" and len(parts) > 1
|
|
100
|
+
else []
|
|
101
|
+
)
|
|
102
|
+
summary = f"IMS {region_type} region ({step.program})"
|
|
103
|
+
if classes:
|
|
104
|
+
summary += f", classes {' '.join(classes)}"
|
|
105
|
+
notes.append(f"transaction classes: {' '.join(classes)}")
|
|
106
|
+
notes.append(
|
|
107
|
+
f"an IMS {region_type} region runs the transactions scheduled to it; "
|
|
108
|
+
"its PARM names no application program"
|
|
109
|
+
)
|
|
110
|
+
return ProgramAnalysis(
|
|
111
|
+
program=step.program or "DFSRRC00",
|
|
112
|
+
step_name=step.name or "",
|
|
113
|
+
summary=summary,
|
|
114
|
+
dataset_events=[],
|
|
115
|
+
notes=notes,
|
|
116
|
+
raw_control=parm or None,
|
|
117
|
+
)
|
|
@@ -15,6 +15,7 @@ from .model import (
|
|
|
15
15
|
Job,
|
|
16
16
|
Step,
|
|
17
17
|
_dataset_dds,
|
|
18
|
+
dd_label,
|
|
18
19
|
iter_execution_steps,
|
|
19
20
|
)
|
|
20
21
|
from .parameter_validators import DatasetNameValidator, DISPValidator
|
|
@@ -320,7 +321,10 @@ def validate_backward_references(job: Job) -> list[ValidationError]:
|
|
|
320
321
|
|
|
321
322
|
|
|
322
323
|
def validate_procedure_references(job: Job) -> list[ValidationError]:
|
|
323
|
-
"""
|
|
324
|
+
"""Check ``EXEC procname`` calls against the PROCs they call.
|
|
325
|
+
|
|
326
|
+
A symbolic the call assigns that the PROC never declares or references
|
|
327
|
+
is an ERROR (``IEFC657I``, issue #172).
|
|
324
328
|
|
|
325
329
|
:func:`jclparser.parse_jcl` adds every PROC it can load (in-stream,
|
|
326
330
|
JCLLIB / system PROCLIB under the base path, built-in) to
|
|
@@ -329,9 +333,24 @@ def validate_procedure_references(job: Job) -> list[ValidationError]:
|
|
|
329
333
|
a WARNING, not an ERROR (issue #116). A strict parse of a job with a
|
|
330
334
|
``JCLLIB ORDER`` still fails on it before this runs.
|
|
331
335
|
"""
|
|
336
|
+
from .explosion import unused_proc_symbols
|
|
337
|
+
|
|
332
338
|
errors: list[ValidationError] = []
|
|
333
339
|
for step in iter_execution_steps(job.steps):
|
|
334
|
-
|
|
340
|
+
procedure = job.get_procedure(step.procedure) if step.procedure else None
|
|
341
|
+
if procedure is not None:
|
|
342
|
+
# A symbolic the PROC never declares or references (IEFC657I).
|
|
343
|
+
for name in unused_proc_symbols(step, procedure):
|
|
344
|
+
errors.append(
|
|
345
|
+
ValidationError(
|
|
346
|
+
f"Step '{step.name or ''}': symbol {name} is not used by "
|
|
347
|
+
f"procedure '{step.procedure}' (IEFC657I); the "
|
|
348
|
+
"procedure's defaults apply",
|
|
349
|
+
ErrorType.SEMANTIC_ERROR,
|
|
350
|
+
line_number=step.line_number,
|
|
351
|
+
)
|
|
352
|
+
)
|
|
353
|
+
if step.procedure and procedure is None:
|
|
335
354
|
errors.append(
|
|
336
355
|
ValidationError(
|
|
337
356
|
f"Procedure '{step.procedure}' not found in-stream or in "
|
|
@@ -408,7 +427,7 @@ def validate_step_structure(step: Step) -> list[ValidationError]:
|
|
|
408
427
|
|
|
409
428
|
# Validate each DD statement
|
|
410
429
|
for dd in step.dd_statements:
|
|
411
|
-
errors.extend(validate_dd_statement(dd))
|
|
430
|
+
errors.extend(validate_dd_statement(dd, label=dd_label(step.dd_statements, dd)))
|
|
412
431
|
|
|
413
432
|
return errors
|
|
414
433
|
|
|
@@ -430,17 +449,25 @@ _TEMP_ALLOCATION_KEYS = (
|
|
|
430
449
|
)
|
|
431
450
|
|
|
432
451
|
|
|
433
|
-
def validate_dd_statement(
|
|
452
|
+
def validate_dd_statement(
|
|
453
|
+
dd: DDStatement, label: str | None = None
|
|
454
|
+
) -> list[ValidationError]:
|
|
434
455
|
"""
|
|
435
456
|
Validate DD statement structure and parameters.
|
|
436
457
|
|
|
437
458
|
Args:
|
|
438
459
|
dd: DDStatement object to validate
|
|
460
|
+
label: How messages name the DD (default ``dd.name``); the step
|
|
461
|
+
check passes ``model.dd_label`` so an unnamed concatenation
|
|
462
|
+
member reads ``IN, concatenation member 2``.
|
|
439
463
|
|
|
440
464
|
Returns:
|
|
441
465
|
List of validation errors
|
|
442
466
|
"""
|
|
443
467
|
errors = []
|
|
468
|
+
# How messages name the DD: *label* when the caller knows the DD's place
|
|
469
|
+
# in its step (``IN, concatenation member 2``, see ``model.dd_label``).
|
|
470
|
+
name = label if label is not None else dd.name
|
|
444
471
|
|
|
445
472
|
# DD name validation — empty names are valid for DD concatenation statements
|
|
446
473
|
if dd.name and not validate_jcl_identifier(dd.name):
|
|
@@ -470,7 +497,7 @@ def validate_dd_statement(dd: DDStatement) -> list[ValidationError]:
|
|
|
470
497
|
if dd.disposition and not validate_disposition(dd.disposition):
|
|
471
498
|
errors.append(
|
|
472
499
|
ValidationError(
|
|
473
|
-
f"Invalid DISP parameter '{dd.disposition}' on DD {
|
|
500
|
+
f"Invalid DISP parameter '{dd.disposition}' on DD {name}",
|
|
474
501
|
ErrorType.VALIDATION_ERROR,
|
|
475
502
|
line_number=dd.line_number,
|
|
476
503
|
)
|
|
@@ -500,7 +527,7 @@ def validate_dd_statement(dd: DDStatement) -> list[ValidationError]:
|
|
|
500
527
|
if conflict_count > 1:
|
|
501
528
|
errors.append(
|
|
502
529
|
ValidationError(
|
|
503
|
-
f"DD {
|
|
530
|
+
f"DD {name} has conflicting data source specifications",
|
|
504
531
|
ErrorType.VALIDATION_ERROR,
|
|
505
532
|
line_number=dd.line_number,
|
|
506
533
|
)
|
|
@@ -508,7 +535,7 @@ def validate_dd_statement(dd: DDStatement) -> list[ValidationError]:
|
|
|
508
535
|
elif source_count == 0:
|
|
509
536
|
errors.append(
|
|
510
537
|
ValidationError(
|
|
511
|
-
f"DD {
|
|
538
|
+
f"DD {name} must specify a data source (DSN, SYSOUT, DUMMY, or inline data)",
|
|
512
539
|
ErrorType.VALIDATION_ERROR,
|
|
513
540
|
line_number=dd.line_number,
|
|
514
541
|
)
|
|
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "jcl-parser"
|
|
7
|
-
version = "2.
|
|
7
|
+
version = "2.3.0"
|
|
8
8
|
description = "Parse IBM JCL (Job Control Language) scripts into Python objects"
|
|
9
9
|
authors = [
|
|
10
10
|
{name = "Samuel Dion", email = "sam94dion@gmail.com"}
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|