jcl-parser 2.0.0__tar.gz → 2.0.2__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {jcl_parser-2.0.0 → jcl_parser-2.0.2}/PKG-INFO +1 -1
- {jcl_parser-2.0.0 → jcl_parser-2.0.2}/jcl_parser.egg-info/PKG-INFO +1 -1
- {jcl_parser-2.0.0 → jcl_parser-2.0.2}/jcl_parser.egg-info/SOURCES.txt +1 -0
- {jcl_parser-2.0.0 → jcl_parser-2.0.2}/jclparser/__init__.py +1 -1
- {jcl_parser-2.0.0 → jcl_parser-2.0.2}/jclparser/_job_builder.py +24 -4
- {jcl_parser-2.0.0 → jcl_parser-2.0.2}/jclparser/_model_factories.py +56 -6
- {jcl_parser-2.0.0 → jcl_parser-2.0.2}/jclparser/_post_parse.py +158 -51
- {jcl_parser-2.0.0 → jcl_parser-2.0.2}/jclparser/_source_scanner.py +6 -3
- {jcl_parser-2.0.0 → jcl_parser-2.0.2}/jclparser/_statement_parser.py +36 -5
- {jcl_parser-2.0.0 → jcl_parser-2.0.2}/jclparser/condition_parser.py +63 -60
- {jcl_parser-2.0.0 → jcl_parser-2.0.2}/jclparser/continuation_handler.py +6 -6
- {jcl_parser-2.0.0 → jcl_parser-2.0.2}/jclparser/dataset_lifecycle.py +229 -110
- {jcl_parser-2.0.0 → jcl_parser-2.0.2}/jclparser/explosion.py +70 -24
- {jcl_parser-2.0.0 → jcl_parser-2.0.2}/jclparser/lint/rules.py +15 -4
- {jcl_parser-2.0.0 → jcl_parser-2.0.2}/jclparser/lint/runner.py +39 -7
- {jcl_parser-2.0.0 → jcl_parser-2.0.2}/jclparser/model.py +41 -1
- {jcl_parser-2.0.0 → jcl_parser-2.0.2}/jclparser/parser.py +123 -38
- jcl_parser-2.0.2/jclparser/program_profiles/_control.py +136 -0
- {jcl_parser-2.0.0 → jcl_parser-2.0.2}/jclparser/program_profiles/base.py +24 -2
- {jcl_parser-2.0.0 → jcl_parser-2.0.2}/jclparser/program_profiles/idcams.py +25 -5
- jcl_parser-2.0.2/jclparser/program_profiles/iebgener.py +197 -0
- {jcl_parser-2.0.0 → jcl_parser-2.0.2}/jclparser/program_profiles/ikjeft01.py +11 -3
- jcl_parser-2.0.2/jclparser/program_profiles/sort.py +144 -0
- {jcl_parser-2.0.0 → jcl_parser-2.0.2}/jclparser/rc_analysis.py +198 -109
- {jcl_parser-2.0.0 → jcl_parser-2.0.2}/jclparser/system_symbols.py +1 -1
- {jcl_parser-2.0.0 → jcl_parser-2.0.2}/jclparser/utils.py +71 -16
- {jcl_parser-2.0.0 → jcl_parser-2.0.2}/jclparser/validators.py +9 -2
- {jcl_parser-2.0.0 → jcl_parser-2.0.2}/pyproject.toml +1 -1
- jcl_parser-2.0.0/jclparser/program_profiles/iebgener.py +0 -67
- jcl_parser-2.0.0/jclparser/program_profiles/sort.py +0 -74
- {jcl_parser-2.0.0 → jcl_parser-2.0.2}/LICENSE +0 -0
- {jcl_parser-2.0.0 → jcl_parser-2.0.2}/README.md +0 -0
- {jcl_parser-2.0.0 → jcl_parser-2.0.2}/jcl_parser.egg-info/dependency_links.txt +0 -0
- {jcl_parser-2.0.0 → jcl_parser-2.0.2}/jcl_parser.egg-info/entry_points.txt +0 -0
- {jcl_parser-2.0.0 → jcl_parser-2.0.2}/jcl_parser.egg-info/requires.txt +0 -0
- {jcl_parser-2.0.0 → jcl_parser-2.0.2}/jcl_parser.egg-info/top_level.txt +0 -0
- {jcl_parser-2.0.0 → jcl_parser-2.0.2}/jclparser/__main__.py +0 -0
- {jcl_parser-2.0.0 → jcl_parser-2.0.2}/jclparser/builtin_procs.py +0 -0
- {jcl_parser-2.0.0 → jcl_parser-2.0.2}/jclparser/cli.py +0 -0
- {jcl_parser-2.0.0 → jcl_parser-2.0.2}/jclparser/errors.py +0 -0
- {jcl_parser-2.0.0 → jcl_parser-2.0.2}/jclparser/include_loader.py +0 -0
- {jcl_parser-2.0.0 → jcl_parser-2.0.2}/jclparser/jcl_file_type.py +0 -0
- {jcl_parser-2.0.0 → jcl_parser-2.0.2}/jclparser/jes_statements.py +0 -0
- {jcl_parser-2.0.0 → jcl_parser-2.0.2}/jclparser/lint/__init__.py +0 -0
- {jcl_parser-2.0.0 → jcl_parser-2.0.2}/jclparser/lint/base.py +0 -0
- {jcl_parser-2.0.0 → jcl_parser-2.0.2}/jclparser/lint/config.py +0 -0
- {jcl_parser-2.0.0 → jcl_parser-2.0.2}/jclparser/metrics.py +0 -0
- {jcl_parser-2.0.0 → jcl_parser-2.0.2}/jclparser/parameter_ast.py +0 -0
- {jcl_parser-2.0.0 → jcl_parser-2.0.2}/jclparser/parameter_validators.py +0 -0
- {jcl_parser-2.0.0 → jcl_parser-2.0.2}/jclparser/plugins/__init__.py +0 -0
- {jcl_parser-2.0.0 → jcl_parser-2.0.2}/jclparser/plugins/base.py +0 -0
- {jcl_parser-2.0.0 → jcl_parser-2.0.2}/jclparser/proc_loader.py +0 -0
- {jcl_parser-2.0.0 → jcl_parser-2.0.2}/jclparser/profiling.py +0 -0
- {jcl_parser-2.0.0 → jcl_parser-2.0.2}/jclparser/program_profiles/__init__.py +0 -0
- {jcl_parser-2.0.0 → jcl_parser-2.0.2}/jclparser/program_profiles/_invoked.py +0 -0
- {jcl_parser-2.0.0 → jcl_parser-2.0.2}/jclparser/program_profiles/adrdssu.py +0 -0
- {jcl_parser-2.0.0 → jcl_parser-2.0.2}/jclparser/program_profiles/binder.py +0 -0
- {jcl_parser-2.0.0 → jcl_parser-2.0.2}/jclparser/program_profiles/bpxbatch.py +0 -0
- {jcl_parser-2.0.0 → jcl_parser-2.0.2}/jclparser/program_profiles/compilers.py +0 -0
- {jcl_parser-2.0.0 → jcl_parser-2.0.2}/jclparser/program_profiles/db2_batch.py +0 -0
- {jcl_parser-2.0.0 → jcl_parser-2.0.2}/jclparser/program_profiles/dfsrrc00.py +0 -0
- {jcl_parser-2.0.0 → jcl_parser-2.0.2}/jclparser/program_profiles/dsnutilb.py +0 -0
- {jcl_parser-2.0.0 → jcl_parser-2.0.2}/jclparser/program_profiles/ftp.py +0 -0
- {jcl_parser-2.0.0 → jcl_parser-2.0.2}/jclparser/program_profiles/icetool.py +0 -0
- {jcl_parser-2.0.0 → jcl_parser-2.0.2}/jclparser/program_profiles/iebcompr.py +0 -0
- {jcl_parser-2.0.0 → jcl_parser-2.0.2}/jclparser/program_profiles/iebdg.py +0 -0
- {jcl_parser-2.0.0 → jcl_parser-2.0.2}/jclparser/program_profiles/iebptpch.py +0 -0
- {jcl_parser-2.0.0 → jcl_parser-2.0.2}/jclparser/program_profiles/iebupdte.py +0 -0
- {jcl_parser-2.0.0 → jcl_parser-2.0.2}/jclparser/program_profiles/iefbr14.py +0 -0
- {jcl_parser-2.0.0 → jcl_parser-2.0.2}/jclparser/program_profiles/iehinitt.py +0 -0
- {jcl_parser-2.0.0 → jcl_parser-2.0.2}/jclparser/program_profiles/iehlist.py +0 -0
- {jcl_parser-2.0.0 → jcl_parser-2.0.2}/jclparser/program_profiles/iehprogm.py +0 -0
- {jcl_parser-2.0.0 → jcl_parser-2.0.2}/jclparser/program_profiles/irxjcl.py +0 -0
- {jcl_parser-2.0.0 → jcl_parser-2.0.2}/jclparser/program_profiles/registry.py +0 -0
- {jcl_parser-2.0.0 → jcl_parser-2.0.2}/jclparser/py.typed +0 -0
- {jcl_parser-2.0.0 → jcl_parser-2.0.2}/jclparser/sql_detector.py +0 -0
- {jcl_parser-2.0.0 → jcl_parser-2.0.2}/setup.cfg +0 -0
|
@@ -45,6 +45,7 @@ jclparser/lint/runner.py
|
|
|
45
45
|
jclparser/plugins/__init__.py
|
|
46
46
|
jclparser/plugins/base.py
|
|
47
47
|
jclparser/program_profiles/__init__.py
|
|
48
|
+
jclparser/program_profiles/_control.py
|
|
48
49
|
jclparser/program_profiles/_invoked.py
|
|
49
50
|
jclparser/program_profiles/adrdssu.py
|
|
50
51
|
jclparser/program_profiles/base.py
|
|
@@ -20,7 +20,7 @@ application, for example::
|
|
|
20
20
|
import logging as _logging
|
|
21
21
|
|
|
22
22
|
# Package metadata
|
|
23
|
-
__version__ = "2.0.
|
|
23
|
+
__version__ = "2.0.2" # x-release-please-version
|
|
24
24
|
__author__ = "JCL Parser Team"
|
|
25
25
|
__description__ = "Parse IBM JCL scripts into Python objects"
|
|
26
26
|
|
|
@@ -131,10 +131,13 @@ class JobBuilder:
|
|
|
131
131
|
"XMIT": "_on_xmit",
|
|
132
132
|
}
|
|
133
133
|
|
|
134
|
-
def __init__(self, dialect: str = "standard") -> None:
|
|
134
|
+
def __init__(self, dialect: str = "standard", strict: bool = False) -> None:
|
|
135
135
|
#: ``ParserConfig.dialect``: decides which JES control statements are
|
|
136
136
|
#: interpreted (see ``jclparser.plugins.SUPPORTED_DIALECTS``).
|
|
137
137
|
self.dialect = dialect
|
|
138
|
+
#: Strict parsing: a SET value over ``MAX_SYMBOL_VALUE_LENGTH`` raises
|
|
139
|
+
#: instead of being left unresolved with a warning.
|
|
140
|
+
self.strict = strict
|
|
138
141
|
self.job: Job | None = None
|
|
139
142
|
#: tuples: (ConditionalBlock, active_branch)
|
|
140
143
|
self.conditional_stack: list[tuple[ConditionalBlock, str]] = []
|
|
@@ -437,7 +440,14 @@ class JobBuilder:
|
|
|
437
440
|
symbols = self.job.set_statements if self.job else self.pending_sets
|
|
438
441
|
|
|
439
442
|
def _record(symbol: str, value: str) -> None:
|
|
440
|
-
resolved = _resolve_set_value(
|
|
443
|
+
resolved = _resolve_set_value(
|
|
444
|
+
value,
|
|
445
|
+
symbols,
|
|
446
|
+
symbol=symbol,
|
|
447
|
+
line_number=stmt.line_number,
|
|
448
|
+
strict=self.strict,
|
|
449
|
+
warnings=self._warnings_sink(),
|
|
450
|
+
)
|
|
441
451
|
if procedure is not None:
|
|
442
452
|
procedure.set_statements[symbol] = resolved
|
|
443
453
|
elif self.job:
|
|
@@ -611,9 +621,19 @@ class JobBuilder:
|
|
|
611
621
|
self.pending_jes_typed.append(typed)
|
|
612
622
|
|
|
613
623
|
def _on_output(self, stmt: Statement, line_num: int) -> None:
|
|
614
|
-
# OUTPUT statement - JES output descriptor; promote to typed model
|
|
624
|
+
# OUTPUT statement - JES output descriptor; promote to typed model.
|
|
625
|
+
# Coded after an EXEC it is step-level, before the first EXEC
|
|
626
|
+
# job-level (or, in a PROC body, procedure-level): DD OUTPUT=*.name
|
|
627
|
+
# resolution depends on which (#85).
|
|
628
|
+
step = self.current_step
|
|
629
|
+
procedure = self.current_procedure
|
|
630
|
+
output = OutputStatement.from_statement(
|
|
631
|
+
stmt,
|
|
632
|
+
step_name=(step.name or "") if step is not None else None,
|
|
633
|
+
procedure_name=procedure.name if procedure is not None else None,
|
|
634
|
+
)
|
|
615
635
|
sink = self.job.output_statements if self.job else self.pending_outputs
|
|
616
|
-
sink.append(
|
|
636
|
+
sink.append(output)
|
|
617
637
|
|
|
618
638
|
def _on_null(self, stmt: Statement, line_num: int) -> None:
|
|
619
639
|
# NULL statement (//) - just ignore, used as delimiter
|
|
@@ -21,13 +21,22 @@ import re
|
|
|
21
21
|
from collections.abc import Callable
|
|
22
22
|
from typing import Any
|
|
23
23
|
|
|
24
|
+
from .errors import JCLSyntaxError
|
|
24
25
|
from .model import DDStatement, Job, Statement, Step
|
|
25
|
-
from .utils import POSITIONAL_PARAM_PREFIX, is_positional_key
|
|
26
|
+
from .utils import MAX_SYMBOL_VALUE_LENGTH, POSITIONAL_PARAM_PREFIX, is_positional_key
|
|
26
27
|
|
|
27
28
|
_SET_SYMBOL_PATTERN = re.compile(r"&&|&([A-Z0-9@#$]+)\.?")
|
|
28
29
|
|
|
29
30
|
|
|
30
|
-
def _resolve_set_value(
|
|
31
|
+
def _resolve_set_value(
|
|
32
|
+
value: str,
|
|
33
|
+
known_symbols: dict[str, str],
|
|
34
|
+
*,
|
|
35
|
+
symbol: str = "",
|
|
36
|
+
line_number: int = 0,
|
|
37
|
+
strict: bool = False,
|
|
38
|
+
warnings: list[str] | None = None,
|
|
39
|
+
) -> str:
|
|
31
40
|
"""Resolve &SYMBOL references in a SET value using already-accumulated SET symbols.
|
|
32
41
|
|
|
33
42
|
Mirrors the mainframe rule: SET values are evaluated left-to-right so a
|
|
@@ -36,15 +45,56 @@ def _resolve_set_value(value: str, known_symbols: dict[str, str]) -> str:
|
|
|
36
45
|
itself holds a "&&NAME" temporary-dataset marker (e.g. ``SET
|
|
37
46
|
MYTEMP=&&WORK``) before it ever reaches a DD statement. That collapse is
|
|
38
47
|
explode_job()'s job (Tier 2), not this Tier-1 SET-value resolution.
|
|
48
|
+
|
|
49
|
+
A resolved value longer than :data:`~jclparser.utils.MAX_SYMBOL_VALUE_LENGTH`
|
|
50
|
+
is never built (#70): its length is computed first, and the value is
|
|
51
|
+
returned unresolved with a message on *warnings* -- or, when *strict*,
|
|
52
|
+
:class:`~jclparser.errors.JCLSyntaxError` is raised. *symbol* and
|
|
53
|
+
*line_number* only label that message.
|
|
39
54
|
"""
|
|
55
|
+
matches = list(_SET_SYMBOL_PATTERN.finditer(value))
|
|
56
|
+
if not matches:
|
|
57
|
+
return value
|
|
40
58
|
|
|
41
|
-
def
|
|
59
|
+
def _replacement(m: re.Match[str]) -> str:
|
|
42
60
|
if m.group(0) == "&&":
|
|
43
61
|
return "&&"
|
|
44
|
-
|
|
45
|
-
return known_symbols.get(sym, m.group(0))
|
|
62
|
+
return known_symbols.get(m.group(1).upper(), m.group(0))
|
|
46
63
|
|
|
47
|
-
|
|
64
|
+
replacements = [_replacement(m) for m in matches]
|
|
65
|
+
length = len(value) + sum(
|
|
66
|
+
len(r) - len(m.group(0)) for m, r in zip(matches, replacements)
|
|
67
|
+
)
|
|
68
|
+
if length > MAX_SYMBOL_VALUE_LENGTH:
|
|
69
|
+
_report_set_overflow(symbol, length, line_number, strict, warnings)
|
|
70
|
+
return value
|
|
71
|
+
parts: list[str] = []
|
|
72
|
+
pos = 0
|
|
73
|
+
for m, r in zip(matches, replacements):
|
|
74
|
+
parts.append(value[pos : m.start()])
|
|
75
|
+
parts.append(r)
|
|
76
|
+
pos = m.end()
|
|
77
|
+
parts.append(value[pos:])
|
|
78
|
+
return "".join(parts)
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
def _report_set_overflow(
|
|
82
|
+
symbol: str,
|
|
83
|
+
length: int,
|
|
84
|
+
line_number: int,
|
|
85
|
+
strict: bool,
|
|
86
|
+
warnings: list[str] | None,
|
|
87
|
+
) -> None:
|
|
88
|
+
"""Raise (strict) or warn that SET *symbol* would resolve to *length* chars."""
|
|
89
|
+
message = (
|
|
90
|
+
f"SET symbol &{symbol or '?'} would resolve to {length:,} characters, "
|
|
91
|
+
f"more than the {MAX_SYMBOL_VALUE_LENGTH:,}-character limit"
|
|
92
|
+
)
|
|
93
|
+
if strict:
|
|
94
|
+
raise JCLSyntaxError(message, line_number)
|
|
95
|
+
if warnings is not None:
|
|
96
|
+
prefix = f"Line {line_number}: " if line_number else ""
|
|
97
|
+
warnings.append(f"{prefix}{message}; value left unresolved")
|
|
48
98
|
|
|
49
99
|
|
|
50
100
|
def _has_unresolved_symbol_ref(
|
|
@@ -24,7 +24,7 @@ from typing import TYPE_CHECKING, Any
|
|
|
24
24
|
|
|
25
25
|
from ._model_factories import _has_unresolved_symbol_ref
|
|
26
26
|
from .errors import JCLStructureError, JCLSyntaxError
|
|
27
|
-
from .model import Job, Step, iter_execution_steps
|
|
27
|
+
from .model import Job, OutputStatement, Step, iter_execution_steps
|
|
28
28
|
from .utils import split_source_lines
|
|
29
29
|
|
|
30
30
|
if TYPE_CHECKING:
|
|
@@ -48,64 +48,124 @@ def _extract_output_ref_names(value: Any) -> list[str]:
|
|
|
48
48
|
Values that do not start with ``*.`` (e.g. a bare nodename, a qualified
|
|
49
49
|
``STEP.PROCSTEP.N`` form without the leading ``*.``, or ``None``) yield
|
|
50
50
|
an empty list. Callers rely on this to distinguish output-statement
|
|
51
|
-
references from other uses of the OUTPUT= parameter.
|
|
51
|
+
references from other uses of the OUTPUT= parameter. The step
|
|
52
|
+
qualifiers are kept by :func:`_extract_output_refs`.
|
|
53
|
+
"""
|
|
54
|
+
return [ref[-1] for ref in _extract_output_refs(value)]
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
def _extract_output_refs(value: Any) -> list[tuple[str, ...]]:
|
|
58
|
+
"""Return the ``*.`` OUTPUT references of a DD ``OUTPUT=`` value, qualified.
|
|
59
|
+
|
|
60
|
+
Each reference is the tuple of its uppercased segments::
|
|
61
|
+
|
|
62
|
+
OUTPUT=*.NAME -> [("NAME",)]
|
|
63
|
+
OUTPUT=(*.STEP.NAME,*.N2) -> [("STEP", "NAME"), ("N2",)]
|
|
64
|
+
OUTPUT=*.STEP.PROCSTEP.NAME -> [("STEP", "PROCSTEP", "NAME")]
|
|
52
65
|
"""
|
|
53
|
-
names: list[str] = []
|
|
54
66
|
items: list[str]
|
|
55
67
|
if isinstance(value, list):
|
|
56
68
|
items = [str(v) for v in value]
|
|
57
69
|
elif isinstance(value, str):
|
|
58
70
|
items = [value]
|
|
59
71
|
else:
|
|
60
|
-
return
|
|
72
|
+
return []
|
|
61
73
|
|
|
74
|
+
refs: list[tuple[str, ...]] = []
|
|
62
75
|
for item in items:
|
|
63
76
|
token = item.strip().strip("()")
|
|
64
77
|
if not token.startswith("*."):
|
|
65
|
-
# Qualified form step.procstep.name uses ".name" — last segment after *
|
|
66
|
-
if ".*" in token:
|
|
67
|
-
# Not a back-reference either
|
|
68
|
-
continue
|
|
69
|
-
continue
|
|
70
|
-
ref = token[2:].strip()
|
|
71
|
-
if not ref:
|
|
72
78
|
continue
|
|
73
|
-
|
|
74
|
-
if
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
79
|
+
segments = tuple(seg.strip().upper() for seg in token[2:].split("."))
|
|
80
|
+
if segments[-1]:
|
|
81
|
+
refs.append(segments)
|
|
82
|
+
return refs
|
|
83
|
+
|
|
84
|
+
|
|
85
|
+
#: Key of an OUTPUT statement in :class:`_OutputScopes`:
|
|
86
|
+
#: ``(procedure_name, step_name, name)``, uppercased; ``None`` for a level
|
|
87
|
+
#: the statement is not coded in.
|
|
88
|
+
_OutputKey = tuple[str | None, str | None, str]
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
def _upper(name: str | None) -> str | None:
|
|
92
|
+
return name.upper() if name is not None else None
|
|
93
|
+
|
|
94
|
+
|
|
95
|
+
class _OutputScopes:
|
|
96
|
+
"""The job's OUTPUT statements, by the step (and PROC) they are coded in."""
|
|
97
|
+
|
|
98
|
+
def __init__(self, job: Job) -> None:
|
|
99
|
+
self.by_key: dict[_OutputKey, OutputStatement] = {}
|
|
100
|
+
for out in job.output_statements:
|
|
101
|
+
if out.name:
|
|
102
|
+
key = (
|
|
103
|
+
_upper(out.procedure_name),
|
|
104
|
+
_upper(out.step_name),
|
|
105
|
+
out.name.upper(),
|
|
106
|
+
)
|
|
107
|
+
self.by_key[key] = out # last wins if duplicated
|
|
108
|
+
self.instream_procs = {p.name.upper() for p in job.procedures}
|
|
109
|
+
self.step_procs = {
|
|
110
|
+
step.name.upper(): (step.procedure or "").upper()
|
|
111
|
+
for step in iter_execution_steps(job.steps)
|
|
112
|
+
if step.name
|
|
113
|
+
}
|
|
114
|
+
|
|
115
|
+
def checkable(self, ref: tuple[str, ...]) -> bool:
|
|
116
|
+
"""False for ``*.step.procstep.name`` into a PROC that is not in-stream.
|
|
117
|
+
|
|
118
|
+
Its OUTPUT statements are not part of the job, so whether the
|
|
119
|
+
reference resolves cannot be told.
|
|
120
|
+
"""
|
|
121
|
+
return len(ref) != 3 or self.step_procs.get(ref[0]) in self.instream_procs
|
|
122
|
+
|
|
123
|
+
def resolve(self, ref: tuple[str, ...], step_name: str) -> OutputStatement | None:
|
|
124
|
+
"""The OUTPUT statement *ref* names from a DD in job step *step_name*.
|
|
125
|
+
|
|
126
|
+
``*.name`` is the one in the DD's own step, else the job-level one;
|
|
127
|
+
``*.step.name`` the one in job step *step*; ``*.step.procstep.name``
|
|
128
|
+
the one in step *procstep* of the in-stream PROC job step *step*
|
|
129
|
+
calls.
|
|
130
|
+
"""
|
|
131
|
+
name = ref[-1]
|
|
132
|
+
if len(ref) == 1:
|
|
133
|
+
own = self.by_key.get((None, step_name.upper(), name))
|
|
134
|
+
return own if own is not None else self.by_key.get((None, None, name))
|
|
135
|
+
if len(ref) == 2:
|
|
136
|
+
return self.by_key.get((None, ref[0], name))
|
|
137
|
+
if len(ref) == 3:
|
|
138
|
+
return self.by_key.get((self.step_procs.get(ref[0]), ref[1], name))
|
|
139
|
+
return None
|
|
78
140
|
|
|
79
141
|
|
|
80
142
|
def _resolve_output_refs(job: Job) -> None:
|
|
81
143
|
"""Populate ``DDStatement.output_refs`` for every DD that uses ``OUTPUT=*.X``.
|
|
82
144
|
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
145
|
+
A reference resolves by scope, not by name alone (#85): see
|
|
146
|
+
:meth:`_OutputScopes.resolve`. Unresolved references (no matching
|
|
147
|
+
:class:`OutputStatement`) are recorded on ``job.unresolved_output_refs``
|
|
148
|
+
as ``(step_name, ref, line_number)`` tuples, *ref* as coded after ``*.``
|
|
149
|
+
(``"NAME"``, ``"STEP.NAME"``), so
|
|
150
|
+
:func:`jclparser.validators.validate_job_structure` can emit warnings
|
|
151
|
+
without re-scanning the parameter string. A reference into a PROC that
|
|
152
|
+
is not in-stream cannot be checked and is neither.
|
|
87
153
|
"""
|
|
88
154
|
if not job:
|
|
89
155
|
return
|
|
90
|
-
|
|
91
|
-
output_by_name: dict[str, Any] = {
|
|
92
|
-
out.name.upper(): out for out in job.output_statements if out.name
|
|
93
|
-
}
|
|
94
|
-
|
|
95
|
-
from .model import iter_execution_steps
|
|
96
|
-
|
|
156
|
+
scopes = _OutputScopes(job)
|
|
97
157
|
for step in iter_execution_steps(job.steps):
|
|
98
158
|
for dd in step.dd_statements:
|
|
99
159
|
value = dd.parameters.get("OUTPUT")
|
|
100
160
|
if value is None:
|
|
101
161
|
continue
|
|
102
|
-
for ref in
|
|
103
|
-
target =
|
|
162
|
+
for ref in _extract_output_refs(value):
|
|
163
|
+
target = scopes.resolve(ref, step.name or "")
|
|
104
164
|
if target is not None:
|
|
105
165
|
dd.output_refs.append(target)
|
|
106
|
-
|
|
166
|
+
elif scopes.checkable(ref):
|
|
107
167
|
job.unresolved_output_refs.append(
|
|
108
|
-
(step.name or "", ref, dd.line_number)
|
|
168
|
+
(step.name or "", ".".join(ref), dd.line_number)
|
|
109
169
|
)
|
|
110
170
|
|
|
111
171
|
|
|
@@ -186,7 +246,7 @@ def enforce_strict_dd_structure(content: str) -> None:
|
|
|
186
246
|
)
|
|
187
247
|
|
|
188
248
|
|
|
189
|
-
def resolve_set_symbols(job: Job) -> None:
|
|
249
|
+
def resolve_set_symbols(job: Job, strict: bool = False) -> None:
|
|
190
250
|
"""Substitute fully-resolved SET symbols into DD and EXEC-step parameters.
|
|
191
251
|
|
|
192
252
|
SET statements may appear before or after the DD statements that reference
|
|
@@ -214,11 +274,17 @@ def resolve_set_symbols(job: Job) -> None:
|
|
|
214
274
|
pass. It also covers EXEC step parameters (e.g. ``PARM=&SYM``), not just
|
|
215
275
|
DD dataset names/dispositions, so a job-level SET symbol resolves the same
|
|
216
276
|
way regardless of which kind of statement references it.
|
|
277
|
+
|
|
278
|
+
A SET value whose expansion would exceed ``utils.MAX_SYMBOL_VALUE_LENGTH``
|
|
279
|
+
characters is left unresolved with one warning on ``job.warnings`` (#70);
|
|
280
|
+
a statement referencing it keeps the reference as written (again with a
|
|
281
|
+
warning). With *strict*, :class:`~jclparser.errors.JCLSyntaxError` is
|
|
282
|
+
raised instead.
|
|
217
283
|
"""
|
|
218
284
|
from .explosion import substitute_dd_statement
|
|
219
285
|
from .system_symbols import is_system_symbol_name
|
|
220
286
|
|
|
221
|
-
_set_scope = _scope_for_table(job.set_statements)
|
|
287
|
+
_set_scope = _scope_for_table(job.set_statements, job.warnings, strict)
|
|
222
288
|
# A step read before a SET redefined a symbol sees the value in effect at
|
|
223
289
|
# that point (Step.set_symbols); symbols it has no snapshot for (forward
|
|
224
290
|
# references) still fall back to the job's final table.
|
|
@@ -230,7 +296,7 @@ def resolve_set_symbols(job: Job) -> None:
|
|
|
230
296
|
for _owner, _dd in _dds_with_owners(_flat_steps):
|
|
231
297
|
substitute_dd_statement(
|
|
232
298
|
_dd,
|
|
233
|
-
_scope_for_step(_owner, job, _set_scope, _step_scopes),
|
|
299
|
+
_scope_for_step(_owner, job, _set_scope, _step_scopes, strict),
|
|
234
300
|
strict=False,
|
|
235
301
|
warnings=job.warnings,
|
|
236
302
|
collapse_double_amp=False,
|
|
@@ -238,14 +304,18 @@ def resolve_set_symbols(job: Job) -> None:
|
|
|
238
304
|
)
|
|
239
305
|
|
|
240
306
|
for _step in _flat_steps:
|
|
241
|
-
_step_scope = _scope_for_step(_step, job, _set_scope, _step_scopes)
|
|
307
|
+
_step_scope = _scope_for_step(_step, job, _set_scope, _step_scopes, strict)
|
|
242
308
|
_substitute_step_parameters(_step, _step_scope, job.warnings)
|
|
243
309
|
_substitute_step_names(_step, _step_scope)
|
|
244
310
|
|
|
245
311
|
|
|
246
|
-
def _preresolve_set_table(table: dict[str, str]) -> dict[str, str]:
|
|
247
|
-
"""Return a copy of SET *table* with symbol references between values expanded.
|
|
248
|
-
|
|
312
|
+
def _preresolve_set_table(table: dict[str, str]) -> tuple[dict[str, str], list[str]]:
|
|
313
|
+
"""Return a copy of SET *table* with symbol references between values expanded.
|
|
314
|
+
|
|
315
|
+
Also returns the names of the symbols whose expansion would exceed
|
|
316
|
+
``utils.MAX_SYMBOL_VALUE_LENGTH``; those keep their unexpanded value.
|
|
317
|
+
"""
|
|
318
|
+
from .explosion import SymbolScope, _substitute_symbols_bounded, _SymbolOverflow
|
|
249
319
|
|
|
250
320
|
# Pre-resolve the symbol table so every value is fully expanded before
|
|
251
321
|
# we apply it to DD dataset names. On each pass we substitute symbols
|
|
@@ -253,36 +323,67 @@ def _preresolve_set_table(table: dict[str, str]) -> dict[str, str]:
|
|
|
253
323
|
# (excluding the symbol itself prevents direct self-reference loops).
|
|
254
324
|
# We repeat until no value changes or we reach the pass limit.
|
|
255
325
|
_raw: dict[str, str] = dict(table)
|
|
326
|
+
_overflowed: list[str] = []
|
|
256
327
|
for _pass in range(10):
|
|
257
328
|
_changed = False
|
|
258
329
|
for _name, _val in list(_raw.items()):
|
|
259
|
-
if "&" not in _val:
|
|
330
|
+
if "&" not in _val or _name in _overflowed:
|
|
260
331
|
continue
|
|
261
332
|
# Build a temporary scope that excludes this symbol's own name
|
|
262
333
|
# so "&A." inside SET A's value does not expand to itself.
|
|
263
334
|
_scope_for_val = SymbolScope(
|
|
264
335
|
set_symbols={k: v for k, v in _raw.items() if k != _name}
|
|
265
336
|
)
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
|
|
337
|
+
try:
|
|
338
|
+
_new_val = _substitute_symbols_bounded(
|
|
339
|
+
_val,
|
|
340
|
+
_scope_for_val,
|
|
341
|
+
allow_undefined=True,
|
|
342
|
+
collapse_double_amp=False,
|
|
343
|
+
)
|
|
344
|
+
except _SymbolOverflow:
|
|
345
|
+
# Values only grow from pass to pass: never retry it.
|
|
346
|
+
_overflowed.append(_name)
|
|
347
|
+
continue
|
|
272
348
|
if _new_val != _val:
|
|
273
349
|
_raw[_name] = _new_val
|
|
274
350
|
_changed = True
|
|
275
351
|
if not _changed:
|
|
276
352
|
break # table has stabilised – no need for more passes
|
|
277
|
-
return _raw
|
|
353
|
+
return _raw, _overflowed
|
|
278
354
|
|
|
279
355
|
|
|
280
|
-
def
|
|
281
|
-
|
|
356
|
+
def _report_table_overflow(
|
|
357
|
+
overflowed: list[str], warnings: list[str], strict: bool
|
|
358
|
+
) -> None:
|
|
359
|
+
"""Raise (strict) or warn once that the *overflowed* SET symbols stay unresolved."""
|
|
360
|
+
from .utils import MAX_SYMBOL_VALUE_LENGTH
|
|
361
|
+
|
|
362
|
+
names = ", ".join(f"&{n}" for n in overflowed)
|
|
363
|
+
message = (
|
|
364
|
+
f"SET symbol(s) {names} expand to more than "
|
|
365
|
+
f"{MAX_SYMBOL_VALUE_LENGTH:,} characters"
|
|
366
|
+
)
|
|
367
|
+
if strict:
|
|
368
|
+
raise JCLSyntaxError(message)
|
|
369
|
+
message += "; left unresolved"
|
|
370
|
+
if message not in warnings:
|
|
371
|
+
warnings.append(message)
|
|
372
|
+
|
|
373
|
+
|
|
374
|
+
def _scope_for_table(
|
|
375
|
+
table: dict[str, str], warnings: list[str], strict: bool = False
|
|
376
|
+
) -> SymbolScope:
|
|
377
|
+
"""Build the substitution scope for SET symbol *table* (see resolve_set_symbols).
|
|
378
|
+
|
|
379
|
+
Overflow of a SET value is reported on *warnings* (or raised, *strict*).
|
|
380
|
+
"""
|
|
282
381
|
from .explosion import SymbolScope
|
|
283
382
|
from .system_symbols import is_system_symbol_name
|
|
284
383
|
|
|
285
|
-
_raw = _preresolve_set_table(table)
|
|
384
|
+
_raw, _overflowed = _preresolve_set_table(table)
|
|
385
|
+
if _overflowed:
|
|
386
|
+
_report_table_overflow(_overflowed, warnings, strict)
|
|
286
387
|
|
|
287
388
|
# Symbols whose values still contain a genuinely unresolved
|
|
288
389
|
# &-reference after all passes are excluded from the final scope.
|
|
@@ -295,10 +396,15 @@ def _scope_for_table(table: dict[str, str]) -> SymbolScope:
|
|
|
295
396
|
# ``DSN=&SYSUID..WORK`` before that pass runs (keep_undefined below). A value that legitimately contains "&&" (e.g.
|
|
296
397
|
# SET MYTEMP=&&WORK) is NOT excluded -- it has nothing left to
|
|
297
398
|
# resolve, it just happens to contain an escaped literal ampersand.
|
|
399
|
+
#
|
|
400
|
+
# An overflowed symbol keeps its unexpanded value in the scope, so a
|
|
401
|
+
# statement referencing it overflows the substitution budget in turn and
|
|
402
|
+
# keeps the reference as written instead of having it blanked (#70).
|
|
298
403
|
_resolved_symbols = {
|
|
299
404
|
k: v
|
|
300
405
|
for k, v in _raw.items()
|
|
301
|
-
if
|
|
406
|
+
if k in _overflowed
|
|
407
|
+
or not _has_unresolved_symbol_ref(v, ignore=is_system_symbol_name)
|
|
302
408
|
}
|
|
303
409
|
return SymbolScope(set_symbols=_resolved_symbols)
|
|
304
410
|
|
|
@@ -308,6 +414,7 @@ def _scope_for_step(
|
|
|
308
414
|
job: Job,
|
|
309
415
|
set_scope: SymbolScope,
|
|
310
416
|
step_scopes: dict[int, SymbolScope],
|
|
417
|
+
strict: bool = False,
|
|
311
418
|
) -> SymbolScope:
|
|
312
419
|
"""Return the SET scope in effect for *step*, memoised in *step_scopes*.
|
|
313
420
|
|
|
@@ -318,7 +425,7 @@ def _scope_for_step(
|
|
|
318
425
|
return set_scope
|
|
319
426
|
if id(step) not in step_scopes:
|
|
320
427
|
step_scopes[id(step)] = _scope_for_table(
|
|
321
|
-
{**job.set_statements, **step.set_symbols}
|
|
428
|
+
{**job.set_statements, **step.set_symbols}, job.warnings, strict
|
|
322
429
|
)
|
|
323
430
|
return step_scopes[id(step)]
|
|
324
431
|
|
|
@@ -31,7 +31,7 @@ from .errors import JCLSyntaxError
|
|
|
31
31
|
from .utils import (
|
|
32
32
|
instream_delimiter,
|
|
33
33
|
is_continued_card_image,
|
|
34
|
-
|
|
34
|
+
normalize_jcl_lines,
|
|
35
35
|
split_source_lines,
|
|
36
36
|
)
|
|
37
37
|
|
|
@@ -60,7 +60,7 @@ def _merge_source_lines(content: str) -> ScannedSource:
|
|
|
60
60
|
or garbled records).
|
|
61
61
|
"""
|
|
62
62
|
raw_lines = split_source_lines(content)
|
|
63
|
-
lines =
|
|
63
|
+
lines = normalize_jcl_lines(raw_lines)
|
|
64
64
|
|
|
65
65
|
classifier = _RecordClassifier(lines)
|
|
66
66
|
jcl_indices = classifier.run()
|
|
@@ -116,7 +116,10 @@ def merge_continuations(
|
|
|
116
116
|
card_images={
|
|
117
117
|
n
|
|
118
118
|
for n, orig in enumerate(indices)
|
|
119
|
-
if is_continued_card_image(
|
|
119
|
+
if is_continued_card_image(
|
|
120
|
+
raw_lines[orig],
|
|
121
|
+
raw_lines[orig + 1] if orig + 1 < len(raw_lines) else None,
|
|
122
|
+
)
|
|
120
123
|
},
|
|
121
124
|
)
|
|
122
125
|
|
|
@@ -173,17 +173,36 @@ def _resumes_operands(text: str) -> bool:
|
|
|
173
173
|
return "=" in token.split(",")[0]
|
|
174
174
|
|
|
175
175
|
|
|
176
|
-
def
|
|
176
|
+
def _closes_parens(text: str, depth: int) -> bool:
|
|
177
|
+
"""Return True if *text*, read at paren *depth*, brings the depth to 0."""
|
|
178
|
+
in_quotes = False
|
|
179
|
+
for char in text:
|
|
180
|
+
if char == "'":
|
|
181
|
+
in_quotes = not in_quotes
|
|
182
|
+
elif not in_quotes:
|
|
183
|
+
if char == "(":
|
|
184
|
+
depth += 1
|
|
185
|
+
elif char == ")":
|
|
186
|
+
depth -= 1
|
|
187
|
+
if depth <= 0:
|
|
188
|
+
return True
|
|
189
|
+
return False
|
|
190
|
+
|
|
191
|
+
|
|
192
|
+
def _find_comment_start(operands: str, depth: int = 0) -> int | None:
|
|
177
193
|
"""Index of the comment field inside *operands*, or ``None`` if there is none.
|
|
178
194
|
|
|
179
195
|
The operand field ends at the first blank that is outside quotes and
|
|
180
196
|
outside parentheses; everything after it is the comment field (z/OS JCL
|
|
181
|
-
Reference, "Comment field").
|
|
197
|
+
Reference, "Comment field"). Inside a parenthesised list, a blank that
|
|
198
|
+
follows a subparameter's comma also starts a comment when the rest of
|
|
199
|
+
the record neither resumes operands nor closes the list: the list goes
|
|
200
|
+
on in the next record. *depth* is the paren depth the record starts at
|
|
201
|
+
(non-zero for a record continuing an open list).
|
|
182
202
|
"""
|
|
183
203
|
if " " not in operands and "\t" not in operands:
|
|
184
204
|
# No blank at all: the operand field runs to the end of the record.
|
|
185
205
|
return None
|
|
186
|
-
depth = 0
|
|
187
206
|
in_quotes = False
|
|
188
207
|
i = 0
|
|
189
208
|
n = len(operands)
|
|
@@ -207,6 +226,16 @@ def _find_comment_start(operands: str) -> int | None:
|
|
|
207
226
|
i = n - len(rest)
|
|
208
227
|
continue
|
|
209
228
|
return i
|
|
229
|
+
elif (
|
|
230
|
+
char.isspace()
|
|
231
|
+
and operands[:i].endswith(",")
|
|
232
|
+
and not _resumes_operands(operands[i:].lstrip())
|
|
233
|
+
and not _closes_parens(operands[i:], depth)
|
|
234
|
+
):
|
|
235
|
+
# A parenthesised list interrupted after a subparameter's
|
|
236
|
+
# comma may carry a comment (z/OS JCL Reference, "Continuing
|
|
237
|
+
# the operand field"); the list resumes on the next record.
|
|
238
|
+
return i
|
|
210
239
|
i += 1
|
|
211
240
|
return None
|
|
212
241
|
|
|
@@ -240,7 +269,9 @@ def _find_then_end(operands: str) -> int | None:
|
|
|
240
269
|
return None
|
|
241
270
|
|
|
242
271
|
|
|
243
|
-
def split_comment_field(
|
|
272
|
+
def split_comment_field(
|
|
273
|
+
line: str, group_operation: str = "", paren_depth: int = 0
|
|
274
|
+
) -> tuple[str, str]:
|
|
244
275
|
"""Split a JCL statement record into ``(statement, comment)``.
|
|
245
276
|
|
|
246
277
|
In JCL a statement's operand field ends at the first blank outside quotes
|
|
@@ -270,7 +301,7 @@ def split_comment_field(line: str, group_operation: str = "") -> tuple[str, str]
|
|
|
270
301
|
if cut is None:
|
|
271
302
|
return line, ""
|
|
272
303
|
return (line[:start] + operands[:cut]).rstrip(), operands[cut:].strip()
|
|
273
|
-
cut = _find_comment_start(operands)
|
|
304
|
+
cut = _find_comment_start(operands, paren_depth)
|
|
274
305
|
if cut is None:
|
|
275
306
|
return line, ""
|
|
276
307
|
return (line[:start] + operands[:cut]).rstrip(), operands[cut:].strip()
|