dataspector 1.0.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- dataspector/__init__.py +64 -0
- dataspector/__main__.py +10 -0
- dataspector/_connectors.py +260 -0
- dataspector/_reporting.py +480 -0
- dataspector/checkers/__init__.py +85 -0
- dataspector/checkers/record.py +671 -0
- dataspector/checkers/source.py +344 -0
- dataspector/checkers/utils.py +342 -0
- dataspector/cli.py +207 -0
- dataspector/coboller/__init__.py +123 -0
- dataspector/coboller/codecs.py +841 -0
- dataspector/coboller/constants.py +324 -0
- dataspector/coboller/copybook.py +1448 -0
- dataspector/coboller/parser.py +201 -0
- dataspector/core.py +740 -0
- dataspector/py.typed +0 -0
- dataspector/settings.py +101 -0
- dataspector-1.0.0.dist-info/METADATA +181 -0
- dataspector-1.0.0.dist-info/RECORD +22 -0
- dataspector-1.0.0.dist-info/WHEEL +4 -0
- dataspector-1.0.0.dist-info/entry_points.txt +3 -0
- dataspector-1.0.0.dist-info/licenses/LICENSE +21 -0
|
@@ -0,0 +1,1448 @@
|
|
|
1
|
+
"""COBOL copybook AST classes (:class:`Copybook`, :class:`GroupItem`, :class:`ElementaryItem`)."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import copy
|
|
6
|
+
import pathlib
|
|
7
|
+
from collections.abc import Callable, Iterator
|
|
8
|
+
from dataclasses import dataclass
|
|
9
|
+
from decimal import Decimal
|
|
10
|
+
from typing import Any, Literal
|
|
11
|
+
|
|
12
|
+
import pandas as pd
|
|
13
|
+
import pandera.pandas as pa
|
|
14
|
+
from loguru import logger
|
|
15
|
+
|
|
16
|
+
from dataspector._reporting import chunks_to_dataframe, export_chunks
|
|
17
|
+
from dataspector.coboller.codecs import (
|
|
18
|
+
bit_unpack,
|
|
19
|
+
comp3_unpack,
|
|
20
|
+
comp6_unpack,
|
|
21
|
+
comp_float_unpack,
|
|
22
|
+
comp_unpack,
|
|
23
|
+
compute_decimal_exponent,
|
|
24
|
+
expand_pic,
|
|
25
|
+
external_float_unpack,
|
|
26
|
+
is_ebcdic,
|
|
27
|
+
parse_numeric_edited,
|
|
28
|
+
predict_encoding,
|
|
29
|
+
zoned_unpack,
|
|
30
|
+
)
|
|
31
|
+
from dataspector.coboller.constants import (
|
|
32
|
+
BINARY_USAGES,
|
|
33
|
+
DECIMAL_COMMA_RE,
|
|
34
|
+
FIXED_BINARY_SIZES,
|
|
35
|
+
PIC_ALPHA_ONLY,
|
|
36
|
+
PIC_BIT_ONLY,
|
|
37
|
+
PIC_DBCS_ONLY,
|
|
38
|
+
PIC_EDITED_MARKERS,
|
|
39
|
+
PIC_EXTERNAL_FLOAT,
|
|
40
|
+
PIC_NATIONAL_ONLY,
|
|
41
|
+
PIC_NUMERIC_PURE,
|
|
42
|
+
PIC_UTF8_ONLY,
|
|
43
|
+
SELF_SIZED_USAGES,
|
|
44
|
+
USAGE_ALIASES,
|
|
45
|
+
DataCategory,
|
|
46
|
+
SignMode,
|
|
47
|
+
Usage,
|
|
48
|
+
)
|
|
49
|
+
from dataspector.coboller.parser import (
|
|
50
|
+
extract_statements,
|
|
51
|
+
parse_condition_values,
|
|
52
|
+
parse_statement,
|
|
53
|
+
)
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
@dataclass(frozen=True, slots=True)
|
|
57
|
+
class ConditionName:
|
|
58
|
+
"""Level-88 condition-name attached to a COBOL data item."""
|
|
59
|
+
|
|
60
|
+
name: str
|
|
61
|
+
raw_value: str
|
|
62
|
+
values: tuple[str, ...] = ()
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
@dataclass(frozen=True, slots=True)
|
|
66
|
+
class ResolvedItem:
|
|
67
|
+
"""Flattened elementary item with exact byte offset and width in a record."""
|
|
68
|
+
|
|
69
|
+
name: str
|
|
70
|
+
field: ElementaryItem
|
|
71
|
+
offset: int
|
|
72
|
+
byte_length: int
|
|
73
|
+
is_redefines: bool = False
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
class Copybook:
|
|
77
|
+
"""COBOL copybook parser, layout inspector, data reader, and format converter.
|
|
78
|
+
|
|
79
|
+
Parameters
|
|
80
|
+
----------
|
|
81
|
+
copybook
|
|
82
|
+
Path to a COBOL copybook file.
|
|
83
|
+
"""
|
|
84
|
+
|
|
85
|
+
def __init__(self, copybook: str | pathlib.Path) -> None:
|
|
86
|
+
self._copybook = copybook
|
|
87
|
+
self.copybook: pathlib.Path = pathlib.Path(copybook)
|
|
88
|
+
logger.trace("Loading COBOL copybook from {}", self.copybook)
|
|
89
|
+
self.raw: str = self.copybook.read_text(encoding="utf-8", errors="replace")
|
|
90
|
+
self.records: list[GroupItem | ElementaryItem] = parse_copybook(self.raw)
|
|
91
|
+
|
|
92
|
+
@classmethod
|
|
93
|
+
def from_file(cls, path: str | pathlib.Path) -> Copybook:
|
|
94
|
+
"""Construct a :class:`Copybook` from a copybook file path.
|
|
95
|
+
|
|
96
|
+
Parameters
|
|
97
|
+
----------
|
|
98
|
+
path
|
|
99
|
+
Path to a COBOL copybook file.
|
|
100
|
+
|
|
101
|
+
Returns
|
|
102
|
+
-------
|
|
103
|
+
Copybook
|
|
104
|
+
Parsed copybook instance.
|
|
105
|
+
"""
|
|
106
|
+
return cls(path)
|
|
107
|
+
|
|
108
|
+
@classmethod
|
|
109
|
+
def from_string(cls, text: str, name: str = "<string>") -> Copybook:
|
|
110
|
+
"""Construct a :class:`Copybook` directly from raw copybook source text.
|
|
111
|
+
|
|
112
|
+
Parameters
|
|
113
|
+
----------
|
|
114
|
+
text
|
|
115
|
+
Raw COBOL copybook source code.
|
|
116
|
+
name
|
|
117
|
+
Synthetic display name assigned to :attr:`copybook`.
|
|
118
|
+
|
|
119
|
+
Returns
|
|
120
|
+
-------
|
|
121
|
+
Copybook
|
|
122
|
+
Parsed copybook instance.
|
|
123
|
+
"""
|
|
124
|
+
instance = cls.__new__(cls)
|
|
125
|
+
instance._copybook = name
|
|
126
|
+
instance.copybook = pathlib.Path(name)
|
|
127
|
+
instance.raw = text
|
|
128
|
+
instance.records = parse_copybook(text)
|
|
129
|
+
return instance
|
|
130
|
+
|
|
131
|
+
@property
|
|
132
|
+
def columns(self) -> list[str]:
|
|
133
|
+
"""Alias for :attr:`fields`."""
|
|
134
|
+
return self.fields
|
|
135
|
+
|
|
136
|
+
@property
|
|
137
|
+
def decimal_comma(self) -> bool:
|
|
138
|
+
"""``True`` if the copybook declares ``DECIMAL-POINT IS COMMA``."""
|
|
139
|
+
return bool(DECIMAL_COMMA_RE.search(self.raw))
|
|
140
|
+
|
|
141
|
+
@property
|
|
142
|
+
def exists(self) -> bool:
|
|
143
|
+
"""``True`` if the underlying copybook file exists on disk."""
|
|
144
|
+
return self.copybook.exists()
|
|
145
|
+
|
|
146
|
+
@property
|
|
147
|
+
def fields(self) -> list[str]:
|
|
148
|
+
"""Ordered elementary item names in primary record order."""
|
|
149
|
+
return [rf.name for rf in self.layout(include_redefines=False, include_filler=True)]
|
|
150
|
+
|
|
151
|
+
@property
|
|
152
|
+
def has_binary_fields(self) -> bool:
|
|
153
|
+
"""``True`` if any item uses binary, packed, float, pointer, index, or wide storage."""
|
|
154
|
+
return any(rf.field.is_binary for rf in self.layout(include_redefines=True))
|
|
155
|
+
|
|
156
|
+
@property
|
|
157
|
+
def record_length(self) -> int:
|
|
158
|
+
"""Total physical byte length of a single record."""
|
|
159
|
+
return sum(rec.byte_length * rec.occurs for rec in self.records if rec.redefines is None)
|
|
160
|
+
|
|
161
|
+
def convert(
|
|
162
|
+
self,
|
|
163
|
+
datfile: str | pathlib.Path,
|
|
164
|
+
output: str | pathlib.Path,
|
|
165
|
+
chunksize: int = 5000,
|
|
166
|
+
encoding: str | None = None,
|
|
167
|
+
float_format: Literal["ieee", "ibm"] = "ieee",
|
|
168
|
+
byteorder: Literal["big", "little"] = "big",
|
|
169
|
+
include_redefines: bool = False,
|
|
170
|
+
include_filler: bool = False,
|
|
171
|
+
line_ending: Literal["auto", "none", "lf", "crlf"] = "auto",
|
|
172
|
+
dbcs_encoding: str = "utf-16-be",
|
|
173
|
+
decimal_comma: bool | None = None,
|
|
174
|
+
) -> pathlib.Path:
|
|
175
|
+
"""Convert a fixed-width or compressed COBOL data file into a modern readable format.
|
|
176
|
+
|
|
177
|
+
Streams chunk-by-chunk via :meth:`yield_data` for ``.csv``, ``.tsv``,
|
|
178
|
+
and ``.jsonl`` outputs; also supports ``.json``, ``.xlsx``, and
|
|
179
|
+
``.parquet``. Aligned with :meth:`~dataspector.core.GenericData.convert`.
|
|
180
|
+
|
|
181
|
+
Parameters
|
|
182
|
+
----------
|
|
183
|
+
datfile
|
|
184
|
+
Path to the raw COBOL data file.
|
|
185
|
+
output
|
|
186
|
+
Destination file path whose extension selects the output format.
|
|
187
|
+
chunksize
|
|
188
|
+
Number of rows per streaming chunk.
|
|
189
|
+
encoding
|
|
190
|
+
Character encoding for ``DISPLAY`` items (e.g. ``'cp500'``, ``'ascii'``).
|
|
191
|
+
float_format
|
|
192
|
+
Floating-point representation for ``COMP-1``/``COMP-2`` (``'ieee'`` or ``'ibm'``).
|
|
193
|
+
byteorder
|
|
194
|
+
Byte order for binary fields (``'big'`` or ``'little'``).
|
|
195
|
+
include_redefines
|
|
196
|
+
Include ``REDEFINES`` overlay columns in the converted output.
|
|
197
|
+
include_filler
|
|
198
|
+
Include ``FILLER`` columns (omitted by default for clean exports).
|
|
199
|
+
line_ending
|
|
200
|
+
Record delimiter mode (``'auto'``, ``'none'``, ``'lf'``, or ``'crlf'``).
|
|
201
|
+
dbcs_encoding
|
|
202
|
+
Character encoding for ``PIC G`` / ``DISPLAY-1`` DBCS fields.
|
|
203
|
+
decimal_comma
|
|
204
|
+
Override copybook ``DECIMAL-POINT IS COMMA`` detection.
|
|
205
|
+
|
|
206
|
+
Returns
|
|
207
|
+
-------
|
|
208
|
+
pathlib.Path
|
|
209
|
+
Path to the written output file.
|
|
210
|
+
"""
|
|
211
|
+
logger.debug("Converting COBOL data {} -> {}", datfile, output)
|
|
212
|
+
resolved = self.layout(
|
|
213
|
+
include_redefines=include_redefines,
|
|
214
|
+
include_filler=include_filler,
|
|
215
|
+
)
|
|
216
|
+
empty_cols = _deduplicate_names([rf.name for rf in resolved])
|
|
217
|
+
chunks = self.yield_data(
|
|
218
|
+
datfile,
|
|
219
|
+
chunksize=chunksize,
|
|
220
|
+
encoding=encoding,
|
|
221
|
+
float_format=float_format,
|
|
222
|
+
byteorder=byteorder,
|
|
223
|
+
include_redefines=include_redefines,
|
|
224
|
+
include_filler=include_filler,
|
|
225
|
+
line_ending=line_ending,
|
|
226
|
+
dbcs_encoding=dbcs_encoding,
|
|
227
|
+
decimal_comma=decimal_comma,
|
|
228
|
+
)
|
|
229
|
+
return export_chunks(chunks, output, empty_columns=empty_cols)
|
|
230
|
+
|
|
231
|
+
def find_record(self, name: str) -> GroupItem | ElementaryItem | None:
|
|
232
|
+
"""Find the first :class:`GroupItem` or :class:`ElementaryItem` matching *name*.
|
|
233
|
+
|
|
234
|
+
Parameters
|
|
235
|
+
----------
|
|
236
|
+
name
|
|
237
|
+
Exact COBOL item name to search for.
|
|
238
|
+
|
|
239
|
+
Returns
|
|
240
|
+
-------
|
|
241
|
+
GroupItem | ElementaryItem | None
|
|
242
|
+
First matching item in depth-first order, or ``None`` if absent.
|
|
243
|
+
"""
|
|
244
|
+
for item in self.records:
|
|
245
|
+
if item.name == name:
|
|
246
|
+
return item
|
|
247
|
+
if isinstance(item, GroupItem):
|
|
248
|
+
found = item.find_record(name)
|
|
249
|
+
if found is not None:
|
|
250
|
+
return found
|
|
251
|
+
return None
|
|
252
|
+
|
|
253
|
+
def layout(
|
|
254
|
+
self,
|
|
255
|
+
include_redefines: bool = False,
|
|
256
|
+
include_filler: bool = True,
|
|
257
|
+
) -> list[ResolvedItem]:
|
|
258
|
+
"""Compute flat byte-offset layout of all elementary items in the copybook.
|
|
259
|
+
|
|
260
|
+
Parameters
|
|
261
|
+
----------
|
|
262
|
+
include_redefines
|
|
263
|
+
Include items defined under ``REDEFINES`` overlays.
|
|
264
|
+
include_filler
|
|
265
|
+
Include ``FILLER`` items in the resolved layout.
|
|
266
|
+
|
|
267
|
+
Returns
|
|
268
|
+
-------
|
|
269
|
+
list[ResolvedItem]
|
|
270
|
+
Ordered elementary items with exact byte offsets and widths.
|
|
271
|
+
"""
|
|
272
|
+
resolved: list[ResolvedItem] = []
|
|
273
|
+
|
|
274
|
+
def _walk(
|
|
275
|
+
nodes: list[GroupItem | ElementaryItem],
|
|
276
|
+
start_offset: int,
|
|
277
|
+
is_redef_branch: bool,
|
|
278
|
+
suffix: str = "",
|
|
279
|
+
) -> int:
|
|
280
|
+
offset = start_offset
|
|
281
|
+
sibling_offsets: dict[str, int] = {}
|
|
282
|
+
|
|
283
|
+
for node in nodes:
|
|
284
|
+
is_redef = is_redef_branch or (node.redefines is not None)
|
|
285
|
+
if node.redefines is not None:
|
|
286
|
+
node_start = sibling_offsets.get(node.redefines, offset)
|
|
287
|
+
if not include_redefines:
|
|
288
|
+
continue
|
|
289
|
+
else:
|
|
290
|
+
node_start = offset
|
|
291
|
+
sibling_offsets[node.name] = node_start
|
|
292
|
+
|
|
293
|
+
curr = node_start
|
|
294
|
+
for occ_idx in range(node.occurs):
|
|
295
|
+
occ_suffix = ""
|
|
296
|
+
if node.occurs > 1 and occ_idx > 0:
|
|
297
|
+
width = 3 if node.occurs >= 100 else 2
|
|
298
|
+
occ_suffix = f"-{occ_idx:0{width}d}"
|
|
299
|
+
|
|
300
|
+
if isinstance(node, ElementaryItem):
|
|
301
|
+
col_name = f"{node.name}{suffix}{occ_suffix}"
|
|
302
|
+
if include_filler or not node.is_filler:
|
|
303
|
+
resolved.append(
|
|
304
|
+
ResolvedItem(
|
|
305
|
+
name=col_name,
|
|
306
|
+
field=node,
|
|
307
|
+
offset=curr,
|
|
308
|
+
byte_length=node.byte_length,
|
|
309
|
+
is_redefines=is_redef,
|
|
310
|
+
)
|
|
311
|
+
)
|
|
312
|
+
curr += node.byte_length
|
|
313
|
+
else:
|
|
314
|
+
curr = _walk(
|
|
315
|
+
node.records,
|
|
316
|
+
curr,
|
|
317
|
+
is_redef_branch=is_redef,
|
|
318
|
+
suffix=f"{suffix}{occ_suffix}",
|
|
319
|
+
)
|
|
320
|
+
|
|
321
|
+
if node.redefines is None:
|
|
322
|
+
offset = curr
|
|
323
|
+
return offset
|
|
324
|
+
|
|
325
|
+
_walk(self.records, 0, is_redef_branch=False)
|
|
326
|
+
return resolved
|
|
327
|
+
|
|
328
|
+
def pic_to_fwf_kwargs(self, convert_compressed: bool = True) -> dict[str, Any]:
|
|
329
|
+
"""Return keyword arguments for :func:`pandas.read_fwf` on text copybooks.
|
|
330
|
+
|
|
331
|
+
Parameters
|
|
332
|
+
----------
|
|
333
|
+
convert_compressed
|
|
334
|
+
Attach byte-level decoders for binary/compressed fields when ``True``;
|
|
335
|
+
keep raw representations when ``False``.
|
|
336
|
+
|
|
337
|
+
Returns
|
|
338
|
+
-------
|
|
339
|
+
dict[str, Any]
|
|
340
|
+
Dictionary containing ``colspecs``, ``names``, and ``converters``
|
|
341
|
+
suitable for unpacking into :func:`pandas.read_fwf`.
|
|
342
|
+
"""
|
|
343
|
+
resolved = self.layout(include_redefines=False, include_filler=True)
|
|
344
|
+
colspecs: list[tuple[int, int]] = []
|
|
345
|
+
names: list[str] = []
|
|
346
|
+
converters: dict[int, Callable[[Any], Any]] = {}
|
|
347
|
+
|
|
348
|
+
for idx, rf in enumerate(resolved):
|
|
349
|
+
names.append(rf.name)
|
|
350
|
+
colspecs.append((rf.offset, rf.offset + rf.byte_length))
|
|
351
|
+
field_kwargs = rf.field.pic_to_fwf_kwargs(convert_compressed=convert_compressed)
|
|
352
|
+
conv = field_kwargs[0].get("converter")
|
|
353
|
+
if conv is not None:
|
|
354
|
+
converters[idx] = conv
|
|
355
|
+
|
|
356
|
+
return {"colspecs": colspecs, "names": names, "converters": converters}
|
|
357
|
+
|
|
358
|
+
def summary(
|
|
359
|
+
self,
|
|
360
|
+
include_redefines: bool = False,
|
|
361
|
+
include_filler: bool = True,
|
|
362
|
+
) -> pd.DataFrame:
|
|
363
|
+
"""Return a tabular overview of every elementary item's byte layout and COBOL type.
|
|
364
|
+
|
|
365
|
+
Parameters
|
|
366
|
+
----------
|
|
367
|
+
include_redefines
|
|
368
|
+
Include ``REDEFINES`` overlay items in the summary table.
|
|
369
|
+
include_filler
|
|
370
|
+
Include ``FILLER`` items in the summary table.
|
|
371
|
+
|
|
372
|
+
Returns
|
|
373
|
+
-------
|
|
374
|
+
pandas.DataFrame
|
|
375
|
+
Table with columns ``name``, ``level``, ``pic``, ``usage``, ``kind``,
|
|
376
|
+
``offset``, ``byte_length``, ``signed``, ``decimals``, and
|
|
377
|
+
``is_redefines``.
|
|
378
|
+
"""
|
|
379
|
+
resolved = self.layout(
|
|
380
|
+
include_redefines=include_redefines,
|
|
381
|
+
include_filler=include_filler,
|
|
382
|
+
)
|
|
383
|
+
return pd.DataFrame(
|
|
384
|
+
[
|
|
385
|
+
{
|
|
386
|
+
"name": rf.name,
|
|
387
|
+
"level": rf.field.level,
|
|
388
|
+
"pic": (rf.field.description.get("pic") or "").strip(),
|
|
389
|
+
"usage": rf.field.usage.value,
|
|
390
|
+
"kind": rf.field.kind.value,
|
|
391
|
+
"offset": rf.offset,
|
|
392
|
+
"byte_length": rf.byte_length,
|
|
393
|
+
"signed": rf.field.signed,
|
|
394
|
+
"decimals": rf.field.decimals,
|
|
395
|
+
"is_redefines": rf.is_redefines,
|
|
396
|
+
}
|
|
397
|
+
for rf in resolved
|
|
398
|
+
]
|
|
399
|
+
)
|
|
400
|
+
|
|
401
|
+
def to_dataframe(
|
|
402
|
+
self,
|
|
403
|
+
datfile: str | pathlib.Path,
|
|
404
|
+
nrows: int | None = None,
|
|
405
|
+
chunksize: int = 500,
|
|
406
|
+
encoding: str | None = None,
|
|
407
|
+
convert_compressed: bool = True,
|
|
408
|
+
drop_converters: bool = False,
|
|
409
|
+
float_format: Literal["ieee", "ibm"] = "ieee",
|
|
410
|
+
byteorder: Literal["big", "little"] = "big",
|
|
411
|
+
as_decimal: bool = False,
|
|
412
|
+
include_redefines: bool = False,
|
|
413
|
+
include_filler: bool = True,
|
|
414
|
+
line_ending: Literal["auto", "none", "lf", "crlf"] = "auto",
|
|
415
|
+
dbcs_encoding: str = "utf-16-be",
|
|
416
|
+
decimal_comma: bool | None = None,
|
|
417
|
+
) -> pd.DataFrame:
|
|
418
|
+
"""Decode *datfile* into a single materialized :class:`pandas.DataFrame`.
|
|
419
|
+
|
|
420
|
+
Reads internally via chunked streaming (:meth:`yield_data`) and stops
|
|
421
|
+
early once *nrows* records have been collected. Aligned with
|
|
422
|
+
:meth:`~dataspector.core.GenericData.to_dataframe`.
|
|
423
|
+
|
|
424
|
+
Notes
|
|
425
|
+
-----
|
|
426
|
+
Because the return value is a single in-memory DataFrame, pass *nrows*
|
|
427
|
+
for quick previews of very large mainframe files, or iterate directly
|
|
428
|
+
over :meth:`yield_data` when streaming files larger than memory.
|
|
429
|
+
|
|
430
|
+
Parameters
|
|
431
|
+
----------
|
|
432
|
+
datfile
|
|
433
|
+
Path to the COBOL data file.
|
|
434
|
+
nrows
|
|
435
|
+
Optional maximum number of records to return (``None`` reads all).
|
|
436
|
+
chunksize
|
|
437
|
+
Number of rows decoded per internal streaming chunk.
|
|
438
|
+
encoding
|
|
439
|
+
Character encoding for ``DISPLAY`` items (auto-detected if ``None``).
|
|
440
|
+
convert_compressed
|
|
441
|
+
Decode binary/compressed items; retain hex representation if ``False``.
|
|
442
|
+
drop_converters
|
|
443
|
+
Keep all items as raw decoded strings / hex without numeric casting.
|
|
444
|
+
float_format
|
|
445
|
+
Floating-point representation for ``COMP-1``/``COMP-2`` (``'ieee'`` or ``'ibm'``).
|
|
446
|
+
byteorder
|
|
447
|
+
Byte order for binary fields (``'big'`` or ``'little'``).
|
|
448
|
+
as_decimal
|
|
449
|
+
Preserve exact precision with :class:`decimal.Decimal` for fixed-point numbers.
|
|
450
|
+
include_redefines
|
|
451
|
+
Include ``REDEFINES`` overlay columns.
|
|
452
|
+
include_filler
|
|
453
|
+
Include ``FILLER`` columns.
|
|
454
|
+
line_ending
|
|
455
|
+
Record delimiter mode (``'auto'``, ``'none'``, ``'lf'``, or ``'crlf'``).
|
|
456
|
+
dbcs_encoding
|
|
457
|
+
Character encoding for ``PIC G`` / ``DISPLAY-1`` DBCS items.
|
|
458
|
+
decimal_comma
|
|
459
|
+
Override copybook ``DECIMAL-POINT IS COMMA`` detection.
|
|
460
|
+
|
|
461
|
+
Returns
|
|
462
|
+
-------
|
|
463
|
+
pandas.DataFrame
|
|
464
|
+
Decoded records (empty DataFrame with copybook columns if no rows).
|
|
465
|
+
"""
|
|
466
|
+
eff_chunk = min(chunksize, nrows) if nrows is not None and nrows > 0 else chunksize
|
|
467
|
+
resolved = self.layout(
|
|
468
|
+
include_redefines=include_redefines,
|
|
469
|
+
include_filler=include_filler,
|
|
470
|
+
)
|
|
471
|
+
empty_cols = _deduplicate_names([rf.name for rf in resolved])
|
|
472
|
+
chunks = self.yield_data(
|
|
473
|
+
datfile,
|
|
474
|
+
chunksize=eff_chunk,
|
|
475
|
+
encoding=encoding,
|
|
476
|
+
convert_compressed=convert_compressed,
|
|
477
|
+
drop_converters=drop_converters,
|
|
478
|
+
float_format=float_format,
|
|
479
|
+
byteorder=byteorder,
|
|
480
|
+
as_decimal=as_decimal,
|
|
481
|
+
include_redefines=include_redefines,
|
|
482
|
+
include_filler=include_filler,
|
|
483
|
+
line_ending=line_ending,
|
|
484
|
+
dbcs_encoding=dbcs_encoding,
|
|
485
|
+
decimal_comma=decimal_comma,
|
|
486
|
+
)
|
|
487
|
+
return chunks_to_dataframe(chunks, nrows=nrows, empty_columns=empty_cols)
|
|
488
|
+
|
|
489
|
+
def to_pandera_schema(
|
|
490
|
+
self,
|
|
491
|
+
include_redefines: bool = False,
|
|
492
|
+
include_filler: bool = True,
|
|
493
|
+
enforce_conditions: bool = False,
|
|
494
|
+
coerce: bool = False,
|
|
495
|
+
) -> pa.DataFrameSchema:
|
|
496
|
+
"""Generate a :class:`pandera.pandas.DataFrameSchema` from this copybook.
|
|
497
|
+
|
|
498
|
+
Parameters
|
|
499
|
+
----------
|
|
500
|
+
include_redefines
|
|
501
|
+
Include ``REDEFINES`` overlay items as schema columns.
|
|
502
|
+
include_filler
|
|
503
|
+
Include ``FILLER`` items as schema columns.
|
|
504
|
+
enforce_conditions
|
|
505
|
+
Add ``isin`` checks derived from level-88 condition-name values.
|
|
506
|
+
coerce
|
|
507
|
+
Whether the returned schema coerces column dtypes on validation.
|
|
508
|
+
|
|
509
|
+
Returns
|
|
510
|
+
-------
|
|
511
|
+
pandera.pandas.DataFrameSchema
|
|
512
|
+
Schema encoding column dtypes, length/range bounds, and optional
|
|
513
|
+
level-88 domain checks.
|
|
514
|
+
"""
|
|
515
|
+
resolved = self.layout(
|
|
516
|
+
include_redefines=include_redefines,
|
|
517
|
+
include_filler=include_filler,
|
|
518
|
+
)
|
|
519
|
+
col_names = _deduplicate_names([rf.name for rf in resolved])
|
|
520
|
+
columns: dict[str, pa.Column] = {}
|
|
521
|
+
|
|
522
|
+
for col_name, rf in zip(col_names, resolved, strict=True):
|
|
523
|
+
field = rf.field
|
|
524
|
+
checks: list[pa.Check] = []
|
|
525
|
+
dtype: Any = None
|
|
526
|
+
nullable = field.kind is DataCategory.NUMERIC_EDITED and not field.blank_when_zero
|
|
527
|
+
|
|
528
|
+
if field.kind in (
|
|
529
|
+
DataCategory.ALPHABETIC,
|
|
530
|
+
DataCategory.ALPHANUMERIC,
|
|
531
|
+
DataCategory.NATIONAL,
|
|
532
|
+
DataCategory.DBCS,
|
|
533
|
+
DataCategory.UTF8,
|
|
534
|
+
DataCategory.BIT,
|
|
535
|
+
):
|
|
536
|
+
dtype = str
|
|
537
|
+
if field.kind is DataCategory.BIT:
|
|
538
|
+
bit_len = max(len(field.expanded_pic), 1)
|
|
539
|
+
checks.append(pa.Check.str_length(min_value=bit_len, max_value=bit_len))
|
|
540
|
+
checks.append(pa.Check.str_matches(r"^[01]+$"))
|
|
541
|
+
else:
|
|
542
|
+
max_chars = (
|
|
543
|
+
len(field.expanded_pic)
|
|
544
|
+
if field.kind in (DataCategory.NATIONAL, DataCategory.DBCS)
|
|
545
|
+
else field.byte_length
|
|
546
|
+
)
|
|
547
|
+
checks.append(pa.Check.str_length(max_value=max_chars))
|
|
548
|
+
elif field.kind in (
|
|
549
|
+
DataCategory.INTEGER,
|
|
550
|
+
DataCategory.BINARY,
|
|
551
|
+
DataCategory.PACKED_DECIMAL,
|
|
552
|
+
DataCategory.PACKED_UNSIGNED,
|
|
553
|
+
):
|
|
554
|
+
dtype = float if field.decimals > 0 else int
|
|
555
|
+
if field.usage in FIXED_BINARY_SIZES:
|
|
556
|
+
bits = FIXED_BINARY_SIZES[field.usage] * 8
|
|
557
|
+
if field.signed:
|
|
558
|
+
checks.append(
|
|
559
|
+
pa.Check.in_range(
|
|
560
|
+
min_value=-(1 << (bits - 1)),
|
|
561
|
+
max_value=(1 << (bits - 1)) - 1,
|
|
562
|
+
)
|
|
563
|
+
)
|
|
564
|
+
else:
|
|
565
|
+
checks.append(pa.Check.in_range(min_value=0, max_value=(1 << bits) - 1))
|
|
566
|
+
elif field.usage is Usage.COMP_X:
|
|
567
|
+
bits = field.byte_length * 8
|
|
568
|
+
checks.append(pa.Check.in_range(min_value=0, max_value=(1 << bits) - 1))
|
|
569
|
+
elif field.digits > 0 and field.usage is not Usage.COMP_5:
|
|
570
|
+
max_abs = (10**field.digits - 1) * (10**field.exponent)
|
|
571
|
+
min_val = -max_abs if field.signed else 0
|
|
572
|
+
checks.append(pa.Check.in_range(min_value=min_val, max_value=max_abs))
|
|
573
|
+
elif field.kind in (
|
|
574
|
+
DataCategory.DECIMAL,
|
|
575
|
+
DataCategory.FLOAT,
|
|
576
|
+
DataCategory.EXTERNAL_FLOAT,
|
|
577
|
+
DataCategory.NUMERIC_EDITED,
|
|
578
|
+
):
|
|
579
|
+
dtype = float
|
|
580
|
+
if field.kind is DataCategory.DECIMAL and field.digits > 0:
|
|
581
|
+
max_abs = float((10**field.digits - 1) * (10**field.exponent))
|
|
582
|
+
min_val = -max_abs if field.signed else 0.0
|
|
583
|
+
checks.append(pa.Check.in_range(min_value=min_val, max_value=max_abs))
|
|
584
|
+
elif field.kind in (DataCategory.POINTER, DataCategory.INDEX):
|
|
585
|
+
dtype = int
|
|
586
|
+
if field.kind is DataCategory.POINTER:
|
|
587
|
+
checks.append(pa.Check.greater_than_or_equal_to(0))
|
|
588
|
+
|
|
589
|
+
if enforce_conditions and field.conditions:
|
|
590
|
+
allowed = [v for cond in field.conditions for v in cond.values]
|
|
591
|
+
if allowed:
|
|
592
|
+
checks.append(pa.Check.isin(allowed))
|
|
593
|
+
|
|
594
|
+
columns[col_name] = pa.Column(dtype, checks=checks, nullable=nullable)
|
|
595
|
+
|
|
596
|
+
return pa.DataFrameSchema(columns=columns, coerce=coerce)
|
|
597
|
+
|
|
598
|
+
def yield_data(
|
|
599
|
+
self,
|
|
600
|
+
datfile: str | pathlib.Path,
|
|
601
|
+
chunksize: int = 500,
|
|
602
|
+
encoding: str | None = None,
|
|
603
|
+
convert_compressed: bool = True,
|
|
604
|
+
drop_converters: bool = False,
|
|
605
|
+
float_format: Literal["ieee", "ibm"] = "ieee",
|
|
606
|
+
byteorder: Literal["big", "little"] = "big",
|
|
607
|
+
as_decimal: bool = False,
|
|
608
|
+
include_redefines: bool = False,
|
|
609
|
+
include_filler: bool = True,
|
|
610
|
+
line_ending: Literal["auto", "none", "lf", "crlf"] = "auto",
|
|
611
|
+
dbcs_encoding: str = "utf-16-be",
|
|
612
|
+
decimal_comma: bool | None = None,
|
|
613
|
+
) -> Iterator[pd.DataFrame]:
|
|
614
|
+
"""Stream records from *datfile* as :class:`pandas.DataFrame` chunks.
|
|
615
|
+
|
|
616
|
+
Parameters
|
|
617
|
+
----------
|
|
618
|
+
datfile
|
|
619
|
+
Path to the raw COBOL data file.
|
|
620
|
+
chunksize
|
|
621
|
+
Maximum number of decoded records per yielded chunk.
|
|
622
|
+
encoding
|
|
623
|
+
Character encoding for ``DISPLAY`` items (auto-detected if ``None``).
|
|
624
|
+
convert_compressed
|
|
625
|
+
Decode binary/compressed items; retain hex representation if ``False``.
|
|
626
|
+
drop_converters
|
|
627
|
+
Keep all items as raw decoded strings / hex without numeric casting.
|
|
628
|
+
float_format
|
|
629
|
+
Floating-point representation for ``COMP-1``/``COMP-2`` (``'ieee'`` or ``'ibm'``).
|
|
630
|
+
byteorder
|
|
631
|
+
Byte order for binary fields (``'big'`` or ``'little'``).
|
|
632
|
+
as_decimal
|
|
633
|
+
Preserve exact precision with :class:`decimal.Decimal` for fixed-point numbers.
|
|
634
|
+
include_redefines
|
|
635
|
+
Include ``REDEFINES`` overlay columns.
|
|
636
|
+
include_filler
|
|
637
|
+
Include ``FILLER`` columns.
|
|
638
|
+
line_ending
|
|
639
|
+
Record delimiter mode (``'auto'``, ``'none'``, ``'lf'``, or ``'crlf'``).
|
|
640
|
+
dbcs_encoding
|
|
641
|
+
Character encoding for ``PIC G`` / ``DISPLAY-1`` DBCS items.
|
|
642
|
+
decimal_comma
|
|
643
|
+
Override copybook ``DECIMAL-POINT IS COMMA`` detection.
|
|
644
|
+
|
|
645
|
+
Yields
|
|
646
|
+
------
|
|
647
|
+
pandas.DataFrame
|
|
648
|
+
Decoded chunk of up to *chunksize* records.
|
|
649
|
+
"""
|
|
650
|
+
if drop_converters and convert_compressed:
|
|
651
|
+
logger.debug("drop_converters=True: disabling compressed field conversion.")
|
|
652
|
+
convert_compressed = False
|
|
653
|
+
|
|
654
|
+
datpath = pathlib.Path(datfile)
|
|
655
|
+
resolved = self.layout(
|
|
656
|
+
include_redefines=include_redefines,
|
|
657
|
+
include_filler=include_filler,
|
|
658
|
+
)
|
|
659
|
+
rec_len = self.record_length
|
|
660
|
+
if not resolved or rec_len == 0:
|
|
661
|
+
return
|
|
662
|
+
|
|
663
|
+
resolved_encoding: str = (
|
|
664
|
+
encoding
|
|
665
|
+
if encoding is not None
|
|
666
|
+
else ("cp500" if self.has_binary_fields else predict_encoding(datpath))
|
|
667
|
+
)
|
|
668
|
+
col_names = _deduplicate_names([rf.name for rf in resolved])
|
|
669
|
+
detected_ending = self._detect_line_ending(datpath, rec_len, line_ending)
|
|
670
|
+
pad_byte = b"\x40" if is_ebcdic(resolved_encoding) else b" "
|
|
671
|
+
resolved_decimal_comma = self.decimal_comma if decimal_comma is None else decimal_comma
|
|
672
|
+
logger.trace(
|
|
673
|
+
"Streaming {} (rec_len={}, encoding={}, line_ending={})",
|
|
674
|
+
datpath,
|
|
675
|
+
rec_len,
|
|
676
|
+
resolved_encoding,
|
|
677
|
+
detected_ending,
|
|
678
|
+
)
|
|
679
|
+
|
|
680
|
+
def _parse_record(raw_rec: bytes) -> list[Any]:
|
|
681
|
+
if len(raw_rec) < rec_len:
|
|
682
|
+
raw_rec = raw_rec + pad_byte * (rec_len - len(raw_rec))
|
|
683
|
+
return [
|
|
684
|
+
rf.field.decode_bytes(
|
|
685
|
+
raw_rec[rf.offset : rf.offset + rf.byte_length],
|
|
686
|
+
encoding=resolved_encoding,
|
|
687
|
+
float_format=float_format,
|
|
688
|
+
byteorder=byteorder,
|
|
689
|
+
as_decimal=as_decimal,
|
|
690
|
+
convert_compressed=convert_compressed,
|
|
691
|
+
drop_converters=drop_converters,
|
|
692
|
+
dbcs_encoding=dbcs_encoding,
|
|
693
|
+
decimal_comma=resolved_decimal_comma,
|
|
694
|
+
)
|
|
695
|
+
for rf in resolved
|
|
696
|
+
]
|
|
697
|
+
|
|
698
|
+
rows: list[list[Any]] = []
|
|
699
|
+
with datpath.open("rb") as fh:
|
|
700
|
+
if detected_ending == "none":
|
|
701
|
+
while True:
|
|
702
|
+
block = fh.read(rec_len)
|
|
703
|
+
if not block:
|
|
704
|
+
break
|
|
705
|
+
if len(block) < rec_len:
|
|
706
|
+
logger.warning(
|
|
707
|
+
"Truncated trailing COBOL record: expected {} bytes, got {}.",
|
|
708
|
+
rec_len,
|
|
709
|
+
len(block),
|
|
710
|
+
)
|
|
711
|
+
rows.append(_parse_record(block))
|
|
712
|
+
if len(rows) >= chunksize:
|
|
713
|
+
yield pd.DataFrame(rows, columns=col_names)
|
|
714
|
+
rows = []
|
|
715
|
+
else:
|
|
716
|
+
for line in fh:
|
|
717
|
+
stripped = line.rstrip(b"\r\n")
|
|
718
|
+
if not stripped:
|
|
719
|
+
continue
|
|
720
|
+
rows.append(_parse_record(stripped))
|
|
721
|
+
if len(rows) >= chunksize:
|
|
722
|
+
yield pd.DataFrame(rows, columns=col_names)
|
|
723
|
+
rows = []
|
|
724
|
+
|
|
725
|
+
if rows:
|
|
726
|
+
yield pd.DataFrame(rows, columns=col_names)
|
|
727
|
+
|
|
728
|
+
def _detect_line_ending(
|
|
729
|
+
self,
|
|
730
|
+
datpath: pathlib.Path,
|
|
731
|
+
rec_len: int,
|
|
732
|
+
line_ending: Literal["auto", "none", "lf", "crlf"],
|
|
733
|
+
) -> Literal["none", "lf", "crlf"]:
|
|
734
|
+
"""Determine whether *datpath* uses fixed binary records or line terminators."""
|
|
735
|
+
if line_ending != "auto":
|
|
736
|
+
return line_ending
|
|
737
|
+
if rec_len <= 0:
|
|
738
|
+
return "lf"
|
|
739
|
+
|
|
740
|
+
file_size = datpath.stat().st_size
|
|
741
|
+
with datpath.open("rb") as fh:
|
|
742
|
+
probe = fh.read(rec_len + 2)
|
|
743
|
+
|
|
744
|
+
if len(probe) >= rec_len + 2 and probe[rec_len : rec_len + 2] == b"\r\n":
|
|
745
|
+
return "crlf"
|
|
746
|
+
if len(probe) >= rec_len + 1 and probe[rec_len : rec_len + 1] == b"\n":
|
|
747
|
+
return "lf"
|
|
748
|
+
|
|
749
|
+
if self.has_binary_fields:
|
|
750
|
+
if file_size % rec_len == 0:
|
|
751
|
+
return "none"
|
|
752
|
+
if file_size % (rec_len + 1) == 0:
|
|
753
|
+
return "lf"
|
|
754
|
+
if file_size % (rec_len + 2) == 0:
|
|
755
|
+
return "crlf"
|
|
756
|
+
return "none"
|
|
757
|
+
|
|
758
|
+
if b"\r\n" in probe:
|
|
759
|
+
return "crlf"
|
|
760
|
+
if b"\n" in probe:
|
|
761
|
+
return "lf"
|
|
762
|
+
return "none"
|
|
763
|
+
|
|
764
|
+
|
|
765
|
+
class GroupItem:
|
|
766
|
+
"""Hierarchical COBOL group item containing subordinate items."""
|
|
767
|
+
|
|
768
|
+
def __init__(self, description: dict[str, Any]) -> None:
|
|
769
|
+
self.description: dict[str, Any] = description
|
|
770
|
+
self.redefinedby: list[str] = []
|
|
771
|
+
self.records: list[GroupItem | ElementaryItem] = []
|
|
772
|
+
self.conditions: list[ConditionName] = []
|
|
773
|
+
raw_usage = (description.get("usage") or description.get("comp") or "").upper()
|
|
774
|
+
self._usage: Usage | None = USAGE_ALIASES.get(raw_usage) if raw_usage else None
|
|
775
|
+
|
|
776
|
+
def __repr__(self) -> str:
|
|
777
|
+
suffix = f", {self.occurs} OCCURS" if self.occurs > 1 else ""
|
|
778
|
+
return (
|
|
779
|
+
f"<Group (Level {self.description.get('level')}, {self.name}, "
|
|
780
|
+
f"{len(self.records)} subrecords{suffix}>"
|
|
781
|
+
)
|
|
782
|
+
|
|
783
|
+
@property
|
|
784
|
+
def byte_length(self) -> int:
|
|
785
|
+
"""Physical byte width of a single occurrence of this group item (excluding redefines)."""
|
|
786
|
+
return sum(rec.byte_length * rec.occurs for rec in self.records if rec.redefines is None)
|
|
787
|
+
|
|
788
|
+
@property
|
|
789
|
+
def level(self) -> int:
|
|
790
|
+
"""COBOL level number."""
|
|
791
|
+
return int(self.description["level"])
|
|
792
|
+
|
|
793
|
+
@property
|
|
794
|
+
def name(self) -> str:
|
|
795
|
+
"""Group item name."""
|
|
796
|
+
return str(self.description.get("name") or "FILLER")
|
|
797
|
+
|
|
798
|
+
@property
|
|
799
|
+
def occurs(self) -> int:
|
|
800
|
+
"""Repetition count from ``OCCURS`` (``1`` when non-repeating)."""
|
|
801
|
+
occ = self.description.get("occurs")
|
|
802
|
+
return int(occ) if occ is not None else 1
|
|
803
|
+
|
|
804
|
+
@property
|
|
805
|
+
def redefines(self) -> str | None:
|
|
806
|
+
"""Name of the sibling item this group redefines, or ``None``."""
|
|
807
|
+
val = self.description.get("redefines")
|
|
808
|
+
return str(val) if val else None
|
|
809
|
+
|
|
810
|
+
@property
|
|
811
|
+
def usage(self) -> Usage | None:
|
|
812
|
+
"""Optional group-level ``USAGE`` inherited by subordinate elementary items."""
|
|
813
|
+
return self._usage
|
|
814
|
+
|
|
815
|
+
def append(self, level: int, record: GroupItem | ElementaryItem) -> None:
|
|
816
|
+
"""Insert *record* at the deepest nested subgroup whose level is ``< level``.
|
|
817
|
+
|
|
818
|
+
Parameters
|
|
819
|
+
----------
|
|
820
|
+
level
|
|
821
|
+
COBOL level number of the item being inserted.
|
|
822
|
+
record
|
|
823
|
+
Group or elementary item to append.
|
|
824
|
+
"""
|
|
825
|
+
for rec in reversed(self.records):
|
|
826
|
+
if isinstance(rec, GroupItem) and rec.level < level:
|
|
827
|
+
rec.append(level, record)
|
|
828
|
+
return
|
|
829
|
+
self.records.append(record)
|
|
830
|
+
|
|
831
|
+
def find_record(self, name: str) -> GroupItem | ElementaryItem | None:
|
|
832
|
+
"""Depth-first search for the first subordinate item named *name*.
|
|
833
|
+
|
|
834
|
+
Parameters
|
|
835
|
+
----------
|
|
836
|
+
name
|
|
837
|
+
Exact COBOL item name to search for.
|
|
838
|
+
|
|
839
|
+
Returns
|
|
840
|
+
-------
|
|
841
|
+
GroupItem | ElementaryItem | None
|
|
842
|
+
First matching subordinate item, or ``None`` if not found.
|
|
843
|
+
"""
|
|
844
|
+
for gf in self.records:
|
|
845
|
+
if gf.name == name:
|
|
846
|
+
return gf
|
|
847
|
+
if isinstance(gf, GroupItem):
|
|
848
|
+
found = gf.find_record(name)
|
|
849
|
+
if found is not None:
|
|
850
|
+
return found
|
|
851
|
+
return None
|
|
852
|
+
|
|
853
|
+
def pic_to_fwf_kwargs(self, convert_compressed: bool = True) -> list[dict[str, Any]]:
|
|
854
|
+
"""Return flattened :func:`pandas.read_fwf` column descriptors for this group item.
|
|
855
|
+
|
|
856
|
+
Parameters
|
|
857
|
+
----------
|
|
858
|
+
convert_compressed
|
|
859
|
+
Attach byte-level decoders for binary/compressed items when ``True``.
|
|
860
|
+
|
|
861
|
+
Returns
|
|
862
|
+
-------
|
|
863
|
+
list[dict[str, Any]]
|
|
864
|
+
Column descriptor dictionaries expanded across any ``OCCURS`` repetitions.
|
|
865
|
+
"""
|
|
866
|
+
dtypes: list[dict[str, Any]] = []
|
|
867
|
+
for rec in self.records:
|
|
868
|
+
if rec.redefines is not None:
|
|
869
|
+
continue
|
|
870
|
+
dtypes.extend(rec.pic_to_fwf_kwargs(convert_compressed=convert_compressed))
|
|
871
|
+
|
|
872
|
+
if self.occurs == 1:
|
|
873
|
+
return dtypes
|
|
874
|
+
|
|
875
|
+
expanded: list[dict[str, Any]] = []
|
|
876
|
+
for n in range(self.occurs):
|
|
877
|
+
for dtype in dtypes:
|
|
878
|
+
entry = copy.deepcopy(dtype)
|
|
879
|
+
entry["name"] = _occurs_name(entry["name"], n, self.occurs)
|
|
880
|
+
expanded.append(entry)
|
|
881
|
+
return expanded
|
|
882
|
+
|
|
883
|
+
|
|
884
|
+
class ElementaryItem:
|
|
885
|
+
"""COBOL elementary data item carrying a ``PICTURE`` or self-sized ``USAGE``."""
|
|
886
|
+
|
|
887
|
+
def __init__(self, description: dict[str, Any]) -> None:
|
|
888
|
+
self.description: dict[str, Any] = description
|
|
889
|
+
self.redefinedby: list[str] = []
|
|
890
|
+
self.conditions: list[ConditionName] = []
|
|
891
|
+
|
|
892
|
+
raw_usage = (description.get("usage") or description.get("comp") or "DISPLAY").upper()
|
|
893
|
+
self._usage: Usage = USAGE_ALIASES.get(raw_usage, Usage.DISPLAY)
|
|
894
|
+
|
|
895
|
+
raw_pic = (description.get("pic") or "").strip()
|
|
896
|
+
self._expanded_pic: str = expand_pic(raw_pic) if raw_pic else ""
|
|
897
|
+
if self._usage is Usage.DISPLAY and self._expanded_pic:
|
|
898
|
+
if PIC_NATIONAL_ONLY.match(self._expanded_pic):
|
|
899
|
+
self._usage = Usage.NATIONAL
|
|
900
|
+
elif PIC_UTF8_ONLY.match(self._expanded_pic):
|
|
901
|
+
self._usage = Usage.UTF_8
|
|
902
|
+
elif PIC_DBCS_ONLY.match(self._expanded_pic):
|
|
903
|
+
self._usage = Usage.DISPLAY_1
|
|
904
|
+
elif PIC_BIT_ONLY.match(self._expanded_pic):
|
|
905
|
+
self._usage = Usage.BIT
|
|
906
|
+
|
|
907
|
+
raw_bl = description.get("byte_length")
|
|
908
|
+
self._explicit_byte_length: int | None = int(raw_bl) if raw_bl is not None else None
|
|
909
|
+
|
|
910
|
+
bin_qual = (description.get("binary_sign_qualifier") or "").upper()
|
|
911
|
+
if self._usage in FIXED_BINARY_SIZES:
|
|
912
|
+
self._signed = bin_qual != "UNSIGNED"
|
|
913
|
+
elif self._usage in (Usage.COMP_6, Usage.COMP_X):
|
|
914
|
+
self._signed = False
|
|
915
|
+
else:
|
|
916
|
+
self._signed = self._expanded_pic.startswith("S")
|
|
917
|
+
self._sign_mode: SignMode = self._resolve_sign_mode(description)
|
|
918
|
+
self._exponent: int = (
|
|
919
|
+
compute_decimal_exponent(self._expanded_pic)
|
|
920
|
+
if self._expanded_pic and PIC_NUMERIC_PURE.match(self._expanded_pic)
|
|
921
|
+
else 0
|
|
922
|
+
)
|
|
923
|
+
|
|
924
|
+
@classmethod
|
|
925
|
+
def from_clause(cls, statement: str) -> ElementaryItem:
|
|
926
|
+
"""Construct an :class:`ElementaryItem` from a single COBOL declaration line.
|
|
927
|
+
|
|
928
|
+
Parameters
|
|
929
|
+
----------
|
|
930
|
+
statement
|
|
931
|
+
Single COBOL data description clause (with or without trailing period).
|
|
932
|
+
|
|
933
|
+
Returns
|
|
934
|
+
-------
|
|
935
|
+
ElementaryItem
|
|
936
|
+
Parsed elementary item.
|
|
937
|
+
"""
|
|
938
|
+
cleaned = statement.strip().rstrip(".")
|
|
939
|
+
desc = parse_statement(cleaned)
|
|
940
|
+
if desc is None:
|
|
941
|
+
raise ValueError(f"Could not parse COBOL elementary item: {statement!r}")
|
|
942
|
+
return cls(desc)
|
|
943
|
+
|
|
944
|
+
def __repr__(self) -> str:
|
|
945
|
+
pic_display = (self.description.get("pic") or "").strip()
|
|
946
|
+
detail = pic_display or self._usage.value
|
|
947
|
+
if pic_display and self._usage is not Usage.DISPLAY:
|
|
948
|
+
detail = f"{pic_display} {self._usage.value}"
|
|
949
|
+
suffix = f", {self.occurs} OCCURS" if self.occurs > 1 else ""
|
|
950
|
+
return f"<Field (Level {self.description.get('level')}, {self.name}, {detail}{suffix}>"
|
|
951
|
+
|
|
952
|
+
@property
|
|
953
|
+
def blank_when_zero(self) -> bool:
|
|
954
|
+
"""``True`` if ``BLANK WHEN ZERO`` was declared."""
|
|
955
|
+
return bool(self.description.get("blank_when_zero", False))
|
|
956
|
+
|
|
957
|
+
@property
|
|
958
|
+
def byte_length(self) -> int:
|
|
959
|
+
"""Physical byte width of a single occurrence of this elementary item."""
|
|
960
|
+
if self._explicit_byte_length is not None:
|
|
961
|
+
return self._explicit_byte_length
|
|
962
|
+
if self._usage in FIXED_BINARY_SIZES:
|
|
963
|
+
return FIXED_BINARY_SIZES[self._usage]
|
|
964
|
+
if self._usage in (Usage.COMP_1, Usage.FLOAT_SHORT, Usage.INDEX, Usage.POINTER):
|
|
965
|
+
return 4
|
|
966
|
+
if self._usage in (
|
|
967
|
+
Usage.COMP_2,
|
|
968
|
+
Usage.FLOAT_LONG,
|
|
969
|
+
Usage.PROCEDURE_POINTER,
|
|
970
|
+
Usage.FUNCTION_POINTER,
|
|
971
|
+
):
|
|
972
|
+
return 8
|
|
973
|
+
if self._usage is Usage.COMP_3:
|
|
974
|
+
return (self.digits // 2) + 1
|
|
975
|
+
if self._usage is Usage.COMP_6:
|
|
976
|
+
return (self.digits + 1) // 2
|
|
977
|
+
if self._usage is Usage.COMP_X:
|
|
978
|
+
if "X" in self._expanded_pic:
|
|
979
|
+
return self._expanded_pic.count("X")
|
|
980
|
+
d = max(self.digits, 1)
|
|
981
|
+
return ((10**d - 1).bit_length() + 7) // 8
|
|
982
|
+
if self._usage in (Usage.COMP, Usage.COMP_5):
|
|
983
|
+
d = self.digits
|
|
984
|
+
if d <= 4:
|
|
985
|
+
return 2
|
|
986
|
+
if d <= 9:
|
|
987
|
+
return 4
|
|
988
|
+
if d <= 18:
|
|
989
|
+
return 8
|
|
990
|
+
return 16
|
|
991
|
+
if self._usage in (Usage.NATIONAL, Usage.DISPLAY_1):
|
|
992
|
+
return len(self._expanded_pic) * 2
|
|
993
|
+
if self._usage is Usage.UTF_8:
|
|
994
|
+
return len(self._expanded_pic) * 4
|
|
995
|
+
if self._usage is Usage.BIT:
|
|
996
|
+
return (max(len(self._expanded_pic), 1) + 7) // 8
|
|
997
|
+
|
|
998
|
+
physical = self._expanded_pic.replace("S", "").replace("V", "").replace("P", "")
|
|
999
|
+
extra = (
|
|
1000
|
+
1 if self._sign_mode in (SignMode.LEADING_SEPARATE, SignMode.TRAILING_SEPARATE) else 0
|
|
1001
|
+
)
|
|
1002
|
+
return len(physical) + extra
|
|
1003
|
+
|
|
1004
|
+
@property
|
|
1005
|
+
def decimals(self) -> int:
|
|
1006
|
+
"""Number of implied fractional decimal digits."""
|
|
1007
|
+
return max(0, -self._exponent)
|
|
1008
|
+
|
|
1009
|
+
@property
|
|
1010
|
+
def digits(self) -> int:
|
|
1011
|
+
"""Total count of ``9`` digit positions in the ``PIC`` clause."""
|
|
1012
|
+
return self._expanded_pic.count("9")
|
|
1013
|
+
|
|
1014
|
+
@property
|
|
1015
|
+
def expanded_pic(self) -> str:
|
|
1016
|
+
"""Fully expanded uppercase ``PIC`` string."""
|
|
1017
|
+
return self._expanded_pic
|
|
1018
|
+
|
|
1019
|
+
@property
|
|
1020
|
+
def exponent(self) -> int:
|
|
1021
|
+
"""Base-10 scaling exponent from ``V`` and ``P`` symbols."""
|
|
1022
|
+
return self._exponent
|
|
1023
|
+
|
|
1024
|
+
@property
|
|
1025
|
+
def is_binary(self) -> bool:
|
|
1026
|
+
"""``True`` if the item uses binary, packed, float, pointer, index, or wide storage."""
|
|
1027
|
+
return self._usage in BINARY_USAGES
|
|
1028
|
+
|
|
1029
|
+
@property
|
|
1030
|
+
def is_filler(self) -> bool:
|
|
1031
|
+
"""``True`` if this item is a ``FILLER`` item."""
|
|
1032
|
+
return self.name.upper() == "FILLER"
|
|
1033
|
+
|
|
1034
|
+
@property
|
|
1035
|
+
def kind(self) -> DataCategory:
|
|
1036
|
+
"""COBOL :class:`~dataspector.coboller.constants.DataCategory` of this elementary item."""
|
|
1037
|
+
if self._usage in (Usage.COMP_1, Usage.COMP_2, Usage.FLOAT_SHORT, Usage.FLOAT_LONG):
|
|
1038
|
+
return DataCategory.FLOAT
|
|
1039
|
+
if self._usage in (Usage.POINTER, Usage.PROCEDURE_POINTER, Usage.FUNCTION_POINTER):
|
|
1040
|
+
return DataCategory.POINTER
|
|
1041
|
+
if self._usage is Usage.INDEX:
|
|
1042
|
+
return DataCategory.INDEX
|
|
1043
|
+
if self._usage is Usage.COMP_3:
|
|
1044
|
+
return DataCategory.PACKED_DECIMAL
|
|
1045
|
+
if self._usage is Usage.COMP_6:
|
|
1046
|
+
return DataCategory.PACKED_UNSIGNED
|
|
1047
|
+
if self._usage in (Usage.COMP, Usage.COMP_5, Usage.COMP_X) or (
|
|
1048
|
+
self._usage in FIXED_BINARY_SIZES
|
|
1049
|
+
):
|
|
1050
|
+
return DataCategory.BINARY
|
|
1051
|
+
if self._usage is Usage.NATIONAL:
|
|
1052
|
+
return DataCategory.NATIONAL
|
|
1053
|
+
if self._usage is Usage.UTF_8:
|
|
1054
|
+
return DataCategory.UTF8
|
|
1055
|
+
if self._usage is Usage.DISPLAY_1:
|
|
1056
|
+
return DataCategory.DBCS
|
|
1057
|
+
if self._usage is Usage.BIT:
|
|
1058
|
+
return DataCategory.BIT
|
|
1059
|
+
|
|
1060
|
+
pic = self._expanded_pic
|
|
1061
|
+
if not pic:
|
|
1062
|
+
return DataCategory.ALPHANUMERIC
|
|
1063
|
+
if PIC_EXTERNAL_FLOAT.match(pic):
|
|
1064
|
+
return DataCategory.EXTERNAL_FLOAT
|
|
1065
|
+
if PIC_NUMERIC_PURE.match(pic):
|
|
1066
|
+
return DataCategory.DECIMAL if self._exponent < 0 else DataCategory.INTEGER
|
|
1067
|
+
if PIC_ALPHA_ONLY.match(pic):
|
|
1068
|
+
return DataCategory.ALPHABETIC
|
|
1069
|
+
if "X" not in pic and "A" not in pic and PIC_EDITED_MARKERS.search(pic):
|
|
1070
|
+
return DataCategory.NUMERIC_EDITED
|
|
1071
|
+
return DataCategory.ALPHANUMERIC
|
|
1072
|
+
|
|
1073
|
+
@property
|
|
1074
|
+
def level(self) -> int:
|
|
1075
|
+
"""COBOL level number."""
|
|
1076
|
+
return int(self.description["level"])
|
|
1077
|
+
|
|
1078
|
+
@property
|
|
1079
|
+
def name(self) -> str:
|
|
1080
|
+
"""Item name (``'FILLER'`` if unnamed)."""
|
|
1081
|
+
return str(self.description.get("name") or "FILLER")
|
|
1082
|
+
|
|
1083
|
+
@property
|
|
1084
|
+
def occurs(self) -> int:
|
|
1085
|
+
"""Repetition count from ``OCCURS`` (``1`` when non-repeating)."""
|
|
1086
|
+
occ = self.description.get("occurs")
|
|
1087
|
+
return int(occ) if occ is not None else 1
|
|
1088
|
+
|
|
1089
|
+
@property
|
|
1090
|
+
def redefines(self) -> str | None:
|
|
1091
|
+
"""Name of the sibling item this item redefines, or ``None``."""
|
|
1092
|
+
val = self.description.get("redefines")
|
|
1093
|
+
return str(val) if val else None
|
|
1094
|
+
|
|
1095
|
+
@property
|
|
1096
|
+
def sign_mode(self) -> SignMode:
|
|
1097
|
+
"""Sign encoding mode for ``USAGE DISPLAY`` signed numeric items."""
|
|
1098
|
+
return self._sign_mode
|
|
1099
|
+
|
|
1100
|
+
@property
|
|
1101
|
+
def signed(self) -> bool:
|
|
1102
|
+
"""``True`` if this elementary item is signed."""
|
|
1103
|
+
if self._usage in (
|
|
1104
|
+
Usage.COMP_1,
|
|
1105
|
+
Usage.COMP_2,
|
|
1106
|
+
Usage.FLOAT_SHORT,
|
|
1107
|
+
Usage.FLOAT_LONG,
|
|
1108
|
+
Usage.INDEX,
|
|
1109
|
+
):
|
|
1110
|
+
return True
|
|
1111
|
+
return self._signed
|
|
1112
|
+
|
|
1113
|
+
@property
|
|
1114
|
+
def usage(self) -> Usage:
|
|
1115
|
+
"""Normalized :class:`~dataspector.coboller.constants.Usage` storage representation."""
|
|
1116
|
+
return self._usage
|
|
1117
|
+
|
|
1118
|
+
def decode_bytes(
|
|
1119
|
+
self,
|
|
1120
|
+
raw: bytes,
|
|
1121
|
+
encoding: str = "ascii",
|
|
1122
|
+
float_format: Literal["ieee", "ibm"] = "ieee",
|
|
1123
|
+
byteorder: Literal["big", "little"] = "big",
|
|
1124
|
+
as_decimal: bool = False,
|
|
1125
|
+
convert_compressed: bool = True,
|
|
1126
|
+
drop_converters: bool = False,
|
|
1127
|
+
dbcs_encoding: str = "utf-16-be",
|
|
1128
|
+
decimal_comma: bool = False,
|
|
1129
|
+
) -> Any:
|
|
1130
|
+
"""Decode *raw* bytes for this elementary item into a Python value.
|
|
1131
|
+
|
|
1132
|
+
Parameters
|
|
1133
|
+
----------
|
|
1134
|
+
raw
|
|
1135
|
+
Byte slice corresponding to this elementary item.
|
|
1136
|
+
encoding
|
|
1137
|
+
Character encoding for ``DISPLAY`` items (e.g. ``'ascii'``, ``'cp500'``).
|
|
1138
|
+
float_format
|
|
1139
|
+
Floating-point representation for ``COMP-1``/``COMP-2`` (``'ieee'`` or ``'ibm'``).
|
|
1140
|
+
byteorder
|
|
1141
|
+
Byte order for binary and UTF-16 items (``'big'`` or ``'little'``).
|
|
1142
|
+
as_decimal
|
|
1143
|
+
Return :class:`decimal.Decimal` instead of ``float``/``int`` for fixed-point numbers.
|
|
1144
|
+
convert_compressed
|
|
1145
|
+
Decode binary/packed items when ``True``; return hex strings when ``False``.
|
|
1146
|
+
drop_converters
|
|
1147
|
+
Return raw decoded text or hex without numeric conversion.
|
|
1148
|
+
dbcs_encoding
|
|
1149
|
+
Character encoding for ``PIC G`` / ``DISPLAY-1`` DBCS items.
|
|
1150
|
+
decimal_comma
|
|
1151
|
+
Interpret commas as decimal separators per ``DECIMAL-POINT IS COMMA``.
|
|
1152
|
+
|
|
1153
|
+
Returns
|
|
1154
|
+
-------
|
|
1155
|
+
Any
|
|
1156
|
+
Decoded Python scalar (``str``, ``int``, ``float``, ``Decimal``, or ``None``).
|
|
1157
|
+
"""
|
|
1158
|
+
if self.is_binary:
|
|
1159
|
+
if drop_converters or not convert_compressed:
|
|
1160
|
+
if self._usage is Usage.NATIONAL:
|
|
1161
|
+
return raw.decode("utf-16-be" if byteorder == "big" else "utf-16-le").rstrip()
|
|
1162
|
+
if self._usage is Usage.UTF_8:
|
|
1163
|
+
return raw.decode("utf-8", errors="replace").rstrip()
|
|
1164
|
+
if self._usage is Usage.DISPLAY_1:
|
|
1165
|
+
return raw.decode(dbcs_encoding, errors="replace").rstrip(" \u3000")
|
|
1166
|
+
return raw.hex()
|
|
1167
|
+
if self._usage is Usage.COMP_3:
|
|
1168
|
+
return comp3_unpack(raw, exponent=self._exponent, as_decimal=as_decimal)
|
|
1169
|
+
if self._usage is Usage.COMP_6:
|
|
1170
|
+
return comp6_unpack(raw, exponent=self._exponent, as_decimal=as_decimal)
|
|
1171
|
+
if self._usage in (Usage.COMP, Usage.COMP_5, Usage.COMP_X) or (
|
|
1172
|
+
self._usage in FIXED_BINARY_SIZES
|
|
1173
|
+
):
|
|
1174
|
+
return comp_unpack(
|
|
1175
|
+
raw,
|
|
1176
|
+
signed=self._signed,
|
|
1177
|
+
byteorder=byteorder,
|
|
1178
|
+
exponent=self._exponent,
|
|
1179
|
+
as_decimal=as_decimal,
|
|
1180
|
+
)
|
|
1181
|
+
if self._usage in (Usage.COMP_1, Usage.COMP_2):
|
|
1182
|
+
return comp_float_unpack(raw, float_format=float_format, byteorder=byteorder)
|
|
1183
|
+
if self._usage in (Usage.FLOAT_SHORT, Usage.FLOAT_LONG):
|
|
1184
|
+
return comp_float_unpack(raw, float_format="ieee", byteorder=byteorder)
|
|
1185
|
+
if self._usage in (Usage.POINTER, Usage.PROCEDURE_POINTER, Usage.FUNCTION_POINTER):
|
|
1186
|
+
return int.from_bytes(raw, byteorder=byteorder, signed=False)
|
|
1187
|
+
if self._usage is Usage.INDEX:
|
|
1188
|
+
return int.from_bytes(raw, byteorder=byteorder, signed=True)
|
|
1189
|
+
if self._usage is Usage.NATIONAL:
|
|
1190
|
+
return raw.decode("utf-16-be" if byteorder == "big" else "utf-16-le").rstrip()
|
|
1191
|
+
if self._usage is Usage.UTF_8:
|
|
1192
|
+
return raw.decode("utf-8", errors="replace").rstrip()
|
|
1193
|
+
if self._usage is Usage.DISPLAY_1:
|
|
1194
|
+
return raw.decode(dbcs_encoding, errors="replace").rstrip(" \u3000")
|
|
1195
|
+
if self._usage is Usage.BIT:
|
|
1196
|
+
return bit_unpack(raw, max(len(self._expanded_pic), 1))
|
|
1197
|
+
|
|
1198
|
+
if drop_converters:
|
|
1199
|
+
return raw.decode(encoding, errors="replace").rstrip()
|
|
1200
|
+
|
|
1201
|
+
kind = self.kind
|
|
1202
|
+
if kind is DataCategory.EXTERNAL_FLOAT:
|
|
1203
|
+
return external_float_unpack(
|
|
1204
|
+
raw,
|
|
1205
|
+
expanded_pic=self._expanded_pic,
|
|
1206
|
+
encoding=encoding,
|
|
1207
|
+
decimal_comma=decimal_comma,
|
|
1208
|
+
)
|
|
1209
|
+
if kind in (DataCategory.INTEGER, DataCategory.DECIMAL):
|
|
1210
|
+
return zoned_unpack(
|
|
1211
|
+
raw,
|
|
1212
|
+
encoding=encoding,
|
|
1213
|
+
signed=self._signed,
|
|
1214
|
+
sign_mode=self._sign_mode,
|
|
1215
|
+
exponent=self._exponent,
|
|
1216
|
+
as_decimal=as_decimal,
|
|
1217
|
+
)
|
|
1218
|
+
if kind is DataCategory.NUMERIC_EDITED:
|
|
1219
|
+
text = raw.decode(encoding, errors="replace")
|
|
1220
|
+
if "/" in self._expanded_pic:
|
|
1221
|
+
return text.strip()
|
|
1222
|
+
return parse_numeric_edited(
|
|
1223
|
+
text,
|
|
1224
|
+
blank_when_zero=self.blank_when_zero,
|
|
1225
|
+
decimal_comma=decimal_comma,
|
|
1226
|
+
)
|
|
1227
|
+
return raw.decode(encoding, errors="replace").rstrip()
|
|
1228
|
+
|
|
1229
|
+
def pic_to_fwf_kwargs(self, convert_compressed: bool = True) -> list[dict[str, Any]]:
|
|
1230
|
+
"""Convert this item into column descriptor dicts for :func:`pandas.read_fwf`.
|
|
1231
|
+
|
|
1232
|
+
Parameters
|
|
1233
|
+
----------
|
|
1234
|
+
convert_compressed
|
|
1235
|
+
Attach a byte-level converter for binary/compressed storage when ``True``.
|
|
1236
|
+
|
|
1237
|
+
Returns
|
|
1238
|
+
-------
|
|
1239
|
+
list[dict[str, Any]]
|
|
1240
|
+
One descriptor per ``OCCURS`` repetition containing ``name``,
|
|
1241
|
+
``pic``, and an optional ``converter`` callable.
|
|
1242
|
+
"""
|
|
1243
|
+
dtype: dict[str, Any] = {"name": self.name, "pic": self._expanded_pic}
|
|
1244
|
+
if self.is_binary:
|
|
1245
|
+
if convert_compressed:
|
|
1246
|
+
dtype["converter"] = lambda val, f=self: f.decode_bytes(
|
|
1247
|
+
val.encode("latin-1") if isinstance(val, str) else val
|
|
1248
|
+
)
|
|
1249
|
+
else:
|
|
1250
|
+
logger.warning(
|
|
1251
|
+
"{} uses binary storage ({}); keeping raw representation for text FWF.",
|
|
1252
|
+
self.name,
|
|
1253
|
+
self._usage.value,
|
|
1254
|
+
)
|
|
1255
|
+
else:
|
|
1256
|
+
conv = self._make_text_converter()
|
|
1257
|
+
if conv is not None:
|
|
1258
|
+
dtype["converter"] = conv
|
|
1259
|
+
|
|
1260
|
+
if self.occurs == 1:
|
|
1261
|
+
return [dtype]
|
|
1262
|
+
|
|
1263
|
+
return [
|
|
1264
|
+
{**copy.deepcopy(dtype), "name": _occurs_name(self.name, n, self.occurs)}
|
|
1265
|
+
for n in range(self.occurs)
|
|
1266
|
+
]
|
|
1267
|
+
|
|
1268
|
+
def _make_text_converter(self) -> Callable[[str], Any] | None:
|
|
1269
|
+
"""Build a string converter callable for :func:`pandas.read_fwf`."""
|
|
1270
|
+
kind = self.kind
|
|
1271
|
+
if kind in (DataCategory.INTEGER, DataCategory.DECIMAL):
|
|
1272
|
+
signed = self._signed
|
|
1273
|
+
sign_mode = self._sign_mode
|
|
1274
|
+
exponent = self._exponent
|
|
1275
|
+
|
|
1276
|
+
def _convert_zoned(val: str) -> int | float | Decimal:
|
|
1277
|
+
return zoned_unpack(
|
|
1278
|
+
val.encode("ascii"),
|
|
1279
|
+
encoding="ascii",
|
|
1280
|
+
signed=signed,
|
|
1281
|
+
sign_mode=sign_mode,
|
|
1282
|
+
exponent=exponent,
|
|
1283
|
+
)
|
|
1284
|
+
|
|
1285
|
+
if not signed and exponent == 0:
|
|
1286
|
+
return int
|
|
1287
|
+
return _convert_zoned
|
|
1288
|
+
|
|
1289
|
+
if kind is DataCategory.EXTERNAL_FLOAT:
|
|
1290
|
+
pic = self._expanded_pic
|
|
1291
|
+
|
|
1292
|
+
def _convert_ext_float(val: str) -> float:
|
|
1293
|
+
return external_float_unpack(val.encode("ascii"), expanded_pic=pic)
|
|
1294
|
+
|
|
1295
|
+
return _convert_ext_float
|
|
1296
|
+
|
|
1297
|
+
if kind is DataCategory.NUMERIC_EDITED and "/" not in self._expanded_pic:
|
|
1298
|
+
bwz = self.blank_when_zero
|
|
1299
|
+
|
|
1300
|
+
def _convert_edited(val: str) -> float | None:
|
|
1301
|
+
return parse_numeric_edited(val, blank_when_zero=bwz)
|
|
1302
|
+
|
|
1303
|
+
return _convert_edited
|
|
1304
|
+
|
|
1305
|
+
return None
|
|
1306
|
+
|
|
1307
|
+
def _resolve_sign_mode(self, description: dict[str, Any]) -> SignMode:
|
|
1308
|
+
if not self._signed or self._usage is not Usage.DISPLAY:
|
|
1309
|
+
return SignMode.NONE
|
|
1310
|
+
raw_sign = description.get("sign_mode")
|
|
1311
|
+
if isinstance(raw_sign, SignMode):
|
|
1312
|
+
return raw_sign
|
|
1313
|
+
return SignMode.TRAILING_OVERPUNCH
|
|
1314
|
+
|
|
1315
|
+
|
|
1316
|
+
def parse_copybook(rawtext: str | pathlib.Path) -> list[GroupItem | ElementaryItem]:
|
|
1317
|
+
"""Parse COBOL copybook source text or a file path into a record hierarchy.
|
|
1318
|
+
|
|
1319
|
+
Parameters
|
|
1320
|
+
----------
|
|
1321
|
+
rawtext
|
|
1322
|
+
Raw copybook text or path to a copybook file.
|
|
1323
|
+
|
|
1324
|
+
Returns
|
|
1325
|
+
-------
|
|
1326
|
+
list[GroupItem | ElementaryItem]
|
|
1327
|
+
Top-level record definitions parsed from the copybook.
|
|
1328
|
+
"""
|
|
1329
|
+
if isinstance(rawtext, pathlib.Path):
|
|
1330
|
+
text = rawtext.read_text(encoding="utf-8", errors="replace")
|
|
1331
|
+
elif "\n" not in rawtext and "\r" not in rawtext and len(rawtext) < 256:
|
|
1332
|
+
maybe_path = pathlib.Path(rawtext)
|
|
1333
|
+
text = (
|
|
1334
|
+
maybe_path.read_text(encoding="utf-8", errors="replace")
|
|
1335
|
+
if maybe_path.exists()
|
|
1336
|
+
else rawtext
|
|
1337
|
+
)
|
|
1338
|
+
else:
|
|
1339
|
+
text = rawtext
|
|
1340
|
+
|
|
1341
|
+
records: list[GroupItem | ElementaryItem] = []
|
|
1342
|
+
last_item: GroupItem | ElementaryItem | None = None
|
|
1343
|
+
|
|
1344
|
+
for stmt in extract_statements(text):
|
|
1345
|
+
desc = parse_statement(stmt)
|
|
1346
|
+
if desc is None:
|
|
1347
|
+
continue
|
|
1348
|
+
|
|
1349
|
+
level = int(desc["level"])
|
|
1350
|
+
|
|
1351
|
+
if level == 88:
|
|
1352
|
+
if last_item is not None:
|
|
1353
|
+
raw_val = desc.get("value") or ""
|
|
1354
|
+
last_item.conditions.append(
|
|
1355
|
+
ConditionName(
|
|
1356
|
+
name=desc["name"],
|
|
1357
|
+
raw_value=raw_val,
|
|
1358
|
+
values=parse_condition_values(raw_val),
|
|
1359
|
+
)
|
|
1360
|
+
)
|
|
1361
|
+
continue
|
|
1362
|
+
|
|
1363
|
+
if level == 66:
|
|
1364
|
+
continue
|
|
1365
|
+
|
|
1366
|
+
parent = _find_parent_group(records, level) if level not in (1, 77) else None
|
|
1367
|
+
|
|
1368
|
+
raw_usage = desc.get("usage")
|
|
1369
|
+
usage_enum = USAGE_ALIASES.get(raw_usage.upper()) if raw_usage else None
|
|
1370
|
+
if usage_enum is None and parent is not None and parent.usage is not None:
|
|
1371
|
+
desc["usage"] = parent.usage.value
|
|
1372
|
+
desc["comp"] = parent.usage.value
|
|
1373
|
+
usage_enum = parent.usage
|
|
1374
|
+
|
|
1375
|
+
is_elementary = (desc.get("pic") is not None) or (usage_enum in SELF_SIZED_USAGES)
|
|
1376
|
+
node: GroupItem | ElementaryItem = (
|
|
1377
|
+
ElementaryItem(desc) if is_elementary else GroupItem(desc)
|
|
1378
|
+
)
|
|
1379
|
+
|
|
1380
|
+
if level in (1, 77) or parent is None:
|
|
1381
|
+
records.append(node)
|
|
1382
|
+
else:
|
|
1383
|
+
parent.records.append(node)
|
|
1384
|
+
|
|
1385
|
+
if desc.get("redefines"):
|
|
1386
|
+
_register_redefines(records, parent, str(desc["redefines"]), node.name)
|
|
1387
|
+
|
|
1388
|
+
last_item = node
|
|
1389
|
+
|
|
1390
|
+
return records
|
|
1391
|
+
|
|
1392
|
+
|
|
1393
|
+
# Backward-compatible aliases
|
|
1394
|
+
CopyBook = Copybook
|
|
1395
|
+
Group = GroupItem
|
|
1396
|
+
Field = ElementaryItem
|
|
1397
|
+
Condition = ConditionName
|
|
1398
|
+
ResolvedField = ResolvedItem
|
|
1399
|
+
|
|
1400
|
+
|
|
1401
|
+
# ---------------------------------------------------------------------------
|
|
1402
|
+
# Internal Layout & Tree Helpers
|
|
1403
|
+
# ---------------------------------------------------------------------------
|
|
1404
|
+
|
|
1405
|
+
|
|
1406
|
+
def _deduplicate_names(names: list[str]) -> list[str]:
|
|
1407
|
+
"""Ensure unique DataFrame column names while preserving first occurrences."""
|
|
1408
|
+
counts: dict[str, int] = {}
|
|
1409
|
+
unique: list[str] = []
|
|
1410
|
+
for name in names:
|
|
1411
|
+
idx = counts.get(name, 0)
|
|
1412
|
+
counts[name] = idx + 1
|
|
1413
|
+
unique.append(name if idx == 0 else f"{name}_{idx}")
|
|
1414
|
+
return unique
|
|
1415
|
+
|
|
1416
|
+
|
|
1417
|
+
def _find_parent_group(
|
|
1418
|
+
records: list[GroupItem | ElementaryItem],
|
|
1419
|
+
level: int,
|
|
1420
|
+
) -> GroupItem | None:
|
|
1421
|
+
"""Return the deepest open :class:`GroupItem` whose level is strictly less than *level*."""
|
|
1422
|
+
for rec in reversed(records):
|
|
1423
|
+
if isinstance(rec, GroupItem) and rec.level < level:
|
|
1424
|
+
deeper = _find_parent_group(rec.records, level)
|
|
1425
|
+
return deeper if deeper is not None else rec
|
|
1426
|
+
return None
|
|
1427
|
+
|
|
1428
|
+
|
|
1429
|
+
def _occurs_name(base: str, index: int, total: int) -> str:
|
|
1430
|
+
"""Format an ``OCCURS`` column name (index 0 keeps the base name)."""
|
|
1431
|
+
if index == 0:
|
|
1432
|
+
return base
|
|
1433
|
+
width = 3 if total >= 100 else 2
|
|
1434
|
+
return f"{base}-{index:0{width}d}"
|
|
1435
|
+
|
|
1436
|
+
|
|
1437
|
+
def _register_redefines(
|
|
1438
|
+
records: list[GroupItem | ElementaryItem],
|
|
1439
|
+
parent: GroupItem | None,
|
|
1440
|
+
target_name: str,
|
|
1441
|
+
redefiner_name: str,
|
|
1442
|
+
) -> None:
|
|
1443
|
+
"""Record *redefiner_name* in the ``redefinedby`` list of *target_name*."""
|
|
1444
|
+
siblings = parent.records if parent is not None else records
|
|
1445
|
+
for rec in reversed(siblings):
|
|
1446
|
+
if rec.name == target_name:
|
|
1447
|
+
rec.redefinedby.append(redefiner_name)
|
|
1448
|
+
return
|