dataspector 1.0.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,1448 @@
1
+ """COBOL copybook AST classes (:class:`Copybook`, :class:`GroupItem`, :class:`ElementaryItem`)."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import copy
6
+ import pathlib
7
+ from collections.abc import Callable, Iterator
8
+ from dataclasses import dataclass
9
+ from decimal import Decimal
10
+ from typing import Any, Literal
11
+
12
+ import pandas as pd
13
+ import pandera.pandas as pa
14
+ from loguru import logger
15
+
16
+ from dataspector._reporting import chunks_to_dataframe, export_chunks
17
+ from dataspector.coboller.codecs import (
18
+ bit_unpack,
19
+ comp3_unpack,
20
+ comp6_unpack,
21
+ comp_float_unpack,
22
+ comp_unpack,
23
+ compute_decimal_exponent,
24
+ expand_pic,
25
+ external_float_unpack,
26
+ is_ebcdic,
27
+ parse_numeric_edited,
28
+ predict_encoding,
29
+ zoned_unpack,
30
+ )
31
+ from dataspector.coboller.constants import (
32
+ BINARY_USAGES,
33
+ DECIMAL_COMMA_RE,
34
+ FIXED_BINARY_SIZES,
35
+ PIC_ALPHA_ONLY,
36
+ PIC_BIT_ONLY,
37
+ PIC_DBCS_ONLY,
38
+ PIC_EDITED_MARKERS,
39
+ PIC_EXTERNAL_FLOAT,
40
+ PIC_NATIONAL_ONLY,
41
+ PIC_NUMERIC_PURE,
42
+ PIC_UTF8_ONLY,
43
+ SELF_SIZED_USAGES,
44
+ USAGE_ALIASES,
45
+ DataCategory,
46
+ SignMode,
47
+ Usage,
48
+ )
49
+ from dataspector.coboller.parser import (
50
+ extract_statements,
51
+ parse_condition_values,
52
+ parse_statement,
53
+ )
54
+
55
+
56
+ @dataclass(frozen=True, slots=True)
57
+ class ConditionName:
58
+ """Level-88 condition-name attached to a COBOL data item."""
59
+
60
+ name: str
61
+ raw_value: str
62
+ values: tuple[str, ...] = ()
63
+
64
+
65
+ @dataclass(frozen=True, slots=True)
66
+ class ResolvedItem:
67
+ """Flattened elementary item with exact byte offset and width in a record."""
68
+
69
+ name: str
70
+ field: ElementaryItem
71
+ offset: int
72
+ byte_length: int
73
+ is_redefines: bool = False
74
+
75
+
76
+ class Copybook:
77
+ """COBOL copybook parser, layout inspector, data reader, and format converter.
78
+
79
+ Parameters
80
+ ----------
81
+ copybook
82
+ Path to a COBOL copybook file.
83
+ """
84
+
85
+ def __init__(self, copybook: str | pathlib.Path) -> None:
86
+ self._copybook = copybook
87
+ self.copybook: pathlib.Path = pathlib.Path(copybook)
88
+ logger.trace("Loading COBOL copybook from {}", self.copybook)
89
+ self.raw: str = self.copybook.read_text(encoding="utf-8", errors="replace")
90
+ self.records: list[GroupItem | ElementaryItem] = parse_copybook(self.raw)
91
+
92
+ @classmethod
93
+ def from_file(cls, path: str | pathlib.Path) -> Copybook:
94
+ """Construct a :class:`Copybook` from a copybook file path.
95
+
96
+ Parameters
97
+ ----------
98
+ path
99
+ Path to a COBOL copybook file.
100
+
101
+ Returns
102
+ -------
103
+ Copybook
104
+ Parsed copybook instance.
105
+ """
106
+ return cls(path)
107
+
108
+ @classmethod
109
+ def from_string(cls, text: str, name: str = "<string>") -> Copybook:
110
+ """Construct a :class:`Copybook` directly from raw copybook source text.
111
+
112
+ Parameters
113
+ ----------
114
+ text
115
+ Raw COBOL copybook source code.
116
+ name
117
+ Synthetic display name assigned to :attr:`copybook`.
118
+
119
+ Returns
120
+ -------
121
+ Copybook
122
+ Parsed copybook instance.
123
+ """
124
+ instance = cls.__new__(cls)
125
+ instance._copybook = name
126
+ instance.copybook = pathlib.Path(name)
127
+ instance.raw = text
128
+ instance.records = parse_copybook(text)
129
+ return instance
130
+
131
+ @property
132
+ def columns(self) -> list[str]:
133
+ """Alias for :attr:`fields`."""
134
+ return self.fields
135
+
136
+ @property
137
+ def decimal_comma(self) -> bool:
138
+ """``True`` if the copybook declares ``DECIMAL-POINT IS COMMA``."""
139
+ return bool(DECIMAL_COMMA_RE.search(self.raw))
140
+
141
+ @property
142
+ def exists(self) -> bool:
143
+ """``True`` if the underlying copybook file exists on disk."""
144
+ return self.copybook.exists()
145
+
146
+ @property
147
+ def fields(self) -> list[str]:
148
+ """Ordered elementary item names in primary record order."""
149
+ return [rf.name for rf in self.layout(include_redefines=False, include_filler=True)]
150
+
151
+ @property
152
+ def has_binary_fields(self) -> bool:
153
+ """``True`` if any item uses binary, packed, float, pointer, index, or wide storage."""
154
+ return any(rf.field.is_binary for rf in self.layout(include_redefines=True))
155
+
156
+ @property
157
+ def record_length(self) -> int:
158
+ """Total physical byte length of a single record."""
159
+ return sum(rec.byte_length * rec.occurs for rec in self.records if rec.redefines is None)
160
+
161
+ def convert(
162
+ self,
163
+ datfile: str | pathlib.Path,
164
+ output: str | pathlib.Path,
165
+ chunksize: int = 5000,
166
+ encoding: str | None = None,
167
+ float_format: Literal["ieee", "ibm"] = "ieee",
168
+ byteorder: Literal["big", "little"] = "big",
169
+ include_redefines: bool = False,
170
+ include_filler: bool = False,
171
+ line_ending: Literal["auto", "none", "lf", "crlf"] = "auto",
172
+ dbcs_encoding: str = "utf-16-be",
173
+ decimal_comma: bool | None = None,
174
+ ) -> pathlib.Path:
175
+ """Convert a fixed-width or compressed COBOL data file into a modern readable format.
176
+
177
+ Streams chunk-by-chunk via :meth:`yield_data` for ``.csv``, ``.tsv``,
178
+ and ``.jsonl`` outputs; also supports ``.json``, ``.xlsx``, and
179
+ ``.parquet``. Aligned with :meth:`~dataspector.core.GenericData.convert`.
180
+
181
+ Parameters
182
+ ----------
183
+ datfile
184
+ Path to the raw COBOL data file.
185
+ output
186
+ Destination file path whose extension selects the output format.
187
+ chunksize
188
+ Number of rows per streaming chunk.
189
+ encoding
190
+ Character encoding for ``DISPLAY`` items (e.g. ``'cp500'``, ``'ascii'``).
191
+ float_format
192
+ Floating-point representation for ``COMP-1``/``COMP-2`` (``'ieee'`` or ``'ibm'``).
193
+ byteorder
194
+ Byte order for binary fields (``'big'`` or ``'little'``).
195
+ include_redefines
196
+ Include ``REDEFINES`` overlay columns in the converted output.
197
+ include_filler
198
+ Include ``FILLER`` columns (omitted by default for clean exports).
199
+ line_ending
200
+ Record delimiter mode (``'auto'``, ``'none'``, ``'lf'``, or ``'crlf'``).
201
+ dbcs_encoding
202
+ Character encoding for ``PIC G`` / ``DISPLAY-1`` DBCS fields.
203
+ decimal_comma
204
+ Override copybook ``DECIMAL-POINT IS COMMA`` detection.
205
+
206
+ Returns
207
+ -------
208
+ pathlib.Path
209
+ Path to the written output file.
210
+ """
211
+ logger.debug("Converting COBOL data {} -> {}", datfile, output)
212
+ resolved = self.layout(
213
+ include_redefines=include_redefines,
214
+ include_filler=include_filler,
215
+ )
216
+ empty_cols = _deduplicate_names([rf.name for rf in resolved])
217
+ chunks = self.yield_data(
218
+ datfile,
219
+ chunksize=chunksize,
220
+ encoding=encoding,
221
+ float_format=float_format,
222
+ byteorder=byteorder,
223
+ include_redefines=include_redefines,
224
+ include_filler=include_filler,
225
+ line_ending=line_ending,
226
+ dbcs_encoding=dbcs_encoding,
227
+ decimal_comma=decimal_comma,
228
+ )
229
+ return export_chunks(chunks, output, empty_columns=empty_cols)
230
+
231
+ def find_record(self, name: str) -> GroupItem | ElementaryItem | None:
232
+ """Find the first :class:`GroupItem` or :class:`ElementaryItem` matching *name*.
233
+
234
+ Parameters
235
+ ----------
236
+ name
237
+ Exact COBOL item name to search for.
238
+
239
+ Returns
240
+ -------
241
+ GroupItem | ElementaryItem | None
242
+ First matching item in depth-first order, or ``None`` if absent.
243
+ """
244
+ for item in self.records:
245
+ if item.name == name:
246
+ return item
247
+ if isinstance(item, GroupItem):
248
+ found = item.find_record(name)
249
+ if found is not None:
250
+ return found
251
+ return None
252
+
253
+ def layout(
254
+ self,
255
+ include_redefines: bool = False,
256
+ include_filler: bool = True,
257
+ ) -> list[ResolvedItem]:
258
+ """Compute flat byte-offset layout of all elementary items in the copybook.
259
+
260
+ Parameters
261
+ ----------
262
+ include_redefines
263
+ Include items defined under ``REDEFINES`` overlays.
264
+ include_filler
265
+ Include ``FILLER`` items in the resolved layout.
266
+
267
+ Returns
268
+ -------
269
+ list[ResolvedItem]
270
+ Ordered elementary items with exact byte offsets and widths.
271
+ """
272
+ resolved: list[ResolvedItem] = []
273
+
274
+ def _walk(
275
+ nodes: list[GroupItem | ElementaryItem],
276
+ start_offset: int,
277
+ is_redef_branch: bool,
278
+ suffix: str = "",
279
+ ) -> int:
280
+ offset = start_offset
281
+ sibling_offsets: dict[str, int] = {}
282
+
283
+ for node in nodes:
284
+ is_redef = is_redef_branch or (node.redefines is not None)
285
+ if node.redefines is not None:
286
+ node_start = sibling_offsets.get(node.redefines, offset)
287
+ if not include_redefines:
288
+ continue
289
+ else:
290
+ node_start = offset
291
+ sibling_offsets[node.name] = node_start
292
+
293
+ curr = node_start
294
+ for occ_idx in range(node.occurs):
295
+ occ_suffix = ""
296
+ if node.occurs > 1 and occ_idx > 0:
297
+ width = 3 if node.occurs >= 100 else 2
298
+ occ_suffix = f"-{occ_idx:0{width}d}"
299
+
300
+ if isinstance(node, ElementaryItem):
301
+ col_name = f"{node.name}{suffix}{occ_suffix}"
302
+ if include_filler or not node.is_filler:
303
+ resolved.append(
304
+ ResolvedItem(
305
+ name=col_name,
306
+ field=node,
307
+ offset=curr,
308
+ byte_length=node.byte_length,
309
+ is_redefines=is_redef,
310
+ )
311
+ )
312
+ curr += node.byte_length
313
+ else:
314
+ curr = _walk(
315
+ node.records,
316
+ curr,
317
+ is_redef_branch=is_redef,
318
+ suffix=f"{suffix}{occ_suffix}",
319
+ )
320
+
321
+ if node.redefines is None:
322
+ offset = curr
323
+ return offset
324
+
325
+ _walk(self.records, 0, is_redef_branch=False)
326
+ return resolved
327
+
328
+ def pic_to_fwf_kwargs(self, convert_compressed: bool = True) -> dict[str, Any]:
329
+ """Return keyword arguments for :func:`pandas.read_fwf` on text copybooks.
330
+
331
+ Parameters
332
+ ----------
333
+ convert_compressed
334
+ Attach byte-level decoders for binary/compressed fields when ``True``;
335
+ keep raw representations when ``False``.
336
+
337
+ Returns
338
+ -------
339
+ dict[str, Any]
340
+ Dictionary containing ``colspecs``, ``names``, and ``converters``
341
+ suitable for unpacking into :func:`pandas.read_fwf`.
342
+ """
343
+ resolved = self.layout(include_redefines=False, include_filler=True)
344
+ colspecs: list[tuple[int, int]] = []
345
+ names: list[str] = []
346
+ converters: dict[int, Callable[[Any], Any]] = {}
347
+
348
+ for idx, rf in enumerate(resolved):
349
+ names.append(rf.name)
350
+ colspecs.append((rf.offset, rf.offset + rf.byte_length))
351
+ field_kwargs = rf.field.pic_to_fwf_kwargs(convert_compressed=convert_compressed)
352
+ conv = field_kwargs[0].get("converter")
353
+ if conv is not None:
354
+ converters[idx] = conv
355
+
356
+ return {"colspecs": colspecs, "names": names, "converters": converters}
357
+
358
+ def summary(
359
+ self,
360
+ include_redefines: bool = False,
361
+ include_filler: bool = True,
362
+ ) -> pd.DataFrame:
363
+ """Return a tabular overview of every elementary item's byte layout and COBOL type.
364
+
365
+ Parameters
366
+ ----------
367
+ include_redefines
368
+ Include ``REDEFINES`` overlay items in the summary table.
369
+ include_filler
370
+ Include ``FILLER`` items in the summary table.
371
+
372
+ Returns
373
+ -------
374
+ pandas.DataFrame
375
+ Table with columns ``name``, ``level``, ``pic``, ``usage``, ``kind``,
376
+ ``offset``, ``byte_length``, ``signed``, ``decimals``, and
377
+ ``is_redefines``.
378
+ """
379
+ resolved = self.layout(
380
+ include_redefines=include_redefines,
381
+ include_filler=include_filler,
382
+ )
383
+ return pd.DataFrame(
384
+ [
385
+ {
386
+ "name": rf.name,
387
+ "level": rf.field.level,
388
+ "pic": (rf.field.description.get("pic") or "").strip(),
389
+ "usage": rf.field.usage.value,
390
+ "kind": rf.field.kind.value,
391
+ "offset": rf.offset,
392
+ "byte_length": rf.byte_length,
393
+ "signed": rf.field.signed,
394
+ "decimals": rf.field.decimals,
395
+ "is_redefines": rf.is_redefines,
396
+ }
397
+ for rf in resolved
398
+ ]
399
+ )
400
+
401
+ def to_dataframe(
402
+ self,
403
+ datfile: str | pathlib.Path,
404
+ nrows: int | None = None,
405
+ chunksize: int = 500,
406
+ encoding: str | None = None,
407
+ convert_compressed: bool = True,
408
+ drop_converters: bool = False,
409
+ float_format: Literal["ieee", "ibm"] = "ieee",
410
+ byteorder: Literal["big", "little"] = "big",
411
+ as_decimal: bool = False,
412
+ include_redefines: bool = False,
413
+ include_filler: bool = True,
414
+ line_ending: Literal["auto", "none", "lf", "crlf"] = "auto",
415
+ dbcs_encoding: str = "utf-16-be",
416
+ decimal_comma: bool | None = None,
417
+ ) -> pd.DataFrame:
418
+ """Decode *datfile* into a single materialized :class:`pandas.DataFrame`.
419
+
420
+ Reads internally via chunked streaming (:meth:`yield_data`) and stops
421
+ early once *nrows* records have been collected. Aligned with
422
+ :meth:`~dataspector.core.GenericData.to_dataframe`.
423
+
424
+ Notes
425
+ -----
426
+ Because the return value is a single in-memory DataFrame, pass *nrows*
427
+ for quick previews of very large mainframe files, or iterate directly
428
+ over :meth:`yield_data` when streaming files larger than memory.
429
+
430
+ Parameters
431
+ ----------
432
+ datfile
433
+ Path to the COBOL data file.
434
+ nrows
435
+ Optional maximum number of records to return (``None`` reads all).
436
+ chunksize
437
+ Number of rows decoded per internal streaming chunk.
438
+ encoding
439
+ Character encoding for ``DISPLAY`` items (auto-detected if ``None``).
440
+ convert_compressed
441
+ Decode binary/compressed items; retain hex representation if ``False``.
442
+ drop_converters
443
+ Keep all items as raw decoded strings / hex without numeric casting.
444
+ float_format
445
+ Floating-point representation for ``COMP-1``/``COMP-2`` (``'ieee'`` or ``'ibm'``).
446
+ byteorder
447
+ Byte order for binary fields (``'big'`` or ``'little'``).
448
+ as_decimal
449
+ Preserve exact precision with :class:`decimal.Decimal` for fixed-point numbers.
450
+ include_redefines
451
+ Include ``REDEFINES`` overlay columns.
452
+ include_filler
453
+ Include ``FILLER`` columns.
454
+ line_ending
455
+ Record delimiter mode (``'auto'``, ``'none'``, ``'lf'``, or ``'crlf'``).
456
+ dbcs_encoding
457
+ Character encoding for ``PIC G`` / ``DISPLAY-1`` DBCS items.
458
+ decimal_comma
459
+ Override copybook ``DECIMAL-POINT IS COMMA`` detection.
460
+
461
+ Returns
462
+ -------
463
+ pandas.DataFrame
464
+ Decoded records (empty DataFrame with copybook columns if no rows).
465
+ """
466
+ eff_chunk = min(chunksize, nrows) if nrows is not None and nrows > 0 else chunksize
467
+ resolved = self.layout(
468
+ include_redefines=include_redefines,
469
+ include_filler=include_filler,
470
+ )
471
+ empty_cols = _deduplicate_names([rf.name for rf in resolved])
472
+ chunks = self.yield_data(
473
+ datfile,
474
+ chunksize=eff_chunk,
475
+ encoding=encoding,
476
+ convert_compressed=convert_compressed,
477
+ drop_converters=drop_converters,
478
+ float_format=float_format,
479
+ byteorder=byteorder,
480
+ as_decimal=as_decimal,
481
+ include_redefines=include_redefines,
482
+ include_filler=include_filler,
483
+ line_ending=line_ending,
484
+ dbcs_encoding=dbcs_encoding,
485
+ decimal_comma=decimal_comma,
486
+ )
487
+ return chunks_to_dataframe(chunks, nrows=nrows, empty_columns=empty_cols)
488
+
489
+ def to_pandera_schema(
490
+ self,
491
+ include_redefines: bool = False,
492
+ include_filler: bool = True,
493
+ enforce_conditions: bool = False,
494
+ coerce: bool = False,
495
+ ) -> pa.DataFrameSchema:
496
+ """Generate a :class:`pandera.pandas.DataFrameSchema` from this copybook.
497
+
498
+ Parameters
499
+ ----------
500
+ include_redefines
501
+ Include ``REDEFINES`` overlay items as schema columns.
502
+ include_filler
503
+ Include ``FILLER`` items as schema columns.
504
+ enforce_conditions
505
+ Add ``isin`` checks derived from level-88 condition-name values.
506
+ coerce
507
+ Whether the returned schema coerces column dtypes on validation.
508
+
509
+ Returns
510
+ -------
511
+ pandera.pandas.DataFrameSchema
512
+ Schema encoding column dtypes, length/range bounds, and optional
513
+ level-88 domain checks.
514
+ """
515
+ resolved = self.layout(
516
+ include_redefines=include_redefines,
517
+ include_filler=include_filler,
518
+ )
519
+ col_names = _deduplicate_names([rf.name for rf in resolved])
520
+ columns: dict[str, pa.Column] = {}
521
+
522
+ for col_name, rf in zip(col_names, resolved, strict=True):
523
+ field = rf.field
524
+ checks: list[pa.Check] = []
525
+ dtype: Any = None
526
+ nullable = field.kind is DataCategory.NUMERIC_EDITED and not field.blank_when_zero
527
+
528
+ if field.kind in (
529
+ DataCategory.ALPHABETIC,
530
+ DataCategory.ALPHANUMERIC,
531
+ DataCategory.NATIONAL,
532
+ DataCategory.DBCS,
533
+ DataCategory.UTF8,
534
+ DataCategory.BIT,
535
+ ):
536
+ dtype = str
537
+ if field.kind is DataCategory.BIT:
538
+ bit_len = max(len(field.expanded_pic), 1)
539
+ checks.append(pa.Check.str_length(min_value=bit_len, max_value=bit_len))
540
+ checks.append(pa.Check.str_matches(r"^[01]+$"))
541
+ else:
542
+ max_chars = (
543
+ len(field.expanded_pic)
544
+ if field.kind in (DataCategory.NATIONAL, DataCategory.DBCS)
545
+ else field.byte_length
546
+ )
547
+ checks.append(pa.Check.str_length(max_value=max_chars))
548
+ elif field.kind in (
549
+ DataCategory.INTEGER,
550
+ DataCategory.BINARY,
551
+ DataCategory.PACKED_DECIMAL,
552
+ DataCategory.PACKED_UNSIGNED,
553
+ ):
554
+ dtype = float if field.decimals > 0 else int
555
+ if field.usage in FIXED_BINARY_SIZES:
556
+ bits = FIXED_BINARY_SIZES[field.usage] * 8
557
+ if field.signed:
558
+ checks.append(
559
+ pa.Check.in_range(
560
+ min_value=-(1 << (bits - 1)),
561
+ max_value=(1 << (bits - 1)) - 1,
562
+ )
563
+ )
564
+ else:
565
+ checks.append(pa.Check.in_range(min_value=0, max_value=(1 << bits) - 1))
566
+ elif field.usage is Usage.COMP_X:
567
+ bits = field.byte_length * 8
568
+ checks.append(pa.Check.in_range(min_value=0, max_value=(1 << bits) - 1))
569
+ elif field.digits > 0 and field.usage is not Usage.COMP_5:
570
+ max_abs = (10**field.digits - 1) * (10**field.exponent)
571
+ min_val = -max_abs if field.signed else 0
572
+ checks.append(pa.Check.in_range(min_value=min_val, max_value=max_abs))
573
+ elif field.kind in (
574
+ DataCategory.DECIMAL,
575
+ DataCategory.FLOAT,
576
+ DataCategory.EXTERNAL_FLOAT,
577
+ DataCategory.NUMERIC_EDITED,
578
+ ):
579
+ dtype = float
580
+ if field.kind is DataCategory.DECIMAL and field.digits > 0:
581
+ max_abs = float((10**field.digits - 1) * (10**field.exponent))
582
+ min_val = -max_abs if field.signed else 0.0
583
+ checks.append(pa.Check.in_range(min_value=min_val, max_value=max_abs))
584
+ elif field.kind in (DataCategory.POINTER, DataCategory.INDEX):
585
+ dtype = int
586
+ if field.kind is DataCategory.POINTER:
587
+ checks.append(pa.Check.greater_than_or_equal_to(0))
588
+
589
+ if enforce_conditions and field.conditions:
590
+ allowed = [v for cond in field.conditions for v in cond.values]
591
+ if allowed:
592
+ checks.append(pa.Check.isin(allowed))
593
+
594
+ columns[col_name] = pa.Column(dtype, checks=checks, nullable=nullable)
595
+
596
+ return pa.DataFrameSchema(columns=columns, coerce=coerce)
597
+
598
+ def yield_data(
599
+ self,
600
+ datfile: str | pathlib.Path,
601
+ chunksize: int = 500,
602
+ encoding: str | None = None,
603
+ convert_compressed: bool = True,
604
+ drop_converters: bool = False,
605
+ float_format: Literal["ieee", "ibm"] = "ieee",
606
+ byteorder: Literal["big", "little"] = "big",
607
+ as_decimal: bool = False,
608
+ include_redefines: bool = False,
609
+ include_filler: bool = True,
610
+ line_ending: Literal["auto", "none", "lf", "crlf"] = "auto",
611
+ dbcs_encoding: str = "utf-16-be",
612
+ decimal_comma: bool | None = None,
613
+ ) -> Iterator[pd.DataFrame]:
614
+ """Stream records from *datfile* as :class:`pandas.DataFrame` chunks.
615
+
616
+ Parameters
617
+ ----------
618
+ datfile
619
+ Path to the raw COBOL data file.
620
+ chunksize
621
+ Maximum number of decoded records per yielded chunk.
622
+ encoding
623
+ Character encoding for ``DISPLAY`` items (auto-detected if ``None``).
624
+ convert_compressed
625
+ Decode binary/compressed items; retain hex representation if ``False``.
626
+ drop_converters
627
+ Keep all items as raw decoded strings / hex without numeric casting.
628
+ float_format
629
+ Floating-point representation for ``COMP-1``/``COMP-2`` (``'ieee'`` or ``'ibm'``).
630
+ byteorder
631
+ Byte order for binary fields (``'big'`` or ``'little'``).
632
+ as_decimal
633
+ Preserve exact precision with :class:`decimal.Decimal` for fixed-point numbers.
634
+ include_redefines
635
+ Include ``REDEFINES`` overlay columns.
636
+ include_filler
637
+ Include ``FILLER`` columns.
638
+ line_ending
639
+ Record delimiter mode (``'auto'``, ``'none'``, ``'lf'``, or ``'crlf'``).
640
+ dbcs_encoding
641
+ Character encoding for ``PIC G`` / ``DISPLAY-1`` DBCS items.
642
+ decimal_comma
643
+ Override copybook ``DECIMAL-POINT IS COMMA`` detection.
644
+
645
+ Yields
646
+ ------
647
+ pandas.DataFrame
648
+ Decoded chunk of up to *chunksize* records.
649
+ """
650
+ if drop_converters and convert_compressed:
651
+ logger.debug("drop_converters=True: disabling compressed field conversion.")
652
+ convert_compressed = False
653
+
654
+ datpath = pathlib.Path(datfile)
655
+ resolved = self.layout(
656
+ include_redefines=include_redefines,
657
+ include_filler=include_filler,
658
+ )
659
+ rec_len = self.record_length
660
+ if not resolved or rec_len == 0:
661
+ return
662
+
663
+ resolved_encoding: str = (
664
+ encoding
665
+ if encoding is not None
666
+ else ("cp500" if self.has_binary_fields else predict_encoding(datpath))
667
+ )
668
+ col_names = _deduplicate_names([rf.name for rf in resolved])
669
+ detected_ending = self._detect_line_ending(datpath, rec_len, line_ending)
670
+ pad_byte = b"\x40" if is_ebcdic(resolved_encoding) else b" "
671
+ resolved_decimal_comma = self.decimal_comma if decimal_comma is None else decimal_comma
672
+ logger.trace(
673
+ "Streaming {} (rec_len={}, encoding={}, line_ending={})",
674
+ datpath,
675
+ rec_len,
676
+ resolved_encoding,
677
+ detected_ending,
678
+ )
679
+
680
+ def _parse_record(raw_rec: bytes) -> list[Any]:
681
+ if len(raw_rec) < rec_len:
682
+ raw_rec = raw_rec + pad_byte * (rec_len - len(raw_rec))
683
+ return [
684
+ rf.field.decode_bytes(
685
+ raw_rec[rf.offset : rf.offset + rf.byte_length],
686
+ encoding=resolved_encoding,
687
+ float_format=float_format,
688
+ byteorder=byteorder,
689
+ as_decimal=as_decimal,
690
+ convert_compressed=convert_compressed,
691
+ drop_converters=drop_converters,
692
+ dbcs_encoding=dbcs_encoding,
693
+ decimal_comma=resolved_decimal_comma,
694
+ )
695
+ for rf in resolved
696
+ ]
697
+
698
+ rows: list[list[Any]] = []
699
+ with datpath.open("rb") as fh:
700
+ if detected_ending == "none":
701
+ while True:
702
+ block = fh.read(rec_len)
703
+ if not block:
704
+ break
705
+ if len(block) < rec_len:
706
+ logger.warning(
707
+ "Truncated trailing COBOL record: expected {} bytes, got {}.",
708
+ rec_len,
709
+ len(block),
710
+ )
711
+ rows.append(_parse_record(block))
712
+ if len(rows) >= chunksize:
713
+ yield pd.DataFrame(rows, columns=col_names)
714
+ rows = []
715
+ else:
716
+ for line in fh:
717
+ stripped = line.rstrip(b"\r\n")
718
+ if not stripped:
719
+ continue
720
+ rows.append(_parse_record(stripped))
721
+ if len(rows) >= chunksize:
722
+ yield pd.DataFrame(rows, columns=col_names)
723
+ rows = []
724
+
725
+ if rows:
726
+ yield pd.DataFrame(rows, columns=col_names)
727
+
728
+ def _detect_line_ending(
729
+ self,
730
+ datpath: pathlib.Path,
731
+ rec_len: int,
732
+ line_ending: Literal["auto", "none", "lf", "crlf"],
733
+ ) -> Literal["none", "lf", "crlf"]:
734
+ """Determine whether *datpath* uses fixed binary records or line terminators."""
735
+ if line_ending != "auto":
736
+ return line_ending
737
+ if rec_len <= 0:
738
+ return "lf"
739
+
740
+ file_size = datpath.stat().st_size
741
+ with datpath.open("rb") as fh:
742
+ probe = fh.read(rec_len + 2)
743
+
744
+ if len(probe) >= rec_len + 2 and probe[rec_len : rec_len + 2] == b"\r\n":
745
+ return "crlf"
746
+ if len(probe) >= rec_len + 1 and probe[rec_len : rec_len + 1] == b"\n":
747
+ return "lf"
748
+
749
+ if self.has_binary_fields:
750
+ if file_size % rec_len == 0:
751
+ return "none"
752
+ if file_size % (rec_len + 1) == 0:
753
+ return "lf"
754
+ if file_size % (rec_len + 2) == 0:
755
+ return "crlf"
756
+ return "none"
757
+
758
+ if b"\r\n" in probe:
759
+ return "crlf"
760
+ if b"\n" in probe:
761
+ return "lf"
762
+ return "none"
763
+
764
+
765
+ class GroupItem:
766
+ """Hierarchical COBOL group item containing subordinate items."""
767
+
768
+ def __init__(self, description: dict[str, Any]) -> None:
769
+ self.description: dict[str, Any] = description
770
+ self.redefinedby: list[str] = []
771
+ self.records: list[GroupItem | ElementaryItem] = []
772
+ self.conditions: list[ConditionName] = []
773
+ raw_usage = (description.get("usage") or description.get("comp") or "").upper()
774
+ self._usage: Usage | None = USAGE_ALIASES.get(raw_usage) if raw_usage else None
775
+
776
+ def __repr__(self) -> str:
777
+ suffix = f", {self.occurs} OCCURS" if self.occurs > 1 else ""
778
+ return (
779
+ f"<Group (Level {self.description.get('level')}, {self.name}, "
780
+ f"{len(self.records)} subrecords{suffix}>"
781
+ )
782
+
783
+ @property
784
+ def byte_length(self) -> int:
785
+ """Physical byte width of a single occurrence of this group item (excluding redefines)."""
786
+ return sum(rec.byte_length * rec.occurs for rec in self.records if rec.redefines is None)
787
+
788
+ @property
789
+ def level(self) -> int:
790
+ """COBOL level number."""
791
+ return int(self.description["level"])
792
+
793
+ @property
794
+ def name(self) -> str:
795
+ """Group item name."""
796
+ return str(self.description.get("name") or "FILLER")
797
+
798
+ @property
799
+ def occurs(self) -> int:
800
+ """Repetition count from ``OCCURS`` (``1`` when non-repeating)."""
801
+ occ = self.description.get("occurs")
802
+ return int(occ) if occ is not None else 1
803
+
804
+ @property
805
+ def redefines(self) -> str | None:
806
+ """Name of the sibling item this group redefines, or ``None``."""
807
+ val = self.description.get("redefines")
808
+ return str(val) if val else None
809
+
810
+ @property
811
+ def usage(self) -> Usage | None:
812
+ """Optional group-level ``USAGE`` inherited by subordinate elementary items."""
813
+ return self._usage
814
+
815
+ def append(self, level: int, record: GroupItem | ElementaryItem) -> None:
816
+ """Insert *record* at the deepest nested subgroup whose level is ``< level``.
817
+
818
+ Parameters
819
+ ----------
820
+ level
821
+ COBOL level number of the item being inserted.
822
+ record
823
+ Group or elementary item to append.
824
+ """
825
+ for rec in reversed(self.records):
826
+ if isinstance(rec, GroupItem) and rec.level < level:
827
+ rec.append(level, record)
828
+ return
829
+ self.records.append(record)
830
+
831
+ def find_record(self, name: str) -> GroupItem | ElementaryItem | None:
832
+ """Depth-first search for the first subordinate item named *name*.
833
+
834
+ Parameters
835
+ ----------
836
+ name
837
+ Exact COBOL item name to search for.
838
+
839
+ Returns
840
+ -------
841
+ GroupItem | ElementaryItem | None
842
+ First matching subordinate item, or ``None`` if not found.
843
+ """
844
+ for gf in self.records:
845
+ if gf.name == name:
846
+ return gf
847
+ if isinstance(gf, GroupItem):
848
+ found = gf.find_record(name)
849
+ if found is not None:
850
+ return found
851
+ return None
852
+
853
+ def pic_to_fwf_kwargs(self, convert_compressed: bool = True) -> list[dict[str, Any]]:
854
+ """Return flattened :func:`pandas.read_fwf` column descriptors for this group item.
855
+
856
+ Parameters
857
+ ----------
858
+ convert_compressed
859
+ Attach byte-level decoders for binary/compressed items when ``True``.
860
+
861
+ Returns
862
+ -------
863
+ list[dict[str, Any]]
864
+ Column descriptor dictionaries expanded across any ``OCCURS`` repetitions.
865
+ """
866
+ dtypes: list[dict[str, Any]] = []
867
+ for rec in self.records:
868
+ if rec.redefines is not None:
869
+ continue
870
+ dtypes.extend(rec.pic_to_fwf_kwargs(convert_compressed=convert_compressed))
871
+
872
+ if self.occurs == 1:
873
+ return dtypes
874
+
875
+ expanded: list[dict[str, Any]] = []
876
+ for n in range(self.occurs):
877
+ for dtype in dtypes:
878
+ entry = copy.deepcopy(dtype)
879
+ entry["name"] = _occurs_name(entry["name"], n, self.occurs)
880
+ expanded.append(entry)
881
+ return expanded
882
+
883
+
884
+ class ElementaryItem:
885
+ """COBOL elementary data item carrying a ``PICTURE`` or self-sized ``USAGE``."""
886
+
887
+ def __init__(self, description: dict[str, Any]) -> None:
888
+ self.description: dict[str, Any] = description
889
+ self.redefinedby: list[str] = []
890
+ self.conditions: list[ConditionName] = []
891
+
892
+ raw_usage = (description.get("usage") or description.get("comp") or "DISPLAY").upper()
893
+ self._usage: Usage = USAGE_ALIASES.get(raw_usage, Usage.DISPLAY)
894
+
895
+ raw_pic = (description.get("pic") or "").strip()
896
+ self._expanded_pic: str = expand_pic(raw_pic) if raw_pic else ""
897
+ if self._usage is Usage.DISPLAY and self._expanded_pic:
898
+ if PIC_NATIONAL_ONLY.match(self._expanded_pic):
899
+ self._usage = Usage.NATIONAL
900
+ elif PIC_UTF8_ONLY.match(self._expanded_pic):
901
+ self._usage = Usage.UTF_8
902
+ elif PIC_DBCS_ONLY.match(self._expanded_pic):
903
+ self._usage = Usage.DISPLAY_1
904
+ elif PIC_BIT_ONLY.match(self._expanded_pic):
905
+ self._usage = Usage.BIT
906
+
907
+ raw_bl = description.get("byte_length")
908
+ self._explicit_byte_length: int | None = int(raw_bl) if raw_bl is not None else None
909
+
910
+ bin_qual = (description.get("binary_sign_qualifier") or "").upper()
911
+ if self._usage in FIXED_BINARY_SIZES:
912
+ self._signed = bin_qual != "UNSIGNED"
913
+ elif self._usage in (Usage.COMP_6, Usage.COMP_X):
914
+ self._signed = False
915
+ else:
916
+ self._signed = self._expanded_pic.startswith("S")
917
+ self._sign_mode: SignMode = self._resolve_sign_mode(description)
918
+ self._exponent: int = (
919
+ compute_decimal_exponent(self._expanded_pic)
920
+ if self._expanded_pic and PIC_NUMERIC_PURE.match(self._expanded_pic)
921
+ else 0
922
+ )
923
+
924
+ @classmethod
925
+ def from_clause(cls, statement: str) -> ElementaryItem:
926
+ """Construct an :class:`ElementaryItem` from a single COBOL declaration line.
927
+
928
+ Parameters
929
+ ----------
930
+ statement
931
+ Single COBOL data description clause (with or without trailing period).
932
+
933
+ Returns
934
+ -------
935
+ ElementaryItem
936
+ Parsed elementary item.
937
+ """
938
+ cleaned = statement.strip().rstrip(".")
939
+ desc = parse_statement(cleaned)
940
+ if desc is None:
941
+ raise ValueError(f"Could not parse COBOL elementary item: {statement!r}")
942
+ return cls(desc)
943
+
944
+ def __repr__(self) -> str:
945
+ pic_display = (self.description.get("pic") or "").strip()
946
+ detail = pic_display or self._usage.value
947
+ if pic_display and self._usage is not Usage.DISPLAY:
948
+ detail = f"{pic_display} {self._usage.value}"
949
+ suffix = f", {self.occurs} OCCURS" if self.occurs > 1 else ""
950
+ return f"<Field (Level {self.description.get('level')}, {self.name}, {detail}{suffix}>"
951
+
952
+ @property
953
+ def blank_when_zero(self) -> bool:
954
+ """``True`` if ``BLANK WHEN ZERO`` was declared."""
955
+ return bool(self.description.get("blank_when_zero", False))
956
+
957
+ @property
958
+ def byte_length(self) -> int:
959
+ """Physical byte width of a single occurrence of this elementary item."""
960
+ if self._explicit_byte_length is not None:
961
+ return self._explicit_byte_length
962
+ if self._usage in FIXED_BINARY_SIZES:
963
+ return FIXED_BINARY_SIZES[self._usage]
964
+ if self._usage in (Usage.COMP_1, Usage.FLOAT_SHORT, Usage.INDEX, Usage.POINTER):
965
+ return 4
966
+ if self._usage in (
967
+ Usage.COMP_2,
968
+ Usage.FLOAT_LONG,
969
+ Usage.PROCEDURE_POINTER,
970
+ Usage.FUNCTION_POINTER,
971
+ ):
972
+ return 8
973
+ if self._usage is Usage.COMP_3:
974
+ return (self.digits // 2) + 1
975
+ if self._usage is Usage.COMP_6:
976
+ return (self.digits + 1) // 2
977
+ if self._usage is Usage.COMP_X:
978
+ if "X" in self._expanded_pic:
979
+ return self._expanded_pic.count("X")
980
+ d = max(self.digits, 1)
981
+ return ((10**d - 1).bit_length() + 7) // 8
982
+ if self._usage in (Usage.COMP, Usage.COMP_5):
983
+ d = self.digits
984
+ if d <= 4:
985
+ return 2
986
+ if d <= 9:
987
+ return 4
988
+ if d <= 18:
989
+ return 8
990
+ return 16
991
+ if self._usage in (Usage.NATIONAL, Usage.DISPLAY_1):
992
+ return len(self._expanded_pic) * 2
993
+ if self._usage is Usage.UTF_8:
994
+ return len(self._expanded_pic) * 4
995
+ if self._usage is Usage.BIT:
996
+ return (max(len(self._expanded_pic), 1) + 7) // 8
997
+
998
+ physical = self._expanded_pic.replace("S", "").replace("V", "").replace("P", "")
999
+ extra = (
1000
+ 1 if self._sign_mode in (SignMode.LEADING_SEPARATE, SignMode.TRAILING_SEPARATE) else 0
1001
+ )
1002
+ return len(physical) + extra
1003
+
1004
+ @property
1005
+ def decimals(self) -> int:
1006
+ """Number of implied fractional decimal digits."""
1007
+ return max(0, -self._exponent)
1008
+
1009
+ @property
1010
+ def digits(self) -> int:
1011
+ """Total count of ``9`` digit positions in the ``PIC`` clause."""
1012
+ return self._expanded_pic.count("9")
1013
+
1014
+ @property
1015
+ def expanded_pic(self) -> str:
1016
+ """Fully expanded uppercase ``PIC`` string."""
1017
+ return self._expanded_pic
1018
+
1019
+ @property
1020
+ def exponent(self) -> int:
1021
+ """Base-10 scaling exponent from ``V`` and ``P`` symbols."""
1022
+ return self._exponent
1023
+
1024
+ @property
1025
+ def is_binary(self) -> bool:
1026
+ """``True`` if the item uses binary, packed, float, pointer, index, or wide storage."""
1027
+ return self._usage in BINARY_USAGES
1028
+
1029
+ @property
1030
+ def is_filler(self) -> bool:
1031
+ """``True`` if this item is a ``FILLER`` item."""
1032
+ return self.name.upper() == "FILLER"
1033
+
1034
+ @property
1035
+ def kind(self) -> DataCategory:
1036
+ """COBOL :class:`~dataspector.coboller.constants.DataCategory` of this elementary item."""
1037
+ if self._usage in (Usage.COMP_1, Usage.COMP_2, Usage.FLOAT_SHORT, Usage.FLOAT_LONG):
1038
+ return DataCategory.FLOAT
1039
+ if self._usage in (Usage.POINTER, Usage.PROCEDURE_POINTER, Usage.FUNCTION_POINTER):
1040
+ return DataCategory.POINTER
1041
+ if self._usage is Usage.INDEX:
1042
+ return DataCategory.INDEX
1043
+ if self._usage is Usage.COMP_3:
1044
+ return DataCategory.PACKED_DECIMAL
1045
+ if self._usage is Usage.COMP_6:
1046
+ return DataCategory.PACKED_UNSIGNED
1047
+ if self._usage in (Usage.COMP, Usage.COMP_5, Usage.COMP_X) or (
1048
+ self._usage in FIXED_BINARY_SIZES
1049
+ ):
1050
+ return DataCategory.BINARY
1051
+ if self._usage is Usage.NATIONAL:
1052
+ return DataCategory.NATIONAL
1053
+ if self._usage is Usage.UTF_8:
1054
+ return DataCategory.UTF8
1055
+ if self._usage is Usage.DISPLAY_1:
1056
+ return DataCategory.DBCS
1057
+ if self._usage is Usage.BIT:
1058
+ return DataCategory.BIT
1059
+
1060
+ pic = self._expanded_pic
1061
+ if not pic:
1062
+ return DataCategory.ALPHANUMERIC
1063
+ if PIC_EXTERNAL_FLOAT.match(pic):
1064
+ return DataCategory.EXTERNAL_FLOAT
1065
+ if PIC_NUMERIC_PURE.match(pic):
1066
+ return DataCategory.DECIMAL if self._exponent < 0 else DataCategory.INTEGER
1067
+ if PIC_ALPHA_ONLY.match(pic):
1068
+ return DataCategory.ALPHABETIC
1069
+ if "X" not in pic and "A" not in pic and PIC_EDITED_MARKERS.search(pic):
1070
+ return DataCategory.NUMERIC_EDITED
1071
+ return DataCategory.ALPHANUMERIC
1072
+
1073
+ @property
1074
+ def level(self) -> int:
1075
+ """COBOL level number."""
1076
+ return int(self.description["level"])
1077
+
1078
+ @property
1079
+ def name(self) -> str:
1080
+ """Item name (``'FILLER'`` if unnamed)."""
1081
+ return str(self.description.get("name") or "FILLER")
1082
+
1083
+ @property
1084
+ def occurs(self) -> int:
1085
+ """Repetition count from ``OCCURS`` (``1`` when non-repeating)."""
1086
+ occ = self.description.get("occurs")
1087
+ return int(occ) if occ is not None else 1
1088
+
1089
+ @property
1090
+ def redefines(self) -> str | None:
1091
+ """Name of the sibling item this item redefines, or ``None``."""
1092
+ val = self.description.get("redefines")
1093
+ return str(val) if val else None
1094
+
1095
+ @property
1096
+ def sign_mode(self) -> SignMode:
1097
+ """Sign encoding mode for ``USAGE DISPLAY`` signed numeric items."""
1098
+ return self._sign_mode
1099
+
1100
+ @property
1101
+ def signed(self) -> bool:
1102
+ """``True`` if this elementary item is signed."""
1103
+ if self._usage in (
1104
+ Usage.COMP_1,
1105
+ Usage.COMP_2,
1106
+ Usage.FLOAT_SHORT,
1107
+ Usage.FLOAT_LONG,
1108
+ Usage.INDEX,
1109
+ ):
1110
+ return True
1111
+ return self._signed
1112
+
1113
+ @property
1114
+ def usage(self) -> Usage:
1115
+ """Normalized :class:`~dataspector.coboller.constants.Usage` storage representation."""
1116
+ return self._usage
1117
+
1118
+ def decode_bytes(
1119
+ self,
1120
+ raw: bytes,
1121
+ encoding: str = "ascii",
1122
+ float_format: Literal["ieee", "ibm"] = "ieee",
1123
+ byteorder: Literal["big", "little"] = "big",
1124
+ as_decimal: bool = False,
1125
+ convert_compressed: bool = True,
1126
+ drop_converters: bool = False,
1127
+ dbcs_encoding: str = "utf-16-be",
1128
+ decimal_comma: bool = False,
1129
+ ) -> Any:
1130
+ """Decode *raw* bytes for this elementary item into a Python value.
1131
+
1132
+ Parameters
1133
+ ----------
1134
+ raw
1135
+ Byte slice corresponding to this elementary item.
1136
+ encoding
1137
+ Character encoding for ``DISPLAY`` items (e.g. ``'ascii'``, ``'cp500'``).
1138
+ float_format
1139
+ Floating-point representation for ``COMP-1``/``COMP-2`` (``'ieee'`` or ``'ibm'``).
1140
+ byteorder
1141
+ Byte order for binary and UTF-16 items (``'big'`` or ``'little'``).
1142
+ as_decimal
1143
+ Return :class:`decimal.Decimal` instead of ``float``/``int`` for fixed-point numbers.
1144
+ convert_compressed
1145
+ Decode binary/packed items when ``True``; return hex strings when ``False``.
1146
+ drop_converters
1147
+ Return raw decoded text or hex without numeric conversion.
1148
+ dbcs_encoding
1149
+ Character encoding for ``PIC G`` / ``DISPLAY-1`` DBCS items.
1150
+ decimal_comma
1151
+ Interpret commas as decimal separators per ``DECIMAL-POINT IS COMMA``.
1152
+
1153
+ Returns
1154
+ -------
1155
+ Any
1156
+ Decoded Python scalar (``str``, ``int``, ``float``, ``Decimal``, or ``None``).
1157
+ """
1158
+ if self.is_binary:
1159
+ if drop_converters or not convert_compressed:
1160
+ if self._usage is Usage.NATIONAL:
1161
+ return raw.decode("utf-16-be" if byteorder == "big" else "utf-16-le").rstrip()
1162
+ if self._usage is Usage.UTF_8:
1163
+ return raw.decode("utf-8", errors="replace").rstrip()
1164
+ if self._usage is Usage.DISPLAY_1:
1165
+ return raw.decode(dbcs_encoding, errors="replace").rstrip(" \u3000")
1166
+ return raw.hex()
1167
+ if self._usage is Usage.COMP_3:
1168
+ return comp3_unpack(raw, exponent=self._exponent, as_decimal=as_decimal)
1169
+ if self._usage is Usage.COMP_6:
1170
+ return comp6_unpack(raw, exponent=self._exponent, as_decimal=as_decimal)
1171
+ if self._usage in (Usage.COMP, Usage.COMP_5, Usage.COMP_X) or (
1172
+ self._usage in FIXED_BINARY_SIZES
1173
+ ):
1174
+ return comp_unpack(
1175
+ raw,
1176
+ signed=self._signed,
1177
+ byteorder=byteorder,
1178
+ exponent=self._exponent,
1179
+ as_decimal=as_decimal,
1180
+ )
1181
+ if self._usage in (Usage.COMP_1, Usage.COMP_2):
1182
+ return comp_float_unpack(raw, float_format=float_format, byteorder=byteorder)
1183
+ if self._usage in (Usage.FLOAT_SHORT, Usage.FLOAT_LONG):
1184
+ return comp_float_unpack(raw, float_format="ieee", byteorder=byteorder)
1185
+ if self._usage in (Usage.POINTER, Usage.PROCEDURE_POINTER, Usage.FUNCTION_POINTER):
1186
+ return int.from_bytes(raw, byteorder=byteorder, signed=False)
1187
+ if self._usage is Usage.INDEX:
1188
+ return int.from_bytes(raw, byteorder=byteorder, signed=True)
1189
+ if self._usage is Usage.NATIONAL:
1190
+ return raw.decode("utf-16-be" if byteorder == "big" else "utf-16-le").rstrip()
1191
+ if self._usage is Usage.UTF_8:
1192
+ return raw.decode("utf-8", errors="replace").rstrip()
1193
+ if self._usage is Usage.DISPLAY_1:
1194
+ return raw.decode(dbcs_encoding, errors="replace").rstrip(" \u3000")
1195
+ if self._usage is Usage.BIT:
1196
+ return bit_unpack(raw, max(len(self._expanded_pic), 1))
1197
+
1198
+ if drop_converters:
1199
+ return raw.decode(encoding, errors="replace").rstrip()
1200
+
1201
+ kind = self.kind
1202
+ if kind is DataCategory.EXTERNAL_FLOAT:
1203
+ return external_float_unpack(
1204
+ raw,
1205
+ expanded_pic=self._expanded_pic,
1206
+ encoding=encoding,
1207
+ decimal_comma=decimal_comma,
1208
+ )
1209
+ if kind in (DataCategory.INTEGER, DataCategory.DECIMAL):
1210
+ return zoned_unpack(
1211
+ raw,
1212
+ encoding=encoding,
1213
+ signed=self._signed,
1214
+ sign_mode=self._sign_mode,
1215
+ exponent=self._exponent,
1216
+ as_decimal=as_decimal,
1217
+ )
1218
+ if kind is DataCategory.NUMERIC_EDITED:
1219
+ text = raw.decode(encoding, errors="replace")
1220
+ if "/" in self._expanded_pic:
1221
+ return text.strip()
1222
+ return parse_numeric_edited(
1223
+ text,
1224
+ blank_when_zero=self.blank_when_zero,
1225
+ decimal_comma=decimal_comma,
1226
+ )
1227
+ return raw.decode(encoding, errors="replace").rstrip()
1228
+
1229
+ def pic_to_fwf_kwargs(self, convert_compressed: bool = True) -> list[dict[str, Any]]:
1230
+ """Convert this item into column descriptor dicts for :func:`pandas.read_fwf`.
1231
+
1232
+ Parameters
1233
+ ----------
1234
+ convert_compressed
1235
+ Attach a byte-level converter for binary/compressed storage when ``True``.
1236
+
1237
+ Returns
1238
+ -------
1239
+ list[dict[str, Any]]
1240
+ One descriptor per ``OCCURS`` repetition containing ``name``,
1241
+ ``pic``, and an optional ``converter`` callable.
1242
+ """
1243
+ dtype: dict[str, Any] = {"name": self.name, "pic": self._expanded_pic}
1244
+ if self.is_binary:
1245
+ if convert_compressed:
1246
+ dtype["converter"] = lambda val, f=self: f.decode_bytes(
1247
+ val.encode("latin-1") if isinstance(val, str) else val
1248
+ )
1249
+ else:
1250
+ logger.warning(
1251
+ "{} uses binary storage ({}); keeping raw representation for text FWF.",
1252
+ self.name,
1253
+ self._usage.value,
1254
+ )
1255
+ else:
1256
+ conv = self._make_text_converter()
1257
+ if conv is not None:
1258
+ dtype["converter"] = conv
1259
+
1260
+ if self.occurs == 1:
1261
+ return [dtype]
1262
+
1263
+ return [
1264
+ {**copy.deepcopy(dtype), "name": _occurs_name(self.name, n, self.occurs)}
1265
+ for n in range(self.occurs)
1266
+ ]
1267
+
1268
+ def _make_text_converter(self) -> Callable[[str], Any] | None:
1269
+ """Build a string converter callable for :func:`pandas.read_fwf`."""
1270
+ kind = self.kind
1271
+ if kind in (DataCategory.INTEGER, DataCategory.DECIMAL):
1272
+ signed = self._signed
1273
+ sign_mode = self._sign_mode
1274
+ exponent = self._exponent
1275
+
1276
+ def _convert_zoned(val: str) -> int | float | Decimal:
1277
+ return zoned_unpack(
1278
+ val.encode("ascii"),
1279
+ encoding="ascii",
1280
+ signed=signed,
1281
+ sign_mode=sign_mode,
1282
+ exponent=exponent,
1283
+ )
1284
+
1285
+ if not signed and exponent == 0:
1286
+ return int
1287
+ return _convert_zoned
1288
+
1289
+ if kind is DataCategory.EXTERNAL_FLOAT:
1290
+ pic = self._expanded_pic
1291
+
1292
+ def _convert_ext_float(val: str) -> float:
1293
+ return external_float_unpack(val.encode("ascii"), expanded_pic=pic)
1294
+
1295
+ return _convert_ext_float
1296
+
1297
+ if kind is DataCategory.NUMERIC_EDITED and "/" not in self._expanded_pic:
1298
+ bwz = self.blank_when_zero
1299
+
1300
+ def _convert_edited(val: str) -> float | None:
1301
+ return parse_numeric_edited(val, blank_when_zero=bwz)
1302
+
1303
+ return _convert_edited
1304
+
1305
+ return None
1306
+
1307
+ def _resolve_sign_mode(self, description: dict[str, Any]) -> SignMode:
1308
+ if not self._signed or self._usage is not Usage.DISPLAY:
1309
+ return SignMode.NONE
1310
+ raw_sign = description.get("sign_mode")
1311
+ if isinstance(raw_sign, SignMode):
1312
+ return raw_sign
1313
+ return SignMode.TRAILING_OVERPUNCH
1314
+
1315
+
1316
+ def parse_copybook(rawtext: str | pathlib.Path) -> list[GroupItem | ElementaryItem]:
1317
+ """Parse COBOL copybook source text or a file path into a record hierarchy.
1318
+
1319
+ Parameters
1320
+ ----------
1321
+ rawtext
1322
+ Raw copybook text or path to a copybook file.
1323
+
1324
+ Returns
1325
+ -------
1326
+ list[GroupItem | ElementaryItem]
1327
+ Top-level record definitions parsed from the copybook.
1328
+ """
1329
+ if isinstance(rawtext, pathlib.Path):
1330
+ text = rawtext.read_text(encoding="utf-8", errors="replace")
1331
+ elif "\n" not in rawtext and "\r" not in rawtext and len(rawtext) < 256:
1332
+ maybe_path = pathlib.Path(rawtext)
1333
+ text = (
1334
+ maybe_path.read_text(encoding="utf-8", errors="replace")
1335
+ if maybe_path.exists()
1336
+ else rawtext
1337
+ )
1338
+ else:
1339
+ text = rawtext
1340
+
1341
+ records: list[GroupItem | ElementaryItem] = []
1342
+ last_item: GroupItem | ElementaryItem | None = None
1343
+
1344
+ for stmt in extract_statements(text):
1345
+ desc = parse_statement(stmt)
1346
+ if desc is None:
1347
+ continue
1348
+
1349
+ level = int(desc["level"])
1350
+
1351
+ if level == 88:
1352
+ if last_item is not None:
1353
+ raw_val = desc.get("value") or ""
1354
+ last_item.conditions.append(
1355
+ ConditionName(
1356
+ name=desc["name"],
1357
+ raw_value=raw_val,
1358
+ values=parse_condition_values(raw_val),
1359
+ )
1360
+ )
1361
+ continue
1362
+
1363
+ if level == 66:
1364
+ continue
1365
+
1366
+ parent = _find_parent_group(records, level) if level not in (1, 77) else None
1367
+
1368
+ raw_usage = desc.get("usage")
1369
+ usage_enum = USAGE_ALIASES.get(raw_usage.upper()) if raw_usage else None
1370
+ if usage_enum is None and parent is not None and parent.usage is not None:
1371
+ desc["usage"] = parent.usage.value
1372
+ desc["comp"] = parent.usage.value
1373
+ usage_enum = parent.usage
1374
+
1375
+ is_elementary = (desc.get("pic") is not None) or (usage_enum in SELF_SIZED_USAGES)
1376
+ node: GroupItem | ElementaryItem = (
1377
+ ElementaryItem(desc) if is_elementary else GroupItem(desc)
1378
+ )
1379
+
1380
+ if level in (1, 77) or parent is None:
1381
+ records.append(node)
1382
+ else:
1383
+ parent.records.append(node)
1384
+
1385
+ if desc.get("redefines"):
1386
+ _register_redefines(records, parent, str(desc["redefines"]), node.name)
1387
+
1388
+ last_item = node
1389
+
1390
+ return records
1391
+
1392
+
1393
+ # Backward-compatible aliases
1394
+ CopyBook = Copybook
1395
+ Group = GroupItem
1396
+ Field = ElementaryItem
1397
+ Condition = ConditionName
1398
+ ResolvedField = ResolvedItem
1399
+
1400
+
1401
+ # ---------------------------------------------------------------------------
1402
+ # Internal Layout & Tree Helpers
1403
+ # ---------------------------------------------------------------------------
1404
+
1405
+
1406
+ def _deduplicate_names(names: list[str]) -> list[str]:
1407
+ """Ensure unique DataFrame column names while preserving first occurrences."""
1408
+ counts: dict[str, int] = {}
1409
+ unique: list[str] = []
1410
+ for name in names:
1411
+ idx = counts.get(name, 0)
1412
+ counts[name] = idx + 1
1413
+ unique.append(name if idx == 0 else f"{name}_{idx}")
1414
+ return unique
1415
+
1416
+
1417
+ def _find_parent_group(
1418
+ records: list[GroupItem | ElementaryItem],
1419
+ level: int,
1420
+ ) -> GroupItem | None:
1421
+ """Return the deepest open :class:`GroupItem` whose level is strictly less than *level*."""
1422
+ for rec in reversed(records):
1423
+ if isinstance(rec, GroupItem) and rec.level < level:
1424
+ deeper = _find_parent_group(rec.records, level)
1425
+ return deeper if deeper is not None else rec
1426
+ return None
1427
+
1428
+
1429
+ def _occurs_name(base: str, index: int, total: int) -> str:
1430
+ """Format an ``OCCURS`` column name (index 0 keeps the base name)."""
1431
+ if index == 0:
1432
+ return base
1433
+ width = 3 if total >= 100 else 2
1434
+ return f"{base}-{index:0{width}d}"
1435
+
1436
+
1437
+ def _register_redefines(
1438
+ records: list[GroupItem | ElementaryItem],
1439
+ parent: GroupItem | None,
1440
+ target_name: str,
1441
+ redefiner_name: str,
1442
+ ) -> None:
1443
+ """Record *redefiner_name* in the ``redefinedby`` list of *target_name*."""
1444
+ siblings = parent.records if parent is not None else records
1445
+ for rec in reversed(siblings):
1446
+ if rec.name == target_name:
1447
+ rec.redefinedby.append(redefiner_name)
1448
+ return