dataspector 1.0.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- dataspector/__init__.py +64 -0
- dataspector/__main__.py +10 -0
- dataspector/_connectors.py +260 -0
- dataspector/_reporting.py +480 -0
- dataspector/checkers/__init__.py +85 -0
- dataspector/checkers/record.py +671 -0
- dataspector/checkers/source.py +344 -0
- dataspector/checkers/utils.py +342 -0
- dataspector/cli.py +207 -0
- dataspector/coboller/__init__.py +123 -0
- dataspector/coboller/codecs.py +841 -0
- dataspector/coboller/constants.py +324 -0
- dataspector/coboller/copybook.py +1448 -0
- dataspector/coboller/parser.py +201 -0
- dataspector/core.py +740 -0
- dataspector/py.typed +0 -0
- dataspector/settings.py +101 -0
- dataspector-1.0.0.dist-info/METADATA +181 -0
- dataspector-1.0.0.dist-info/RECORD +22 -0
- dataspector-1.0.0.dist-info/WHEEL +4 -0
- dataspector-1.0.0.dist-info/entry_points.txt +3 -0
- dataspector-1.0.0.dist-info/licenses/LICENSE +21 -0
|
@@ -0,0 +1,841 @@
|
|
|
1
|
+
"""Pure binary and text codecs for COBOL data representations (alphabetical)."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import math
|
|
6
|
+
import pathlib
|
|
7
|
+
import struct
|
|
8
|
+
from decimal import Decimal
|
|
9
|
+
from typing import Literal
|
|
10
|
+
|
|
11
|
+
import chardet
|
|
12
|
+
from loguru import logger
|
|
13
|
+
|
|
14
|
+
from dataspector.coboller.constants import (
|
|
15
|
+
ASCII_OVERPUNCH_NEG,
|
|
16
|
+
ASCII_OVERPUNCH_POS,
|
|
17
|
+
DIGIT_TO_ASCII_NEG,
|
|
18
|
+
DIGIT_TO_ASCII_POS,
|
|
19
|
+
EBCDIC_ENCODINGS,
|
|
20
|
+
PIC_REPEAT,
|
|
21
|
+
SignMode,
|
|
22
|
+
)
|
|
23
|
+
|
|
24
|
+
# ---------------------------------------------------------------------------
|
|
25
|
+
# Public Codec Functions (Alphabetical)
|
|
26
|
+
# ---------------------------------------------------------------------------
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
def bit_pack(bits: str) -> bytes:
|
|
30
|
+
"""Pack a ``'0'``/``'1'`` bit string into COBOL ``USAGE BIT`` bytes (zero-padded right).
|
|
31
|
+
|
|
32
|
+
Parameters
|
|
33
|
+
----------
|
|
34
|
+
bits
|
|
35
|
+
Non-empty string of ``'0'`` and ``'1'`` characters.
|
|
36
|
+
|
|
37
|
+
Returns
|
|
38
|
+
-------
|
|
39
|
+
bytes
|
|
40
|
+
MSB-first packed bytes right-padded with zero bits to a byte boundary.
|
|
41
|
+
"""
|
|
42
|
+
if not bits or any(ch not in ("0", "1") for ch in bits):
|
|
43
|
+
raise ValueError(f"Invalid bit string: {bits!r}")
|
|
44
|
+
padded = bits.ljust(((len(bits) + 7) // 8) * 8, "0")
|
|
45
|
+
return bytes(int(padded[i : i + 8], 2) for i in range(0, len(padded), 8))
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
def bit_unpack(raw: bytes, n_bits: int) -> str:
|
|
49
|
+
"""Decode a COBOL-2002 ``PIC 1(n) USAGE BIT`` byte sequence into a bit string.
|
|
50
|
+
|
|
51
|
+
Parameters
|
|
52
|
+
----------
|
|
53
|
+
raw
|
|
54
|
+
Raw packed bytes.
|
|
55
|
+
n_bits
|
|
56
|
+
Number of leading bits to extract.
|
|
57
|
+
|
|
58
|
+
Returns
|
|
59
|
+
-------
|
|
60
|
+
str
|
|
61
|
+
Bit string of length *n_bits* containing ``'0'`` and ``'1'`` characters.
|
|
62
|
+
"""
|
|
63
|
+
bits = "".join(f"{b:08b}" for b in raw)
|
|
64
|
+
return bits[:n_bits]
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
def comp3_pack(
|
|
68
|
+
value: int | float | Decimal,
|
|
69
|
+
digits: int,
|
|
70
|
+
decimals: int = 0,
|
|
71
|
+
exponent: int | None = None,
|
|
72
|
+
signed: bool = True,
|
|
73
|
+
) -> bytes:
|
|
74
|
+
"""Pack a numeric value into COBOL ``COMP-3`` (BCD packed decimal) bytes.
|
|
75
|
+
|
|
76
|
+
Parameters
|
|
77
|
+
----------
|
|
78
|
+
value
|
|
79
|
+
Numeric value to pack.
|
|
80
|
+
digits
|
|
81
|
+
Total number of decimal digit positions in the ``PIC`` clause.
|
|
82
|
+
decimals
|
|
83
|
+
Number of implied fractional decimal digits (ignored when *exponent* is set).
|
|
84
|
+
exponent
|
|
85
|
+
Optional explicit base-10 scaling exponent (e.g. ``-2`` for ``V99``, ``3`` for ``P(3)``).
|
|
86
|
+
signed
|
|
87
|
+
Encode sign nibble ``0xC``/``0xD`` when ``True``, or unsigned ``0xF`` when ``False``.
|
|
88
|
+
|
|
89
|
+
Returns
|
|
90
|
+
-------
|
|
91
|
+
bytes
|
|
92
|
+
Packed BCD byte sequence of length ``(digits // 2) + 1``.
|
|
93
|
+
"""
|
|
94
|
+
exp = -decimals if exponent is None else exponent
|
|
95
|
+
dec = Decimal(str(value))
|
|
96
|
+
negative = dec < 0
|
|
97
|
+
if negative and not signed:
|
|
98
|
+
raise ValueError("Cannot pack negative value into unsigned COMP-3 field.")
|
|
99
|
+
|
|
100
|
+
scaled = int(abs(dec).scaleb(-exp).to_integral_value())
|
|
101
|
+
n_bytes = (digits // 2) + 1
|
|
102
|
+
total_digit_slots = n_bytes * 2 - 1
|
|
103
|
+
digit_str = str(scaled).zfill(total_digit_slots)
|
|
104
|
+
if len(digit_str) > total_digit_slots:
|
|
105
|
+
raise ValueError(f"Value {value} exceeds {digits}-digit COMP-3 capacity.")
|
|
106
|
+
|
|
107
|
+
sign_nibble = 0x0D if negative else (0x0C if signed else 0x0F)
|
|
108
|
+
nibbles = [int(ch) for ch in digit_str] + [sign_nibble]
|
|
109
|
+
return bytes((nibbles[i] << 4) | nibbles[i + 1] for i in range(0, len(nibbles), 2))
|
|
110
|
+
|
|
111
|
+
|
|
112
|
+
def comp3_unpack(
|
|
113
|
+
raw: bytes,
|
|
114
|
+
decimals: int = 0,
|
|
115
|
+
exponent: int | None = None,
|
|
116
|
+
as_decimal: bool = False,
|
|
117
|
+
) -> int | float | Decimal:
|
|
118
|
+
"""Unpack a COBOL ``COMP-3`` / ``PACKED-DECIMAL`` byte sequence.
|
|
119
|
+
|
|
120
|
+
Parameters
|
|
121
|
+
----------
|
|
122
|
+
raw
|
|
123
|
+
Packed BCD bytes whose final low nibble carries the sign.
|
|
124
|
+
decimals
|
|
125
|
+
Number of implied fractional decimal digits (ignored when *exponent* is set).
|
|
126
|
+
exponent
|
|
127
|
+
Optional explicit base-10 scaling exponent.
|
|
128
|
+
as_decimal
|
|
129
|
+
Return a :class:`decimal.Decimal` when ``True`` instead of ``int``/``float``.
|
|
130
|
+
|
|
131
|
+
Returns
|
|
132
|
+
-------
|
|
133
|
+
int | float | Decimal
|
|
134
|
+
Decoded numeric value.
|
|
135
|
+
"""
|
|
136
|
+
if not raw:
|
|
137
|
+
raise ValueError("Cannot unpack empty COMP-3 byte sequence.")
|
|
138
|
+
|
|
139
|
+
exp = -decimals if exponent is None else exponent
|
|
140
|
+
digits: list[int] = []
|
|
141
|
+
for byte in raw[:-1]:
|
|
142
|
+
hi = (byte >> 4) & 0x0F
|
|
143
|
+
lo = byte & 0x0F
|
|
144
|
+
if hi > 9 or lo > 9:
|
|
145
|
+
raise ValueError(f"Invalid BCD nibble in COMP-3 byte 0x{byte:02X}.")
|
|
146
|
+
digits.append(hi)
|
|
147
|
+
digits.append(lo)
|
|
148
|
+
|
|
149
|
+
last = raw[-1]
|
|
150
|
+
last_digit = (last >> 4) & 0x0F
|
|
151
|
+
sign_nibble = last & 0x0F
|
|
152
|
+
if last_digit > 9:
|
|
153
|
+
raise ValueError(f"Invalid BCD digit in final COMP-3 byte 0x{last:02X}.")
|
|
154
|
+
digits.append(last_digit)
|
|
155
|
+
|
|
156
|
+
if sign_nibble in (0xC, 0xF, 0xA, 0xE):
|
|
157
|
+
sign = 1
|
|
158
|
+
elif sign_nibble in (0xD, 0xB):
|
|
159
|
+
sign = -1
|
|
160
|
+
else:
|
|
161
|
+
raise ValueError(f"Invalid COMP-3 sign nibble 0x{sign_nibble:X}.")
|
|
162
|
+
|
|
163
|
+
int_val = 0
|
|
164
|
+
for d in digits:
|
|
165
|
+
int_val = int_val * 10 + d
|
|
166
|
+
int_val *= sign
|
|
167
|
+
return scale_integer(int_val, exp, as_decimal=as_decimal)
|
|
168
|
+
|
|
169
|
+
|
|
170
|
+
def comp3decompress(
|
|
171
|
+
x: bytes,
|
|
172
|
+
decimals: int = 0,
|
|
173
|
+
as_decimal: bool = False,
|
|
174
|
+
) -> int | float | Decimal:
|
|
175
|
+
"""Backward-compatible alias for :func:`comp3_unpack`.
|
|
176
|
+
|
|
177
|
+
Parameters
|
|
178
|
+
----------
|
|
179
|
+
x
|
|
180
|
+
Packed BCD bytes.
|
|
181
|
+
decimals
|
|
182
|
+
Number of implied fractional decimal digits.
|
|
183
|
+
as_decimal
|
|
184
|
+
Return a :class:`decimal.Decimal` when ``True``.
|
|
185
|
+
|
|
186
|
+
Returns
|
|
187
|
+
-------
|
|
188
|
+
int | float | Decimal
|
|
189
|
+
Decoded numeric value.
|
|
190
|
+
"""
|
|
191
|
+
return comp3_unpack(x, decimals=decimals, as_decimal=as_decimal)
|
|
192
|
+
|
|
193
|
+
|
|
194
|
+
def comp6_pack(
|
|
195
|
+
value: int | float | Decimal,
|
|
196
|
+
digits: int,
|
|
197
|
+
decimals: int = 0,
|
|
198
|
+
exponent: int | None = None,
|
|
199
|
+
) -> bytes:
|
|
200
|
+
"""Pack a non-negative value into COBOL ``COMP-6`` (unsigned packed decimal) bytes.
|
|
201
|
+
|
|
202
|
+
Parameters
|
|
203
|
+
----------
|
|
204
|
+
value
|
|
205
|
+
Non-negative numeric value to pack.
|
|
206
|
+
digits
|
|
207
|
+
Total number of decimal digit positions in the ``PIC`` clause.
|
|
208
|
+
decimals
|
|
209
|
+
Number of implied fractional decimal digits (ignored when *exponent* is set).
|
|
210
|
+
exponent
|
|
211
|
+
Optional explicit base-10 scaling exponent.
|
|
212
|
+
|
|
213
|
+
Returns
|
|
214
|
+
-------
|
|
215
|
+
bytes
|
|
216
|
+
Signless packed BCD byte sequence of length ``(digits + 1) // 2``.
|
|
217
|
+
"""
|
|
218
|
+
exp = -decimals if exponent is None else exponent
|
|
219
|
+
dec = Decimal(str(value))
|
|
220
|
+
if dec < 0:
|
|
221
|
+
raise ValueError("COMP-6 is unsigned and cannot pack negative values.")
|
|
222
|
+
scaled = int(dec.scaleb(-exp).to_integral_value())
|
|
223
|
+
n_bytes = (digits + 1) // 2
|
|
224
|
+
slot_count = n_bytes * 2
|
|
225
|
+
digit_str = str(scaled).zfill(slot_count)
|
|
226
|
+
if len(digit_str) > slot_count:
|
|
227
|
+
raise ValueError(f"Value {value} exceeds {digits}-digit COMP-6 capacity.")
|
|
228
|
+
nibbles = [int(ch) for ch in digit_str]
|
|
229
|
+
return bytes((nibbles[i] << 4) | nibbles[i + 1] for i in range(0, len(nibbles), 2))
|
|
230
|
+
|
|
231
|
+
|
|
232
|
+
def comp6_unpack(
|
|
233
|
+
raw: bytes,
|
|
234
|
+
decimals: int = 0,
|
|
235
|
+
exponent: int | None = None,
|
|
236
|
+
as_decimal: bool = False,
|
|
237
|
+
) -> int | float | Decimal:
|
|
238
|
+
"""Unpack a COBOL ``COMP-6`` (unsigned packed decimal, no sign nibble) byte sequence.
|
|
239
|
+
|
|
240
|
+
Parameters
|
|
241
|
+
----------
|
|
242
|
+
raw
|
|
243
|
+
Signless packed BCD bytes (two decimal digits per byte).
|
|
244
|
+
decimals
|
|
245
|
+
Number of implied fractional decimal digits (ignored when *exponent* is set).
|
|
246
|
+
exponent
|
|
247
|
+
Optional explicit base-10 scaling exponent.
|
|
248
|
+
as_decimal
|
|
249
|
+
Return a :class:`decimal.Decimal` when ``True``.
|
|
250
|
+
|
|
251
|
+
Returns
|
|
252
|
+
-------
|
|
253
|
+
int | float | Decimal
|
|
254
|
+
Decoded non-negative numeric value.
|
|
255
|
+
"""
|
|
256
|
+
if not raw:
|
|
257
|
+
raise ValueError("Cannot unpack empty COMP-6 byte sequence.")
|
|
258
|
+
exp = -decimals if exponent is None else exponent
|
|
259
|
+
int_val = 0
|
|
260
|
+
for byte in raw:
|
|
261
|
+
hi = (byte >> 4) & 0x0F
|
|
262
|
+
lo = byte & 0x0F
|
|
263
|
+
if hi > 9 or lo > 9:
|
|
264
|
+
raise ValueError(f"Invalid BCD nibble in COMP-6 byte 0x{byte:02X}.")
|
|
265
|
+
int_val = int_val * 100 + hi * 10 + lo
|
|
266
|
+
return scale_integer(int_val, exp, as_decimal=as_decimal)
|
|
267
|
+
|
|
268
|
+
|
|
269
|
+
def comp_float_pack(
|
|
270
|
+
value: float,
|
|
271
|
+
size: Literal[4, 8] = 4,
|
|
272
|
+
float_format: Literal["ieee", "ibm"] = "ieee",
|
|
273
|
+
byteorder: Literal["big", "little"] = "big",
|
|
274
|
+
) -> bytes:
|
|
275
|
+
"""Pack a Python ``float`` into ``COMP-1`` (4-byte) or ``COMP-2`` (8-byte) bytes.
|
|
276
|
+
|
|
277
|
+
Parameters
|
|
278
|
+
----------
|
|
279
|
+
value
|
|
280
|
+
Floating-point number to encode.
|
|
281
|
+
size
|
|
282
|
+
Target storage width in bytes (``4`` for ``COMP-1``, ``8`` for ``COMP-2``).
|
|
283
|
+
float_format
|
|
284
|
+
Binary floating-point format (``'ieee'`` or ``'ibm'`` HFP).
|
|
285
|
+
byteorder
|
|
286
|
+
Byte order for IEEE 754 encoding (``'big'`` or ``'little'``).
|
|
287
|
+
|
|
288
|
+
Returns
|
|
289
|
+
-------
|
|
290
|
+
bytes
|
|
291
|
+
Encoded 4-byte or 8-byte floating-point sequence.
|
|
292
|
+
"""
|
|
293
|
+
if size not in (4, 8):
|
|
294
|
+
raise ValueError(f"COMP-1/COMP-2 size must be 4 or 8, got {size}.")
|
|
295
|
+
if float_format == "ibm":
|
|
296
|
+
return float_to_ibm_hfp(value, size=size)
|
|
297
|
+
prefix = ">" if byteorder == "big" else "<"
|
|
298
|
+
fmt = f"{prefix}{'f' if size == 4 else 'd'}"
|
|
299
|
+
return struct.pack(fmt, value)
|
|
300
|
+
|
|
301
|
+
|
|
302
|
+
def comp_float_unpack(
|
|
303
|
+
raw: bytes,
|
|
304
|
+
float_format: Literal["ieee", "ibm"] = "ieee",
|
|
305
|
+
byteorder: Literal["big", "little"] = "big",
|
|
306
|
+
) -> float:
|
|
307
|
+
"""Unpack a COBOL ``COMP-1`` (4-byte) or ``COMP-2`` (8-byte) floating-point value.
|
|
308
|
+
|
|
309
|
+
Parameters
|
|
310
|
+
----------
|
|
311
|
+
raw
|
|
312
|
+
4-byte or 8-byte floating-point representation.
|
|
313
|
+
float_format
|
|
314
|
+
Binary floating-point format (``'ieee'`` or ``'ibm'`` HFP).
|
|
315
|
+
byteorder
|
|
316
|
+
Byte order for IEEE 754 decoding (``'big'`` or ``'little'``).
|
|
317
|
+
|
|
318
|
+
Returns
|
|
319
|
+
-------
|
|
320
|
+
float
|
|
321
|
+
Decoded Python floating-point number.
|
|
322
|
+
"""
|
|
323
|
+
if len(raw) not in (4, 8):
|
|
324
|
+
raise ValueError(f"COMP-1/COMP-2 expects 4 or 8 bytes, got {len(raw)}.")
|
|
325
|
+
if float_format == "ibm":
|
|
326
|
+
return ibm_hfp_to_float(raw)
|
|
327
|
+
prefix = ">" if byteorder == "big" else "<"
|
|
328
|
+
fmt = f"{prefix}{'f' if len(raw) == 4 else 'd'}"
|
|
329
|
+
return float(struct.unpack(fmt, raw)[0])
|
|
330
|
+
|
|
331
|
+
|
|
332
|
+
def comp_pack(
|
|
333
|
+
value: int | float | Decimal,
|
|
334
|
+
byte_length: int,
|
|
335
|
+
signed: bool = True,
|
|
336
|
+
byteorder: Literal["big", "little"] = "big",
|
|
337
|
+
exponent: int = 0,
|
|
338
|
+
) -> bytes:
|
|
339
|
+
"""Pack a numeric value into COBOL binary ``COMP`` bytes.
|
|
340
|
+
|
|
341
|
+
Parameters
|
|
342
|
+
----------
|
|
343
|
+
value
|
|
344
|
+
Numeric value to pack.
|
|
345
|
+
byte_length
|
|
346
|
+
Exact width in bytes of the target binary integer field.
|
|
347
|
+
signed
|
|
348
|
+
Use two's-complement signed encoding when ``True``, unsigned when ``False``.
|
|
349
|
+
byteorder
|
|
350
|
+
Byte order (``'big'`` or ``'little'``).
|
|
351
|
+
exponent
|
|
352
|
+
Base-10 scaling exponent applied before integer packing.
|
|
353
|
+
|
|
354
|
+
Returns
|
|
355
|
+
-------
|
|
356
|
+
bytes
|
|
357
|
+
Binary integer byte sequence of length *byte_length*.
|
|
358
|
+
"""
|
|
359
|
+
scaled = int(Decimal(str(value)).scaleb(-exponent).to_integral_value())
|
|
360
|
+
return scaled.to_bytes(byte_length, byteorder=byteorder, signed=signed)
|
|
361
|
+
|
|
362
|
+
|
|
363
|
+
def comp_unpack(
|
|
364
|
+
raw: bytes,
|
|
365
|
+
signed: bool = True,
|
|
366
|
+
byteorder: Literal["big", "little"] = "big",
|
|
367
|
+
exponent: int = 0,
|
|
368
|
+
as_decimal: bool = False,
|
|
369
|
+
) -> int | float | Decimal:
|
|
370
|
+
"""Unpack a COBOL binary integer (``COMP``, ``COMP-4``, ``COMP-5``, ``COMP-X``, ``BINARY-*``).
|
|
371
|
+
|
|
372
|
+
Parameters
|
|
373
|
+
----------
|
|
374
|
+
raw
|
|
375
|
+
Raw binary integer bytes.
|
|
376
|
+
signed
|
|
377
|
+
Interpret as two's-complement signed when ``True``, unsigned when ``False``.
|
|
378
|
+
byteorder
|
|
379
|
+
Byte order (``'big'`` or ``'little'``).
|
|
380
|
+
exponent
|
|
381
|
+
Base-10 scaling exponent applied to the unpacked integer.
|
|
382
|
+
as_decimal
|
|
383
|
+
Return a :class:`decimal.Decimal` when ``True``.
|
|
384
|
+
|
|
385
|
+
Returns
|
|
386
|
+
-------
|
|
387
|
+
int | float | Decimal
|
|
388
|
+
Decoded numeric value.
|
|
389
|
+
"""
|
|
390
|
+
if not raw:
|
|
391
|
+
raise ValueError("Cannot unpack empty binary COMP byte sequence.")
|
|
392
|
+
int_val = int.from_bytes(raw, byteorder=byteorder, signed=signed)
|
|
393
|
+
return scale_integer(int_val, exponent, as_decimal=as_decimal)
|
|
394
|
+
|
|
395
|
+
|
|
396
|
+
def expand_pic(pic: str) -> str:
|
|
397
|
+
"""Expand repeat counts in a COBOL ``PIC`` string (e.g. ``'S9(5)V9(2)'`` -> ``'S99999V99'``).
|
|
398
|
+
|
|
399
|
+
Parameters
|
|
400
|
+
----------
|
|
401
|
+
pic
|
|
402
|
+
Raw COBOL ``PICTURE`` clause string.
|
|
403
|
+
|
|
404
|
+
Returns
|
|
405
|
+
-------
|
|
406
|
+
str
|
|
407
|
+
Uppercase expanded ``PICTURE`` string with repeat factors unfolded.
|
|
408
|
+
"""
|
|
409
|
+
picstr = pic.strip().upper()
|
|
410
|
+
while (match := PIC_REPEAT.search(picstr)) is not None:
|
|
411
|
+
repeat = int(match.group("repeat"))
|
|
412
|
+
section = match.group("section")
|
|
413
|
+
picstr = PIC_REPEAT.sub(section * repeat, picstr, count=1)
|
|
414
|
+
return picstr
|
|
415
|
+
|
|
416
|
+
|
|
417
|
+
def external_float_unpack(
|
|
418
|
+
raw: bytes,
|
|
419
|
+
expanded_pic: str = "",
|
|
420
|
+
encoding: str = "ascii",
|
|
421
|
+
decimal_comma: bool = False,
|
|
422
|
+
) -> float:
|
|
423
|
+
"""Decode a COBOL external floating-point ``USAGE DISPLAY`` item.
|
|
424
|
+
|
|
425
|
+
Parameters
|
|
426
|
+
----------
|
|
427
|
+
raw
|
|
428
|
+
Encoded character bytes representing the external floating-point value.
|
|
429
|
+
expanded_pic
|
|
430
|
+
Expanded ``PICTURE`` clause used to infer implied ``V`` decimal placement.
|
|
431
|
+
encoding
|
|
432
|
+
Character encoding of *raw* (e.g. ``'ascii'``, ``'cp500'``).
|
|
433
|
+
decimal_comma
|
|
434
|
+
Interpret commas as decimal points when ``True``.
|
|
435
|
+
|
|
436
|
+
Returns
|
|
437
|
+
-------
|
|
438
|
+
float
|
|
439
|
+
Decoded floating-point value.
|
|
440
|
+
"""
|
|
441
|
+
text = raw.decode(encoding, errors="replace").strip()
|
|
442
|
+
if not text:
|
|
443
|
+
return 0.0
|
|
444
|
+
if decimal_comma:
|
|
445
|
+
text = text.replace(",", ".")
|
|
446
|
+
|
|
447
|
+
if "." not in text and "V" in expanded_pic.upper():
|
|
448
|
+
mantissa_pic = expanded_pic.upper().split("E", 1)[0]
|
|
449
|
+
frac_digits = mantissa_pic.split("V", 1)[1].count("9")
|
|
450
|
+
mant_str, _, exp_str = text.upper().partition("E")
|
|
451
|
+
sign = -1 if mant_str.startswith("-") else 1
|
|
452
|
+
digits_only = mant_str.lstrip("+-")
|
|
453
|
+
mant_val = sign * int(digits_only or "0") * (10.0 ** (-frac_digits))
|
|
454
|
+
exp_val = int(exp_str) if exp_str else 0
|
|
455
|
+
return mant_val * (10.0**exp_val)
|
|
456
|
+
|
|
457
|
+
return float(text)
|
|
458
|
+
|
|
459
|
+
|
|
460
|
+
def float_to_ibm_hfp(value: float, size: Literal[4, 8] = 4) -> bytes:
|
|
461
|
+
"""Encode a Python ``float`` into 4-byte or 8-byte IBM Hexadecimal Floating-Point.
|
|
462
|
+
|
|
463
|
+
Parameters
|
|
464
|
+
----------
|
|
465
|
+
value
|
|
466
|
+
Floating-point value to encode.
|
|
467
|
+
size
|
|
468
|
+
Target HFP storage size in bytes (``4`` short or ``8`` long).
|
|
469
|
+
|
|
470
|
+
Returns
|
|
471
|
+
-------
|
|
472
|
+
bytes
|
|
473
|
+
Big-endian IBM S/360 / z/Architecture HFP bytes.
|
|
474
|
+
"""
|
|
475
|
+
if size not in (4, 8):
|
|
476
|
+
raise ValueError(f"IBM HFP size must be 4 or 8, got {size}.")
|
|
477
|
+
if value == 0.0:
|
|
478
|
+
return b"\x00" * size
|
|
479
|
+
|
|
480
|
+
sign_bit = 0x80 if value < 0 else 0x00
|
|
481
|
+
abs_val = abs(value)
|
|
482
|
+
_m2, e2 = math.frexp(abs_val)
|
|
483
|
+
e16 = math.ceil(e2 / 4)
|
|
484
|
+
frac = abs_val / math.ldexp(1.0, 4 * e16)
|
|
485
|
+
while frac >= 1.0:
|
|
486
|
+
frac /= 16.0
|
|
487
|
+
e16 += 1
|
|
488
|
+
while 0.0 < frac < 1.0 / 16.0:
|
|
489
|
+
frac *= 16.0
|
|
490
|
+
e16 -= 1
|
|
491
|
+
|
|
492
|
+
biased = e16 + 64
|
|
493
|
+
if not (0 <= biased <= 127):
|
|
494
|
+
raise OverflowError(f"Value {value} out of IBM HFP exponent range.")
|
|
495
|
+
|
|
496
|
+
frac_bits = (size - 1) * 8
|
|
497
|
+
mantissa = round(frac * (1 << frac_bits))
|
|
498
|
+
if mantissa == (1 << frac_bits):
|
|
499
|
+
mantissa >>= 4
|
|
500
|
+
biased += 1
|
|
501
|
+
if biased > 127:
|
|
502
|
+
raise OverflowError(f"Value {value} out of IBM HFP exponent range.")
|
|
503
|
+
|
|
504
|
+
first_byte = sign_bit | (biased & 0x7F)
|
|
505
|
+
return bytes([first_byte]) + mantissa.to_bytes(size - 1, byteorder="big", signed=False)
|
|
506
|
+
|
|
507
|
+
|
|
508
|
+
def ibm_hfp_to_float(raw: bytes) -> float:
|
|
509
|
+
"""Decode a 32-bit or 64-bit IBM Hexadecimal Floating-Point (HFP) byte sequence.
|
|
510
|
+
|
|
511
|
+
Parameters
|
|
512
|
+
----------
|
|
513
|
+
raw
|
|
514
|
+
4-byte (short) or 8-byte (long) big-endian IBM HFP bytes.
|
|
515
|
+
|
|
516
|
+
Returns
|
|
517
|
+
-------
|
|
518
|
+
float
|
|
519
|
+
Decoded Python floating-point value.
|
|
520
|
+
"""
|
|
521
|
+
if len(raw) not in (4, 8):
|
|
522
|
+
raise ValueError(f"IBM HFP requires 4 or 8 bytes, got {len(raw)}.")
|
|
523
|
+
if all(b == 0 for b in raw):
|
|
524
|
+
return 0.0
|
|
525
|
+
|
|
526
|
+
sign = -1.0 if (raw[0] & 0x80) else 1.0
|
|
527
|
+
exponent = (raw[0] & 0x7F) - 64
|
|
528
|
+
frac_bits = (len(raw) - 1) * 8
|
|
529
|
+
mantissa_int = int.from_bytes(raw[1:], byteorder="big", signed=False)
|
|
530
|
+
if mantissa_int == 0:
|
|
531
|
+
return 0.0
|
|
532
|
+
fraction = mantissa_int / (1 << frac_bits)
|
|
533
|
+
return sign * fraction * math.ldexp(1.0, 4 * exponent)
|
|
534
|
+
|
|
535
|
+
|
|
536
|
+
def parse_numeric_edited(
|
|
537
|
+
text: str,
|
|
538
|
+
blank_when_zero: bool = False,
|
|
539
|
+
decimal_comma: bool = False,
|
|
540
|
+
) -> float | None:
|
|
541
|
+
"""Parse a COBOL numeric-edited display string (e.g. ``' $1,234.50-'``) to ``float``.
|
|
542
|
+
|
|
543
|
+
Parameters
|
|
544
|
+
----------
|
|
545
|
+
text
|
|
546
|
+
Formatted numeric-edited string containing currency, suppression, or sign tokens.
|
|
547
|
+
blank_when_zero
|
|
548
|
+
Return ``0.0`` instead of ``None`` for all-blank fields when ``True``.
|
|
549
|
+
decimal_comma
|
|
550
|
+
Interpret commas as decimal points and periods as thousands separators when ``True``.
|
|
551
|
+
|
|
552
|
+
Returns
|
|
553
|
+
-------
|
|
554
|
+
float | None
|
|
555
|
+
Parsed numeric value, or ``None`` for blank fields when *blank_when_zero* is ``False``.
|
|
556
|
+
"""
|
|
557
|
+
cleaned = text.strip()
|
|
558
|
+
if not cleaned:
|
|
559
|
+
return 0.0 if blank_when_zero else None
|
|
560
|
+
|
|
561
|
+
negative = False
|
|
562
|
+
upper = cleaned.upper()
|
|
563
|
+
if upper.endswith(("CR", "DB")):
|
|
564
|
+
negative = True
|
|
565
|
+
cleaned = cleaned[:-2].strip()
|
|
566
|
+
elif cleaned.endswith("-"):
|
|
567
|
+
negative = True
|
|
568
|
+
cleaned = cleaned[:-1].strip()
|
|
569
|
+
elif cleaned.endswith("+"):
|
|
570
|
+
cleaned = cleaned[:-1].strip()
|
|
571
|
+
|
|
572
|
+
if decimal_comma:
|
|
573
|
+
cleaned = cleaned.replace(".", "").replace(",", ".")
|
|
574
|
+
else:
|
|
575
|
+
cleaned = cleaned.replace(",", "")
|
|
576
|
+
|
|
577
|
+
cleaned = cleaned.replace("$", "").replace("*", "").replace(" ", "")
|
|
578
|
+
if cleaned.startswith("-"):
|
|
579
|
+
negative = True
|
|
580
|
+
cleaned = cleaned[1:]
|
|
581
|
+
elif cleaned.startswith("+"):
|
|
582
|
+
cleaned = cleaned[1:]
|
|
583
|
+
|
|
584
|
+
if not cleaned:
|
|
585
|
+
return 0.0
|
|
586
|
+
val = float(cleaned)
|
|
587
|
+
return -val if negative else val
|
|
588
|
+
|
|
589
|
+
|
|
590
|
+
def predict_encoding(file_path: str | pathlib.Path, n_lines: int = 20) -> str:
|
|
591
|
+
"""Predict a file's character encoding via :mod:`chardet`, falling back to ``'ascii'``.
|
|
592
|
+
|
|
593
|
+
Parameters
|
|
594
|
+
----------
|
|
595
|
+
file_path
|
|
596
|
+
Path to the file to inspect.
|
|
597
|
+
n_lines
|
|
598
|
+
Number of leading lines to sample for detection.
|
|
599
|
+
|
|
600
|
+
Returns
|
|
601
|
+
-------
|
|
602
|
+
str
|
|
603
|
+
Detected encoding name, or ``'ascii'`` if detection is inconclusive.
|
|
604
|
+
"""
|
|
605
|
+
path = pathlib.Path(file_path)
|
|
606
|
+
with path.open(mode="rb") as fh:
|
|
607
|
+
rawdata = b"".join(fh.readline() for _ in range(n_lines))
|
|
608
|
+
detected = chardet.detect(rawdata).get("encoding")
|
|
609
|
+
logger.trace("Predicted encoding for {}: {}", path, detected or "ascii")
|
|
610
|
+
return detected or "ascii"
|
|
611
|
+
|
|
612
|
+
|
|
613
|
+
def zoned_pack(
|
|
614
|
+
value: int | float | Decimal,
|
|
615
|
+
digits: int,
|
|
616
|
+
encoding: str = "ascii",
|
|
617
|
+
signed: bool = False,
|
|
618
|
+
sign_mode: SignMode = SignMode.TRAILING_OVERPUNCH,
|
|
619
|
+
exponent: int = 0,
|
|
620
|
+
) -> bytes:
|
|
621
|
+
"""Pack a numeric value into COBOL ``USAGE DISPLAY`` zoned decimal bytes.
|
|
622
|
+
|
|
623
|
+
Parameters
|
|
624
|
+
----------
|
|
625
|
+
value
|
|
626
|
+
Numeric value to encode.
|
|
627
|
+
digits
|
|
628
|
+
Total number of digit positions in the ``PIC`` clause.
|
|
629
|
+
encoding
|
|
630
|
+
Target character encoding (EBCDIC codepage or ASCII-compatible encoding).
|
|
631
|
+
signed
|
|
632
|
+
Encode sign information when ``True``.
|
|
633
|
+
sign_mode
|
|
634
|
+
Sign placement/encoding convention (leading/trailing, overpunch/separate).
|
|
635
|
+
exponent
|
|
636
|
+
Base-10 scaling exponent applied before encoding digits.
|
|
637
|
+
|
|
638
|
+
Returns
|
|
639
|
+
-------
|
|
640
|
+
bytes
|
|
641
|
+
Encoded zoned decimal byte sequence.
|
|
642
|
+
"""
|
|
643
|
+
dec = Decimal(str(value))
|
|
644
|
+
negative = dec < 0
|
|
645
|
+
if negative and not signed:
|
|
646
|
+
raise ValueError("Cannot pack negative value into unsigned zoned decimal field.")
|
|
647
|
+
|
|
648
|
+
scaled = int(abs(dec).scaleb(-exponent).to_integral_value())
|
|
649
|
+
digit_str = str(scaled).zfill(digits)
|
|
650
|
+
if len(digit_str) > digits:
|
|
651
|
+
raise ValueError(f"Value {value} exceeds {digits}-digit zoned decimal capacity.")
|
|
652
|
+
|
|
653
|
+
if signed and sign_mode is SignMode.LEADING_SEPARATE:
|
|
654
|
+
return (("-" if negative else "+") + digit_str).encode(encoding)
|
|
655
|
+
if signed and sign_mode is SignMode.TRAILING_SEPARATE:
|
|
656
|
+
return (digit_str + ("-" if negative else "+")).encode(encoding)
|
|
657
|
+
|
|
658
|
+
if is_ebcdic(encoding):
|
|
659
|
+
out = bytearray(0xF0 | int(ch) for ch in digit_str)
|
|
660
|
+
if signed:
|
|
661
|
+
idx = 0 if sign_mode is SignMode.LEADING_OVERPUNCH else len(out) - 1
|
|
662
|
+
zone = 0xD0 if negative else 0xC0
|
|
663
|
+
out[idx] = zone | (out[idx] & 0x0F)
|
|
664
|
+
return bytes(out)
|
|
665
|
+
|
|
666
|
+
chars = list(digit_str)
|
|
667
|
+
if signed:
|
|
668
|
+
idx = 0 if sign_mode is SignMode.LEADING_OVERPUNCH else len(chars) - 1
|
|
669
|
+
chars[idx] = (DIGIT_TO_ASCII_NEG if negative else DIGIT_TO_ASCII_POS)[chars[idx]]
|
|
670
|
+
return "".join(chars).encode(encoding)
|
|
671
|
+
|
|
672
|
+
|
|
673
|
+
def zoned_unpack(
|
|
674
|
+
raw: bytes,
|
|
675
|
+
encoding: str = "ascii",
|
|
676
|
+
signed: bool = False,
|
|
677
|
+
sign_mode: SignMode = SignMode.TRAILING_OVERPUNCH,
|
|
678
|
+
exponent: int = 0,
|
|
679
|
+
as_decimal: bool = False,
|
|
680
|
+
) -> int | float | Decimal:
|
|
681
|
+
"""Unpack a COBOL ``USAGE DISPLAY`` zoned decimal byte sequence.
|
|
682
|
+
|
|
683
|
+
Parameters
|
|
684
|
+
----------
|
|
685
|
+
raw
|
|
686
|
+
Encoded zoned decimal bytes.
|
|
687
|
+
encoding
|
|
688
|
+
Character encoding of *raw* (EBCDIC or ASCII-compatible).
|
|
689
|
+
signed
|
|
690
|
+
Decode sign overpunch or separate sign character when ``True``.
|
|
691
|
+
sign_mode
|
|
692
|
+
Sign placement/encoding convention (leading/trailing, overpunch/separate).
|
|
693
|
+
exponent
|
|
694
|
+
Base-10 scaling exponent applied to the decoded integer.
|
|
695
|
+
as_decimal
|
|
696
|
+
Return a :class:`decimal.Decimal` when ``True``.
|
|
697
|
+
|
|
698
|
+
Returns
|
|
699
|
+
-------
|
|
700
|
+
int | float | Decimal
|
|
701
|
+
Decoded numeric value.
|
|
702
|
+
"""
|
|
703
|
+
if not raw:
|
|
704
|
+
raise ValueError("Cannot unpack empty zoned decimal bytes.")
|
|
705
|
+
|
|
706
|
+
if signed and sign_mode in (SignMode.LEADING_SEPARATE, SignMode.TRAILING_SEPARATE):
|
|
707
|
+
text = raw.decode(encoding).strip()
|
|
708
|
+
if not text:
|
|
709
|
+
return scale_integer(0, exponent, as_decimal=as_decimal)
|
|
710
|
+
if sign_mode is SignMode.TRAILING_SEPARATE:
|
|
711
|
+
sign = -1 if text.endswith("-") else 1
|
|
712
|
+
digits_str = text[:-1].strip() if text[-1] in "+-" else text
|
|
713
|
+
else:
|
|
714
|
+
sign = -1 if text.startswith("-") else 1
|
|
715
|
+
digits_str = text[1:].strip() if text[0] in "+-" else text
|
|
716
|
+
return scale_integer(sign * int(digits_str or "0"), exponent, as_decimal=as_decimal)
|
|
717
|
+
|
|
718
|
+
if is_ebcdic(encoding):
|
|
719
|
+
if all(b == 0x40 for b in raw):
|
|
720
|
+
return scale_integer(0, exponent, as_decimal=as_decimal)
|
|
721
|
+
sign = 1
|
|
722
|
+
sign_idx = 0 if (signed and sign_mode is SignMode.LEADING_OVERPUNCH) else len(raw) - 1
|
|
723
|
+
digits_list: list[str] = []
|
|
724
|
+
for idx, b in enumerate(raw):
|
|
725
|
+
if b == 0x40:
|
|
726
|
+
continue
|
|
727
|
+
zone = (b >> 4) & 0x0F
|
|
728
|
+
digit = b & 0x0F
|
|
729
|
+
if digit > 9:
|
|
730
|
+
raise ValueError(f"Invalid EBCDIC zoned digit byte 0x{b:02X}.")
|
|
731
|
+
if signed and idx == sign_idx:
|
|
732
|
+
if zone in (0xD, 0xB):
|
|
733
|
+
sign = -1
|
|
734
|
+
elif zone in (0xC, 0xF, 0xA, 0xE):
|
|
735
|
+
sign = 1
|
|
736
|
+
else:
|
|
737
|
+
raise ValueError(f"Invalid EBCDIC sign zone 0x{zone:X} in byte 0x{b:02X}.")
|
|
738
|
+
digits_list.append(str(digit))
|
|
739
|
+
int_val = sign * int("".join(digits_list) or "0")
|
|
740
|
+
return scale_integer(int_val, exponent, as_decimal=as_decimal)
|
|
741
|
+
|
|
742
|
+
text = raw.decode(encoding).strip()
|
|
743
|
+
if not text:
|
|
744
|
+
return scale_integer(0, exponent, as_decimal=as_decimal)
|
|
745
|
+
|
|
746
|
+
sign = 1
|
|
747
|
+
if text[0] in "+-":
|
|
748
|
+
sign = -1 if text[0] == "-" else 1
|
|
749
|
+
text = text[1:].strip()
|
|
750
|
+
elif text[-1] in "+-":
|
|
751
|
+
sign = -1 if text[-1] == "-" else 1
|
|
752
|
+
text = text[:-1].strip()
|
|
753
|
+
elif signed:
|
|
754
|
+
pos = 0 if sign_mode is SignMode.LEADING_OVERPUNCH else len(text) - 1
|
|
755
|
+
ch = text[pos]
|
|
756
|
+
if ch in ASCII_OVERPUNCH_POS:
|
|
757
|
+
text = text[:pos] + ASCII_OVERPUNCH_POS[ch] + text[pos + 1 :]
|
|
758
|
+
elif ch in ASCII_OVERPUNCH_NEG:
|
|
759
|
+
sign = -1
|
|
760
|
+
text = text[:pos] + ASCII_OVERPUNCH_NEG[ch] + text[pos + 1 :]
|
|
761
|
+
|
|
762
|
+
return scale_integer(sign * int(text or "0"), exponent, as_decimal=as_decimal)
|
|
763
|
+
|
|
764
|
+
|
|
765
|
+
# ---------------------------------------------------------------------------
|
|
766
|
+
# Internal Shared Codec Helpers
|
|
767
|
+
# ---------------------------------------------------------------------------
|
|
768
|
+
|
|
769
|
+
|
|
770
|
+
def compute_decimal_exponent(expanded_pic: str) -> int:
|
|
771
|
+
"""Return base-10 exponent implied by ``V`` and ``P`` symbols in a PIC string.
|
|
772
|
+
|
|
773
|
+
Parameters
|
|
774
|
+
----------
|
|
775
|
+
expanded_pic
|
|
776
|
+
Expanded uppercase COBOL ``PICTURE`` string.
|
|
777
|
+
|
|
778
|
+
Returns
|
|
779
|
+
-------
|
|
780
|
+
int
|
|
781
|
+
Base-10 exponent (negative for fractional digits, positive for trailing ``P``).
|
|
782
|
+
"""
|
|
783
|
+
body = expanded_pic.lstrip("S")
|
|
784
|
+
if "V" in body:
|
|
785
|
+
before_v, after_v = body.split("V", 1)
|
|
786
|
+
if "P" in before_v:
|
|
787
|
+
return before_v.count("P")
|
|
788
|
+
return -(after_v.count("9") + after_v.count("P"))
|
|
789
|
+
if body.startswith("P"):
|
|
790
|
+
return -(body.count("P") + body.count("9"))
|
|
791
|
+
if body.endswith("P"):
|
|
792
|
+
return body.count("P")
|
|
793
|
+
return 0
|
|
794
|
+
|
|
795
|
+
|
|
796
|
+
def is_ebcdic(encoding: str) -> bool:
|
|
797
|
+
"""Return ``True`` if *encoding* refers to an EBCDIC codepage.
|
|
798
|
+
|
|
799
|
+
Parameters
|
|
800
|
+
----------
|
|
801
|
+
encoding
|
|
802
|
+
Character encoding name (e.g. ``'cp500'``, ``'cp037'``, ``'ascii'``).
|
|
803
|
+
|
|
804
|
+
Returns
|
|
805
|
+
-------
|
|
806
|
+
bool
|
|
807
|
+
``True`` for recognized EBCDIC codepages, ``False`` otherwise.
|
|
808
|
+
"""
|
|
809
|
+
norm = encoding.lower().replace("-", "").replace("_", "")
|
|
810
|
+
return norm in EBCDIC_ENCODINGS or norm.startswith(("cp037", "cp500", "cp114", "ebcdic"))
|
|
811
|
+
|
|
812
|
+
|
|
813
|
+
def scale_integer(
|
|
814
|
+
value: int,
|
|
815
|
+
exponent: int,
|
|
816
|
+
as_decimal: bool = False,
|
|
817
|
+
) -> int | float | Decimal:
|
|
818
|
+
"""Scale an integer by ``10 ** exponent``.
|
|
819
|
+
|
|
820
|
+
Parameters
|
|
821
|
+
----------
|
|
822
|
+
value
|
|
823
|
+
Unscaled integer value.
|
|
824
|
+
exponent
|
|
825
|
+
Base-10 exponent to apply.
|
|
826
|
+
as_decimal
|
|
827
|
+
Return an exact :class:`decimal.Decimal` when ``True``.
|
|
828
|
+
|
|
829
|
+
Returns
|
|
830
|
+
-------
|
|
831
|
+
int | float | Decimal
|
|
832
|
+
Scaled numeric value.
|
|
833
|
+
"""
|
|
834
|
+
if exponent == 0:
|
|
835
|
+
return Decimal(value) if as_decimal else value
|
|
836
|
+
scaled = Decimal(value).scaleb(exponent)
|
|
837
|
+
if as_decimal:
|
|
838
|
+
return scaled
|
|
839
|
+
if exponent > 0:
|
|
840
|
+
return int(scaled)
|
|
841
|
+
return float(scaled)
|