bitstring 4.2.0b1__tar.gz → 4.2.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {bitstring-4.2.0b1/bitstring.egg-info → bitstring-4.2.1}/PKG-INFO +15 -24
- {bitstring-4.2.0b1 → bitstring-4.2.1}/README.md +14 -23
- {bitstring-4.2.0b1 → bitstring-4.2.1}/bitstring/__init__.py +70 -15
- {bitstring-4.2.0b1 → bitstring-4.2.1}/bitstring/array_.py +81 -46
- {bitstring-4.2.0b1 → bitstring-4.2.1}/bitstring/bitarray_.py +1 -0
- {bitstring-4.2.0b1 → bitstring-4.2.1}/bitstring/bits.py +207 -207
- {bitstring-4.2.0b1 → bitstring-4.2.1}/bitstring/bitstore.py +35 -28
- bitstring-4.2.1/bitstring/bitstore_helpers.py +260 -0
- {bitstring-4.2.0b1 → bitstring-4.2.1}/bitstring/bitstream.py +29 -2
- bitstring-4.2.1/bitstring/bitstring_options.py +94 -0
- bitstring-4.2.1/bitstring/dtypes.py +400 -0
- bitstring-4.2.1/bitstring/fp8.py +97 -0
- bitstring-4.2.1/bitstring/luts.py +248 -0
- {bitstring-4.2.0b1 → bitstring-4.2.1}/bitstring/methods.py +2 -5
- bitstring-4.2.1/bitstring/mxfp.py +200 -0
- {bitstring-4.2.0b1 → bitstring-4.2.1}/bitstring/utils.py +22 -24
- {bitstring-4.2.0b1 → bitstring-4.2.1/bitstring.egg-info}/PKG-INFO +15 -24
- {bitstring-4.2.0b1 → bitstring-4.2.1}/bitstring.egg-info/SOURCES.txt +5 -1
- {bitstring-4.2.0b1 → bitstring-4.2.1}/pyproject.toml +1 -1
- {bitstring-4.2.0b1 → bitstring-4.2.1}/release_notes.txt +60 -11
- bitstring-4.2.1/tests/test_array.py +929 -0
- bitstring-4.2.1/tests/test_bitarray.py +992 -0
- bitstring-4.2.1/tests/test_bits.py +981 -0
- bitstring-4.2.1/tests/test_bitstore.py +94 -0
- bitstring-4.2.1/tests/test_bitstream.py +4178 -0
- {bitstring-4.2.0b1 → bitstring-4.2.1}/tests/test_bitstring.py +73 -74
- bitstring-4.2.1/tests/test_constbitstream.py +265 -0
- bitstring-4.2.1/tests/test_dtypes.py +134 -0
- bitstring-4.2.1/tests/test_fp8.py +295 -0
- bitstring-4.2.1/tests/test_mxfp.py +441 -0
- bitstring-4.2.1/tests/test_scaled_dtypes.py +13 -0
- bitstring-4.2.0b1/bitstring/bitstore_helpers.py +0 -252
- bitstring-4.2.0b1/bitstring/bitstring_options.py +0 -89
- bitstring-4.2.0b1/bitstring/dtypes.py +0 -244
- bitstring-4.2.0b1/bitstring/fp8.py +0 -93
- bitstring-4.2.0b1/tests/test_array.py +0 -915
- bitstring-4.2.0b1/tests/test_bitarray.py +0 -979
- bitstring-4.2.0b1/tests/test_bits.py +0 -974
- bitstring-4.2.0b1/tests/test_bitstore.py +0 -94
- bitstring-4.2.0b1/tests/test_bitstream.py +0 -4174
- bitstring-4.2.0b1/tests/test_constbitstream.py +0 -247
- bitstring-4.2.0b1/tests/test_dtypes.py +0 -86
- bitstring-4.2.0b1/tests/test_fp8.py +0 -220
- {bitstring-4.2.0b1 → bitstring-4.2.1}/LICENSE +0 -0
- {bitstring-4.2.0b1 → bitstring-4.2.1}/MANIFEST.in +0 -0
- {bitstring-4.2.0b1 → bitstring-4.2.1}/bitstring/__main__.py +0 -0
- {bitstring-4.2.0b1 → bitstring-4.2.1}/bitstring/exceptions.py +0 -0
- {bitstring-4.2.0b1 → bitstring-4.2.1}/bitstring/py.typed +0 -0
- {bitstring-4.2.0b1 → bitstring-4.2.1}/bitstring.egg-info/dependency_links.txt +0 -0
- {bitstring-4.2.0b1 → bitstring-4.2.1}/bitstring.egg-info/requires.txt +0 -0
- {bitstring-4.2.0b1 → bitstring-4.2.1}/bitstring.egg-info/top_level.txt +0 -0
- {bitstring-4.2.0b1 → bitstring-4.2.1}/setup.cfg +0 -0
- {bitstring-4.2.0b1 → bitstring-4.2.1}/tests/__init__.py +0 -0
- {bitstring-4.2.0b1 → bitstring-4.2.1}/tests/smalltestfile +0 -0
- {bitstring-4.2.0b1 → bitstring-4.2.1}/tests/test.m1v +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.1
|
|
2
2
|
Name: bitstring
|
|
3
|
-
Version: 4.2.
|
|
3
|
+
Version: 4.2.1
|
|
4
4
|
Summary: Simple construction, analysis and modification of binary data.
|
|
5
5
|
Author-email: Scott Griffiths <dr.scottgriffiths@gmail.com>
|
|
6
6
|
Project-URL: homepage, https://github.com/scott-griffiths/bitstring
|
|
@@ -37,17 +37,23 @@ It has been actively maintained since 2006.
|
|
|
37
37
|
[](https://app.codacy.com/gh/scott-griffiths/bitstring/dashboard?utm_source=gh&utm_medium=referral&utm_content=&utm_campaign=Badge_grade)
|
|
38
38
|
[](https://libraries.io/pypi/bitstring)
|
|
39
39
|
|
|
40
|
-
[](https://
|
|
40
|
+
[](https://www.pepy.tech/projects/bitstring)
|
|
41
41
|
[](https://pypistats.org/packages/bitstring)
|
|
42
42
|
|
|
43
43
|
|
|
44
44
|
News
|
|
45
45
|
----
|
|
46
|
-
**
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
*
|
|
46
|
+
**April 2024**: bitstring 4.2.1 released.
|
|
47
|
+
|
|
48
|
+
New in version 4.2:
|
|
49
|
+
|
|
50
|
+
* Dropped support for Python 3.7. Minimum version is now 3.8.
|
|
51
|
+
* A new `Dtype` class can be optionally used to specify types.
|
|
52
|
+
* The `bitstring.options` object is now the preferred method for changing module options.
|
|
53
|
+
* New `fromstring` method as another way to create bitstrings from formatted strings.
|
|
54
|
+
* More types can now be pretty printed.
|
|
55
|
+
* A range of 8-bit, 6-bit and even 4-bit floating point formats added (beta):
|
|
56
|
+
* Performance improvements.
|
|
51
57
|
|
|
52
58
|
See the [release notes](https://github.com/scott-griffiths/bitstring/blob/main/release_notes.txt) for details. Please let me know if you encounter any problems.
|
|
53
59
|
|
|
@@ -73,7 +79,7 @@ Some starting points are given below:
|
|
|
73
79
|
* [Quick Reference](https://bitstring.readthedocs.io/en/stable/quick_reference.html)
|
|
74
80
|
* [Full Reference](https://bitstring.readthedocs.io/en/stable/reference.html)
|
|
75
81
|
|
|
76
|
-
|
|
82
|
+
There is also an introductory walkthrough notebook on [binder](https://mybinder.org/v2/gh/scott-griffiths/bitstring/main?labpath=doc%2Fwalkthrough.ipynb).
|
|
77
83
|
|
|
78
84
|
Release Notes
|
|
79
85
|
-------------
|
|
@@ -137,20 +143,5 @@ Examples
|
|
|
137
143
|
Array('uint7', [45, 100, 15, 1])
|
|
138
144
|
|
|
139
145
|
|
|
140
|
-
Unit Tests
|
|
141
|
-
----------
|
|
142
|
-
|
|
143
|
-
The 700+ unit tests should all pass. They can be run from the root of the project with
|
|
144
|
-
|
|
145
|
-
python -m unittest
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
Credits
|
|
149
|
-
-------
|
|
150
|
-
|
|
151
|
-
Created in 2006 to help with ad hoc parsing and creation of compressed video files.
|
|
152
|
-
Maintained and expanded ever since as it became unexpectedly popular.
|
|
153
|
-
Thanks to all those who have contributed ideas and code (and bug reports) over the years.
|
|
154
|
-
|
|
155
146
|
|
|
156
|
-
<sub>Copyright (c) 2006 -
|
|
147
|
+
<sub>Copyright (c) 2006 - 2024 Scott Griffiths</sub>
|
|
@@ -13,17 +13,23 @@ It has been actively maintained since 2006.
|
|
|
13
13
|
[](https://app.codacy.com/gh/scott-griffiths/bitstring/dashboard?utm_source=gh&utm_medium=referral&utm_content=&utm_campaign=Badge_grade)
|
|
14
14
|
[](https://libraries.io/pypi/bitstring)
|
|
15
15
|
|
|
16
|
-
[](https://
|
|
16
|
+
[](https://www.pepy.tech/projects/bitstring)
|
|
17
17
|
[](https://pypistats.org/packages/bitstring)
|
|
18
18
|
|
|
19
19
|
|
|
20
20
|
News
|
|
21
21
|
----
|
|
22
|
-
**
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
*
|
|
22
|
+
**April 2024**: bitstring 4.2.1 released.
|
|
23
|
+
|
|
24
|
+
New in version 4.2:
|
|
25
|
+
|
|
26
|
+
* Dropped support for Python 3.7. Minimum version is now 3.8.
|
|
27
|
+
* A new `Dtype` class can be optionally used to specify types.
|
|
28
|
+
* The `bitstring.options` object is now the preferred method for changing module options.
|
|
29
|
+
* New `fromstring` method as another way to create bitstrings from formatted strings.
|
|
30
|
+
* More types can now be pretty printed.
|
|
31
|
+
* A range of 8-bit, 6-bit and even 4-bit floating point formats added (beta):
|
|
32
|
+
* Performance improvements.
|
|
27
33
|
|
|
28
34
|
See the [release notes](https://github.com/scott-griffiths/bitstring/blob/main/release_notes.txt) for details. Please let me know if you encounter any problems.
|
|
29
35
|
|
|
@@ -49,7 +55,7 @@ Some starting points are given below:
|
|
|
49
55
|
* [Quick Reference](https://bitstring.readthedocs.io/en/stable/quick_reference.html)
|
|
50
56
|
* [Full Reference](https://bitstring.readthedocs.io/en/stable/reference.html)
|
|
51
57
|
|
|
52
|
-
|
|
58
|
+
There is also an introductory walkthrough notebook on [binder](https://mybinder.org/v2/gh/scott-griffiths/bitstring/main?labpath=doc%2Fwalkthrough.ipynb).
|
|
53
59
|
|
|
54
60
|
Release Notes
|
|
55
61
|
-------------
|
|
@@ -113,20 +119,5 @@ Examples
|
|
|
113
119
|
Array('uint7', [45, 100, 15, 1])
|
|
114
120
|
|
|
115
121
|
|
|
116
|
-
Unit Tests
|
|
117
|
-
----------
|
|
118
|
-
|
|
119
|
-
The 700+ unit tests should all pass. They can be run from the root of the project with
|
|
120
|
-
|
|
121
|
-
python -m unittest
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
Credits
|
|
125
|
-
-------
|
|
126
|
-
|
|
127
|
-
Created in 2006 to help with ad hoc parsing and creation of compressed video files.
|
|
128
|
-
Maintained and expanded ever since as it became unexpectedly popular.
|
|
129
|
-
Thanks to all those who have contributed ideas and code (and bug reports) over the years.
|
|
130
|
-
|
|
131
122
|
|
|
132
|
-
<sub>Copyright (c) 2006 -
|
|
123
|
+
<sub>Copyright (c) 2006 - 2024 Scott Griffiths</sub>
|
|
@@ -10,15 +10,15 @@ BitArray -- A mutable container for binary data.
|
|
|
10
10
|
ConstBitStream -- An immutable container with streaming methods.
|
|
11
11
|
BitStream -- A mutable container with streaming methods.
|
|
12
12
|
Array -- An efficient list-like container where each item has a fixed-length binary format.
|
|
13
|
+
Dtype -- Encapsulate the data types used in the other classes.
|
|
13
14
|
|
|
14
15
|
Functions:
|
|
15
16
|
|
|
16
17
|
pack -- Create a BitStream from a format string.
|
|
17
18
|
|
|
18
19
|
Data:
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
lsb0 -- If True, the least significant bit (the final bit) is indexed as bit zero.
|
|
20
|
+
|
|
21
|
+
options -- Module-wide options.
|
|
22
22
|
|
|
23
23
|
Exceptions:
|
|
24
24
|
|
|
@@ -55,7 +55,7 @@ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
|
|
|
55
55
|
THE SOFTWARE.
|
|
56
56
|
"""
|
|
57
57
|
|
|
58
|
-
__version__ = "4.2.
|
|
58
|
+
__version__ = "4.2.1"
|
|
59
59
|
|
|
60
60
|
__author__ = "Scott Griffiths"
|
|
61
61
|
|
|
@@ -71,6 +71,12 @@ from .exceptions import Error, ReadError, InterpretError, ByteAlignError, Creati
|
|
|
71
71
|
from .dtypes import DtypeDefinition, dtype_register, Dtype
|
|
72
72
|
import types
|
|
73
73
|
from typing import List, Tuple, Literal
|
|
74
|
+
from .mxfp import decompress_luts as mxfp_decompress_luts
|
|
75
|
+
from .fp8 import decompress_luts as binary8_decompress_luts
|
|
76
|
+
|
|
77
|
+
# Decompress the LUTs for the exotic floating point formats
|
|
78
|
+
mxfp_decompress_luts()
|
|
79
|
+
binary8_decompress_luts()
|
|
74
80
|
|
|
75
81
|
# The Options class returns a singleton.
|
|
76
82
|
options = Options()
|
|
@@ -80,6 +86,7 @@ bytealigned = lsb0 = None
|
|
|
80
86
|
|
|
81
87
|
|
|
82
88
|
# An opaque way of adding module level properties. Taken from https://peps.python.org/pep-0549/
|
|
89
|
+
# This is now deprecated. Use the options object directly instead.
|
|
83
90
|
class _MyModuleType(types.ModuleType):
|
|
84
91
|
@property
|
|
85
92
|
def bytealigned(self) -> bool:
|
|
@@ -136,13 +143,40 @@ def float_bits2chars(bitlength: Literal[16, 32, 64]):
|
|
|
136
143
|
else:
|
|
137
144
|
return 24 # Empirical value
|
|
138
145
|
|
|
139
|
-
def
|
|
146
|
+
def p3binary_bits2chars(bitlength: Literal[8]):
|
|
140
147
|
return 19 # Empirical value
|
|
141
148
|
|
|
142
|
-
def
|
|
149
|
+
def p4binary_bits2chars(bitlength: Literal[8]):
|
|
143
150
|
# Found by looking at all the possible values
|
|
144
151
|
return 13 # Empirical value
|
|
145
152
|
|
|
153
|
+
def e4m3mxfp_bits2chars(bitlength: Literal[8]):
|
|
154
|
+
return 13
|
|
155
|
+
|
|
156
|
+
def e5m2mxfp_bits2chars(bitlength: Literal[8]):
|
|
157
|
+
return 19
|
|
158
|
+
|
|
159
|
+
def e3m2mxfp_bits2chars(bitlength: Literal[6]):
|
|
160
|
+
# Not sure what the best value is here. It's 7 without considering the scale that could be applied.
|
|
161
|
+
return 7
|
|
162
|
+
|
|
163
|
+
def e2m3mxfp_bits2chars(bitlength: Literal[6]):
|
|
164
|
+
# Not sure what the best value is here.
|
|
165
|
+
return 7
|
|
166
|
+
|
|
167
|
+
def e2m1mxfp_bits2chars(bitlength: Literal[4]):
|
|
168
|
+
# Not sure what the best value is here.
|
|
169
|
+
return 7
|
|
170
|
+
|
|
171
|
+
def e8m0mxfp_bits2chars(bitlength: Literal[8]):
|
|
172
|
+
# Can range same as float32
|
|
173
|
+
return 23
|
|
174
|
+
|
|
175
|
+
def mxint_bits2chars(bitlength: Literal[8]):
|
|
176
|
+
# Not sure what the best value is here.
|
|
177
|
+
return 10
|
|
178
|
+
|
|
179
|
+
|
|
146
180
|
def bfloat_bits2chars(bitlength: Literal[16]):
|
|
147
181
|
# Found by looking at all the possible values
|
|
148
182
|
return 23 # Empirical value
|
|
@@ -175,13 +209,9 @@ dtype_definitions = [
|
|
|
175
209
|
allowed_lengths=(0, 4, 8, ...), description="a hexadecimal string"),
|
|
176
210
|
DtypeDefinition('bin', Bits._setbin_safe, Bits._getbin, str, False, bin_bits2chars,
|
|
177
211
|
description="a binary string"),
|
|
178
|
-
DtypeDefinition('oct',Bits._setoct, Bits._getoct, str, False, oct_bits2chars,
|
|
212
|
+
DtypeDefinition('oct', Bits._setoct, Bits._getoct, str, False, oct_bits2chars,
|
|
179
213
|
allowed_lengths=(0, 3, 6, ...), description="an octal string"),
|
|
180
214
|
# Float types
|
|
181
|
-
DtypeDefinition('e5m2float', Bits._sete5m2float, Bits._gete5m2float, float, True, e5m2float_bits2chars,
|
|
182
|
-
allowed_lengths=(8,), description="an 8 bit float with e5m2float format"),
|
|
183
|
-
DtypeDefinition('e4m3float', Bits._sete4m3float, Bits._gete4m3float, float, True, e4m3float_bits2chars,
|
|
184
|
-
allowed_lengths=(8,), description="an 8 bit float with e4m3float format"),
|
|
185
215
|
DtypeDefinition('float', Bits._setfloatbe, Bits._getfloatbe, float, True, float_bits2chars,
|
|
186
216
|
allowed_lengths=(16, 32, 64), description="a big-endian floating point number"),
|
|
187
217
|
DtypeDefinition('floatle', Bits._setfloatle, Bits._getfloatle, float, True, float_bits2chars,
|
|
@@ -209,6 +239,26 @@ dtype_definitions = [
|
|
|
209
239
|
# Special case pad type
|
|
210
240
|
DtypeDefinition('pad', Bits._setpad, Bits._getpad, None, False, None,
|
|
211
241
|
description="a skipped section of padding"),
|
|
242
|
+
|
|
243
|
+
# MXFP and IEEE 8-bit float types
|
|
244
|
+
DtypeDefinition('p3binary', Bits._setp3binary, Bits._getp3binary, float, True, p3binary_bits2chars,
|
|
245
|
+
allowed_lengths=(8,), description="an 8 bit float with binary8p3 format"),
|
|
246
|
+
DtypeDefinition('p4binary', Bits._setp4binary, Bits._getp4binary, float, True, p4binary_bits2chars,
|
|
247
|
+
allowed_lengths=(8,), description="an 8 bit float with binary8p4 format"),
|
|
248
|
+
DtypeDefinition('e4m3mxfp', Bits._sete4m3mxfp, Bits._gete4m3mxfp, float, True, e4m3mxfp_bits2chars,
|
|
249
|
+
allowed_lengths=(8,), description="an 8 bit float with MXFP E4M3 format"),
|
|
250
|
+
DtypeDefinition('e5m2mxfp', Bits._sete5m2mxfp, Bits._gete5m2mxfp, float, True, e5m2mxfp_bits2chars,
|
|
251
|
+
allowed_lengths=(8,), description="an 8 bit float with MXFP E5M2 format"),
|
|
252
|
+
DtypeDefinition('e3m2mxfp', Bits._sete3m2mxfp, Bits._gete3m2mxfp, float, True, e3m2mxfp_bits2chars,
|
|
253
|
+
allowed_lengths=(6,), description="a 6 bit float with MXFP E3M2 format"),
|
|
254
|
+
DtypeDefinition('e2m3mxfp', Bits._sete2m3mxfp, Bits._gete2m3mxfp, float, True, e2m3mxfp_bits2chars,
|
|
255
|
+
allowed_lengths=(6,), description="a 6 bit float with MXFP E2M3 format"),
|
|
256
|
+
DtypeDefinition('e2m1mxfp', Bits._sete2m1mxfp, Bits._gete2m1mxfp, float, True, e2m1mxfp_bits2chars,
|
|
257
|
+
allowed_lengths=(4,), description="a 4 bit float with MXFP E2M1 format"),
|
|
258
|
+
DtypeDefinition('e8m0mxfp', Bits._sete8m0mxfp, Bits._gete8m0mxfp, float, False, e8m0mxfp_bits2chars,
|
|
259
|
+
allowed_lengths=(8,), description="an 8 bit float with MXFP E8M0 format"),
|
|
260
|
+
DtypeDefinition('mxint', Bits._setmxint, Bits._getmxint, float, True, mxint_bits2chars,
|
|
261
|
+
allowed_lengths=(8,), description="an 8 bit float with MXFP INT8 format"),
|
|
212
262
|
]
|
|
213
263
|
|
|
214
264
|
|
|
@@ -252,10 +302,15 @@ for alias in aliases:
|
|
|
252
302
|
property_docstrings = [f'{name} -- Interpret as {dtype_register[name].description}.' for name in dtype_register.names]
|
|
253
303
|
property_docstring = '\n '.join(property_docstrings)
|
|
254
304
|
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
305
|
+
# We can't be sure the docstrings are present, as it might be compiled without docstrings.
|
|
306
|
+
if Bits.__doc__ is not None:
|
|
307
|
+
Bits.__doc__ = Bits.__doc__.replace('[GENERATED_PROPERTY_DESCRIPTIONS]', property_docstring)
|
|
308
|
+
if BitArray.__doc__ is not None:
|
|
309
|
+
BitArray.__doc__ = BitArray.__doc__.replace('[GENERATED_PROPERTY_DESCRIPTIONS]', property_docstring)
|
|
310
|
+
if ConstBitStream.__doc__ is not None:
|
|
311
|
+
ConstBitStream.__doc__ = ConstBitStream.__doc__.replace('[GENERATED_PROPERTY_DESCRIPTIONS]', property_docstring)
|
|
312
|
+
if BitStream.__doc__ is not None:
|
|
313
|
+
BitStream.__doc__ = BitStream.__doc__.replace('[GENERATED_PROPERTY_DESCRIPTIONS]', property_docstring)
|
|
259
314
|
|
|
260
315
|
|
|
261
316
|
__all__ = ['ConstBitStream', 'BitStream', 'BitArray', 'Array',
|
|
@@ -70,6 +70,11 @@ class Array:
|
|
|
70
70
|
def __init__(self, dtype: Union[str, Dtype], initializer: Optional[Union[int, Array, array.array, Iterable, Bits, bytes, bytearray, memoryview, BinaryIO]] = None,
|
|
71
71
|
trailing_bits: Optional[BitsType] = None) -> None:
|
|
72
72
|
self.data = BitArray()
|
|
73
|
+
if isinstance(dtype, Dtype) and dtype.scale == 'auto':
|
|
74
|
+
if isinstance(initializer, (int, Bits, bytes, bytearray, memoryview, BinaryIO)):
|
|
75
|
+
raise TypeError("An Array with an 'auto' scale factor can only be created from an iterable of values.")
|
|
76
|
+
auto_scale = self._calculate_auto_scale(initializer, dtype.name, dtype.length)
|
|
77
|
+
dtype = Dtype(dtype.name, dtype.length, scale=auto_scale)
|
|
73
78
|
try:
|
|
74
79
|
self._set_dtype(dtype)
|
|
75
80
|
except ValueError as e:
|
|
@@ -87,6 +92,43 @@ class Array:
|
|
|
87
92
|
if trailing_bits is not None:
|
|
88
93
|
self.data += BitArray._create_from_bitstype(trailing_bits)
|
|
89
94
|
|
|
95
|
+
_largest_values = None
|
|
96
|
+
@staticmethod
|
|
97
|
+
def _calculate_auto_scale(initializer, name: str, length: Optional[int]) -> float:
|
|
98
|
+
# Now need to find the largest power of 2 representable with this format.
|
|
99
|
+
if Array._largest_values is None:
|
|
100
|
+
Array._largest_values = {
|
|
101
|
+
'mxint8': Bits('0b01111111').mxint8, # 1.0 + 63.0/64.0,
|
|
102
|
+
'e2m1mxfp4': Bits('0b0111').e2m1mxfp4, # 6.0
|
|
103
|
+
'e2m3mxfp6': Bits('0b011111').e2m3mxfp6, # 7.5
|
|
104
|
+
'e3m2mxfp6': Bits('0b011111').e3m2mxfp6, # 28.0
|
|
105
|
+
'e4m3mxfp8': Bits('0b01111110').e4m3mxfp8, # 448.0
|
|
106
|
+
'e5m2mxfp8': Bits('0b01111011').e5m2mxfp8, # 57344.0
|
|
107
|
+
'p4binary8': Bits('0b01111110').p4binary8, # 224.0
|
|
108
|
+
'p3binary8': Bits('0b01111110').p3binary8, # 49152.0
|
|
109
|
+
'float16': Bits('0x7bff').float16, # 65504.0
|
|
110
|
+
# The bfloat range is so large the scaling algorithm doesn't work well, so I'm disallowing it.
|
|
111
|
+
# 'bfloat16': Bits('0x7f7f').bfloat16, # 3.38953139e38,
|
|
112
|
+
}
|
|
113
|
+
if f'{name}{length}' in Array._largest_values.keys():
|
|
114
|
+
float_values = Array('float64', initializer).tolist()
|
|
115
|
+
max_float_value = max(abs(x) for x in float_values)
|
|
116
|
+
if max_float_value == 0:
|
|
117
|
+
# This special case isn't covered in the standard. I'm choosing to return no scale.
|
|
118
|
+
return 1.0
|
|
119
|
+
log2 = int(math.log2(max_float_value))
|
|
120
|
+
lp2 = int(math.log2(Array._largest_values[f'{name}{length}']))
|
|
121
|
+
lg_scale = log2 - lp2
|
|
122
|
+
# Saturate at values representable in E8M0 format.
|
|
123
|
+
if lg_scale > 127:
|
|
124
|
+
lg_scale = 127
|
|
125
|
+
elif lg_scale < -127:
|
|
126
|
+
lg_scale = -127
|
|
127
|
+
return 2 ** lg_scale
|
|
128
|
+
else:
|
|
129
|
+
raise ValueError(f"Can't calculate auto scale for format '{name}{length}'. "
|
|
130
|
+
f"This feature is only available for these formats: {list(Array._largest_values.keys())}.")
|
|
131
|
+
|
|
90
132
|
@property
|
|
91
133
|
def itemsize(self) -> int:
|
|
92
134
|
return self._dtype.length
|
|
@@ -96,19 +138,6 @@ class Array:
|
|
|
96
138
|
trailing_bit_length = len(self.data) % self._dtype.bitlength
|
|
97
139
|
return BitArray() if trailing_bit_length == 0 else self.data[-trailing_bit_length:]
|
|
98
140
|
|
|
99
|
-
# Converting array.array typecodes to our equivalents.
|
|
100
|
-
_array_typecodes: dict[str, str] = {'b': 'int8',
|
|
101
|
-
'B': 'uint8',
|
|
102
|
-
'h': 'intne16',
|
|
103
|
-
'H': 'uintne16',
|
|
104
|
-
'l': 'intne32',
|
|
105
|
-
'L': 'uintne32',
|
|
106
|
-
'q': 'intne64',
|
|
107
|
-
'Q': 'uintne64',
|
|
108
|
-
'e': 'floatne16',
|
|
109
|
-
'f': 'floatne32',
|
|
110
|
-
'd': 'floatne64'}
|
|
111
|
-
|
|
112
141
|
@property
|
|
113
142
|
def dtype(self) -> Dtype:
|
|
114
143
|
return self._dtype
|
|
@@ -120,7 +149,6 @@ class Array:
|
|
|
120
149
|
def _set_dtype(self, new_dtype: Union[str, Dtype]) -> None:
|
|
121
150
|
if isinstance(new_dtype, Dtype):
|
|
122
151
|
self._dtype = new_dtype
|
|
123
|
-
self._fmt = str(self._dtype)
|
|
124
152
|
else:
|
|
125
153
|
try:
|
|
126
154
|
dtype = Dtype(new_dtype)
|
|
@@ -133,14 +161,14 @@ class Array:
|
|
|
133
161
|
if dtype.length is None:
|
|
134
162
|
raise ValueError(f"A fixed length format is needed for an Array, received '{new_dtype}'.")
|
|
135
163
|
self._dtype = dtype
|
|
136
|
-
|
|
164
|
+
if self._dtype.scale == 'auto':
|
|
165
|
+
raise ValueError("A Dtype with an 'auto' scale factor can only be used when creating a new Array.")
|
|
137
166
|
|
|
138
167
|
def _create_element(self, value: ElementType) -> Bits:
|
|
139
168
|
"""Create Bits from value according to the token_name and token_length"""
|
|
140
|
-
b =
|
|
141
|
-
self._dtype.set_fn(b, value)
|
|
169
|
+
b = self._dtype.build(value)
|
|
142
170
|
if len(b) != self._dtype.length:
|
|
143
|
-
raise ValueError(f"The value {value!r} has the wrong length for the format '{self.
|
|
171
|
+
raise ValueError(f"The value {value!r} has the wrong length for the format '{self._dtype}'.")
|
|
144
172
|
return b
|
|
145
173
|
|
|
146
174
|
def __len__(self) -> int:
|
|
@@ -161,11 +189,11 @@ class Array:
|
|
|
161
189
|
d = BitArray()
|
|
162
190
|
for s in range(start * self._dtype.length, stop * self._dtype.length, step * self._dtype.length):
|
|
163
191
|
d.append(self.data[s: s + self._dtype.length])
|
|
164
|
-
a =
|
|
192
|
+
a = self.__class__(self._dtype)
|
|
165
193
|
a.data = d
|
|
166
194
|
return a
|
|
167
195
|
else:
|
|
168
|
-
a =
|
|
196
|
+
a = self.__class__(self._dtype)
|
|
169
197
|
a.data = self.data[start * self._dtype.length: stop * self._dtype.length]
|
|
170
198
|
return a
|
|
171
199
|
else:
|
|
@@ -234,15 +262,15 @@ class Array:
|
|
|
234
262
|
trailing_bit_length = len(self.data) % self._dtype.length
|
|
235
263
|
final_str = "" if trailing_bit_length == 0 else ", trailing_bits=" + repr(
|
|
236
264
|
self.data[-trailing_bit_length:])
|
|
237
|
-
return f"Array('{self.
|
|
265
|
+
return f"Array('{self._dtype}', {list_str}{final_str})"
|
|
238
266
|
|
|
239
267
|
def astype(self, dtype: Union[str, Dtype]) -> Array:
|
|
240
268
|
"""Return Array with elements of new dtype, initialised from current Array."""
|
|
241
|
-
new_array =
|
|
269
|
+
new_array = self.__class__(dtype, self.tolist())
|
|
242
270
|
return new_array
|
|
243
271
|
|
|
244
272
|
def tolist(self) -> List[ElementType]:
|
|
245
|
-
return
|
|
273
|
+
return [self._dtype.read_fn(self.data, start=start)
|
|
246
274
|
for start in range(0, len(self.data) - self._dtype.length + 1, self._dtype.length)]
|
|
247
275
|
|
|
248
276
|
def append(self, x: ElementType) -> None:
|
|
@@ -250,13 +278,13 @@ class Array:
|
|
|
250
278
|
raise ValueError("Cannot append to Array as its length is not a multiple of the format length.")
|
|
251
279
|
self.data += self._create_element(x)
|
|
252
280
|
|
|
253
|
-
def extend(self, iterable: Union[Array, array.array, Iterable]) -> None:
|
|
281
|
+
def extend(self, iterable: Union[Array, array.array, Iterable[Any]]) -> None:
|
|
254
282
|
if len(self.data) % self._dtype.length != 0:
|
|
255
283
|
raise ValueError(f"Cannot extend Array as its data length ({len(self.data)} bits) is not a multiple of the format length ({self._dtype.length} bits).")
|
|
256
284
|
if isinstance(iterable, Array):
|
|
257
285
|
if self._dtype.name != iterable._dtype.name or self._dtype.length != iterable._dtype.length:
|
|
258
286
|
raise TypeError(
|
|
259
|
-
f"Cannot extend an Array with format '{self.
|
|
287
|
+
f"Cannot extend an Array with format '{self._dtype}' from an Array of format '{iterable._dtype}'.")
|
|
260
288
|
# No need to iterate over the elements, we can just append the data
|
|
261
289
|
self.data.append(iterable.data)
|
|
262
290
|
elif isinstance(iterable, array.array):
|
|
@@ -264,10 +292,10 @@ class Array:
|
|
|
264
292
|
name_value = utils.parse_single_struct_token('=' + iterable.typecode)
|
|
265
293
|
if name_value is None:
|
|
266
294
|
raise ValueError(f"Cannot extend from array with typecode {iterable.typecode}.")
|
|
267
|
-
other_dtype = dtype_register.get_dtype(*name_value)
|
|
295
|
+
other_dtype = dtype_register.get_dtype(*name_value, scale=None)
|
|
268
296
|
if self._dtype.name != other_dtype.name or self._dtype.length != other_dtype.length:
|
|
269
297
|
raise ValueError(
|
|
270
|
-
f"Cannot extend an Array with format '{self.
|
|
298
|
+
f"Cannot extend an Array with format '{self._dtype}' from an array with typecode '{iterable.typecode}'.")
|
|
271
299
|
self.data += iterable.tobytes()
|
|
272
300
|
else:
|
|
273
301
|
if isinstance(iterable, str):
|
|
@@ -302,7 +330,7 @@ class Array:
|
|
|
302
330
|
"""
|
|
303
331
|
if self._dtype.length % 8 != 0:
|
|
304
332
|
raise ValueError(
|
|
305
|
-
f"byteswap can only be used for whole-byte elements. The '{self.
|
|
333
|
+
f"byteswap can only be used for whole-byte elements. The '{self._dtype}' format is {self._dtype.length} bits long.")
|
|
306
334
|
self.data.byteswap(self.itemsize // 8)
|
|
307
335
|
|
|
308
336
|
def count(self, value: ElementType) -> int:
|
|
@@ -368,7 +396,7 @@ class Array:
|
|
|
368
396
|
stream -- A TextIO object with a write() method. Defaults to sys.stdout.
|
|
369
397
|
|
|
370
398
|
"""
|
|
371
|
-
colour = Colour(options.
|
|
399
|
+
colour = Colour(not options.no_color)
|
|
372
400
|
sep = ' '
|
|
373
401
|
dtype2 = None
|
|
374
402
|
tidy_fmt = None
|
|
@@ -381,7 +409,6 @@ class Array:
|
|
|
381
409
|
if len(token_list) not in [1, 2]:
|
|
382
410
|
raise ValueError(f"Only one or two tokens can be used in an Array.pp() format - '{fmt}' has {len(token_list)} tokens.")
|
|
383
411
|
dtype1 = Dtype(*utils.parse_name_length_token(token_list[0]))
|
|
384
|
-
parameter_str = f"fmt='{fmt}'"
|
|
385
412
|
if len(token_list) == 2:
|
|
386
413
|
dtype2 = Dtype(*utils.parse_name_length_token(token_list[1]))
|
|
387
414
|
|
|
@@ -405,7 +432,7 @@ class Array:
|
|
|
405
432
|
data = self.data if trailing_bit_length == 0 else self.data[0: -trailing_bit_length]
|
|
406
433
|
length = len(self.data) // token_length
|
|
407
434
|
len_str = colour.green + str(length) + colour.off
|
|
408
|
-
stream.write(f"<
|
|
435
|
+
stream.write(f"<{self.__class__.__name__} {tidy_fmt}, length={len_str}, itemsize={token_length} bits, total data size={(len(self.data) + 7) // 8} bytes> [\n")
|
|
409
436
|
data._pp(dtype1, dtype2, token_length, width, sep, format_sep, show_offset, stream, False, token_length)
|
|
410
437
|
stream.write("]")
|
|
411
438
|
if trailing_bit_length != 0:
|
|
@@ -443,13 +470,13 @@ class Array:
|
|
|
443
470
|
start += self._dtype.length
|
|
444
471
|
|
|
445
472
|
def __copy__(self) -> Array:
|
|
446
|
-
a_copy =
|
|
473
|
+
a_copy = self.__class__(self._dtype)
|
|
447
474
|
a_copy.data = copy.copy(self.data)
|
|
448
475
|
return a_copy
|
|
449
476
|
|
|
450
477
|
def _apply_op_to_all_elements(self, op, value: Union[int, float, None], is_comparison: bool = False) -> Array:
|
|
451
478
|
"""Apply op with value to each element of the Array and return a new Array"""
|
|
452
|
-
new_array =
|
|
479
|
+
new_array = self.__class__('bool' if is_comparison else self._dtype)
|
|
453
480
|
new_data = BitArray()
|
|
454
481
|
failures = index = 0
|
|
455
482
|
msg = ''
|
|
@@ -505,7 +532,7 @@ class Array:
|
|
|
505
532
|
"""Apply op with value to each element of the Array as an unsigned integer in place."""
|
|
506
533
|
value = BitArray._create_from_bitstype(value)
|
|
507
534
|
if len(value) != self._dtype.length:
|
|
508
|
-
raise ValueError(f"Bitwise op needs a bitstring of length {self._dtype.length} to match format {self.
|
|
535
|
+
raise ValueError(f"Bitwise op needs a bitstring of length {self._dtype.length} to match format {self._dtype}.")
|
|
509
536
|
for start in range(0, len(self) * self._dtype.length, self._dtype.length):
|
|
510
537
|
self.data[start: start + self._dtype.length] = op(self.data[start: start + self._dtype.length], value)
|
|
511
538
|
return self
|
|
@@ -513,14 +540,16 @@ class Array:
|
|
|
513
540
|
def _apply_op_between_arrays(self, op, other: Array, is_comparison: bool = False) -> Array:
|
|
514
541
|
if len(self) != len(other):
|
|
515
542
|
msg = f"Cannot operate element-wise on Arrays with different lengths ({len(self)} and {len(other)})."
|
|
516
|
-
if op
|
|
517
|
-
msg += " Use extend()
|
|
543
|
+
if op in [operator.add, operator.iadd]:
|
|
544
|
+
msg += " Use extend() method to concatenate Arrays."
|
|
545
|
+
if op in [operator.eq, operator.ne]:
|
|
546
|
+
msg += " Use equals() method to compare Arrays for a single boolean result."
|
|
518
547
|
raise ValueError(msg)
|
|
519
548
|
if is_comparison:
|
|
520
549
|
new_type = dtype_register.get_dtype('bool', 1)
|
|
521
550
|
else:
|
|
522
551
|
new_type = self._promotetype(self._dtype, other._dtype)
|
|
523
|
-
new_array =
|
|
552
|
+
new_array = self.__class__(new_type)
|
|
524
553
|
new_data = BitArray()
|
|
525
554
|
failures = index = 0
|
|
526
555
|
msg = ''
|
|
@@ -553,7 +582,7 @@ class Array:
|
|
|
553
582
|
|
|
554
583
|
"""
|
|
555
584
|
def is_float(x): return x.return_type is float
|
|
556
|
-
def is_int(x): return x.return_type is int
|
|
585
|
+
def is_int(x): return x.return_type is int or x.return_type is bool
|
|
557
586
|
if is_float(type1) + is_int(type1) + is_float(type2) + is_int(type2) != 2:
|
|
558
587
|
raise ValueError(f"Only integer and floating point types can be combined - not '{type1}' and '{type2}'.")
|
|
559
588
|
# If same type choose the widest
|
|
@@ -722,15 +751,21 @@ class Array:
|
|
|
722
751
|
return self._apply_op_between_arrays(operator.le, other, is_comparison=True)
|
|
723
752
|
return self._apply_op_to_all_elements(operator.le, other, is_comparison=True)
|
|
724
753
|
|
|
725
|
-
def
|
|
726
|
-
if isinstance(other,
|
|
727
|
-
return self.
|
|
728
|
-
|
|
754
|
+
def _eq_ne(self, op, other: Any) -> Array:
|
|
755
|
+
if isinstance(other, (int, float, str, Bits)):
|
|
756
|
+
return self._apply_op_to_all_elements(op, other, is_comparison=True)
|
|
757
|
+
try:
|
|
758
|
+
other = self.__class__(self.dtype, other)
|
|
759
|
+
except:
|
|
760
|
+
return NotImplemented
|
|
761
|
+
finally:
|
|
762
|
+
return self._apply_op_between_arrays(op, other, is_comparison=True)
|
|
729
763
|
|
|
730
|
-
def
|
|
731
|
-
|
|
732
|
-
|
|
733
|
-
|
|
764
|
+
def __eq__(self, other: Any) -> Array:
|
|
765
|
+
return self._eq_ne(operator.eq, other)
|
|
766
|
+
|
|
767
|
+
def __ne__(self, other: Any) -> Array:
|
|
768
|
+
return self._eq_ne(operator.ne, other)
|
|
734
769
|
|
|
735
770
|
# Unary operators
|
|
736
771
|
|
|
@@ -42,6 +42,7 @@ class BitArray(Bits):
|
|
|
42
42
|
endswith() -- Return whether the bitstring ends with a sub-string.
|
|
43
43
|
find() -- Find a sub-bitstring in the current bitstring.
|
|
44
44
|
findall() -- Find all occurrences of a sub-bitstring in the current bitstring.
|
|
45
|
+
fromstring() -- Create a bitstring from a formatted string.
|
|
45
46
|
join() -- Join bitstrings together using current bitstring.
|
|
46
47
|
pp() -- Pretty print the bitstring.
|
|
47
48
|
rfind() -- Seek backwards to find a sub-bitstring.
|