bw-processing 1.5__tar.gz → 2.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {bw_processing-1.5/src/bw_processing.egg-info → bw_processing-2.0}/PKG-INFO +37 -12
- {bw_processing-1.5 → bw_processing-2.0}/README.md +33 -6
- {bw_processing-1.5 → bw_processing-2.0}/pyproject.toml +3 -6
- {bw_processing-1.5 → bw_processing-2.0}/src/bw_processing/__init__.py +2 -3
- {bw_processing-1.5 → bw_processing-2.0}/src/bw_processing/array_creation.py +10 -51
- {bw_processing-1.5 → bw_processing-2.0}/src/bw_processing/constants.py +5 -5
- {bw_processing-1.5 → bw_processing-2.0}/src/bw_processing/datapackage.py +213 -53
- {bw_processing-1.5 → bw_processing-2.0}/src/bw_processing/io_pyarrow_helpers.py +4 -1
- {bw_processing-1.5 → bw_processing-2.0}/src/bw_processing/matrix_entry.py +84 -12
- {bw_processing-1.5 → bw_processing-2.0}/src/bw_processing/merging.py +3 -3
- {bw_processing-1.5 → bw_processing-2.0}/src/bw_processing/utils.py +24 -3
- {bw_processing-1.5 → bw_processing-2.0/src/bw_processing.egg-info}/PKG-INFO +37 -12
- {bw_processing-1.5 → bw_processing-2.0}/src/bw_processing.egg-info/requires.txt +2 -2
- bw_processing-2.0/tests/test_array_creation.py +44 -0
- {bw_processing-1.5 → bw_processing-2.0}/tests/test_datapackage.py +199 -0
- {bw_processing-1.5 → bw_processing-2.0}/tests/test_filtered_datapackage.py +43 -0
- {bw_processing-1.5 → bw_processing-2.0}/tests/test_io_parquet_helpers.py +0 -8
- {bw_processing-1.5 → bw_processing-2.0}/tests/test_io_pyarrow_helpers.py +15 -0
- {bw_processing-1.5 → bw_processing-2.0}/tests/test_matrix_entry.py +317 -3
- {bw_processing-1.5 → bw_processing-2.0}/tests/test_merging.py +9 -0
- {bw_processing-1.5 → bw_processing-2.0}/tests/test_utils.py +17 -5
- bw_processing-1.5/tests/test_array_creation.py +0 -97
- {bw_processing-1.5 → bw_processing-2.0}/LICENSE +0 -0
- {bw_processing-1.5 → bw_processing-2.0}/MANIFEST.in +0 -0
- {bw_processing-1.5 → bw_processing-2.0}/setup.cfg +0 -0
- {bw_processing-1.5 → bw_processing-2.0}/src/bw_processing/errors.py +0 -0
- {bw_processing-1.5 → bw_processing-2.0}/src/bw_processing/examples/__init__.py +0 -0
- {bw_processing-1.5 → bw_processing-2.0}/src/bw_processing/examples/datapackage_1/another name.indices.parquet +0 -0
- {bw_processing-1.5 → bw_processing-2.0}/src/bw_processing/examples/datapackage_1/sa-data-vector-from-dict.data.parquet +0 -0
- {bw_processing-1.5 → bw_processing-2.0}/src/bw_processing/examples/datapackage_1/sa-data-vector-from-dict.distributions.parquet +0 -0
- {bw_processing-1.5 → bw_processing-2.0}/src/bw_processing/examples/datapackage_1/sa-data-vector-from-dict.flip.parquet +0 -0
- {bw_processing-1.5 → bw_processing-2.0}/src/bw_processing/examples/datapackage_1/sa-data-vector-from-dict.indices.parquet +0 -0
- {bw_processing-1.5 → bw_processing-2.0}/src/bw_processing/examples/datapackage_1/some name.data.parquet +0 -0
- {bw_processing-1.5 → bw_processing-2.0}/src/bw_processing/examples/datapackage_1/some name.flip.parquet +0 -0
- {bw_processing-1.5 → bw_processing-2.0}/src/bw_processing/examples/datapackage_1/some name.indices.parquet +0 -0
- {bw_processing-1.5 → bw_processing-2.0}/src/bw_processing/examples/datapackage_2.zip +0 -0
- {bw_processing-1.5 → bw_processing-2.0}/src/bw_processing/examples/interfaces.py +0 -0
- {bw_processing-1.5 → bw_processing-2.0}/src/bw_processing/examples/parquet_files.py +0 -0
- {bw_processing-1.5 → bw_processing-2.0}/src/bw_processing/examples/simple.zip +0 -0
- {bw_processing-1.5 → bw_processing-2.0}/src/bw_processing/filesystem.py +0 -0
- {bw_processing-1.5 → bw_processing-2.0}/src/bw_processing/indexing.py +0 -0
- {bw_processing-1.5 → bw_processing-2.0}/src/bw_processing/io_helpers.py +0 -0
- {bw_processing-1.5 → bw_processing-2.0}/src/bw_processing/io_parquet_helpers.py +0 -0
- {bw_processing-1.5 → bw_processing-2.0}/src/bw_processing/param_labels.py +0 -0
- {bw_processing-1.5 → bw_processing-2.0}/src/bw_processing/proxies.py +0 -0
- {bw_processing-1.5 → bw_processing-2.0}/src/bw_processing/unique_fields.py +0 -0
- {bw_processing-1.5 → bw_processing-2.0}/src/bw_processing.egg-info/SOURCES.txt +0 -0
- {bw_processing-1.5 → bw_processing-2.0}/src/bw_processing.egg-info/dependency_links.txt +0 -0
- {bw_processing-1.5 → bw_processing-2.0}/src/bw_processing.egg-info/top_level.txt +0 -0
- {bw_processing-1.5 → bw_processing-2.0}/tests/test_filesystem.py +0 -0
- {bw_processing-1.5 → bw_processing-2.0}/tests/test_indexing.py +0 -0
- {bw_processing-1.5 → bw_processing-2.0}/tests/test_integration.py +0 -0
- {bw_processing-1.5 → bw_processing-2.0}/tests/test_interfaces.py +0 -0
- {bw_processing-1.5 → bw_processing-2.0}/tests/test_loading.py +0 -0
- {bw_processing-1.5 → bw_processing-2.0}/tests/test_params.py +0 -0
- {bw_processing-1.5 → bw_processing-2.0}/tests/test_proxies.py +0 -0
- {bw_processing-1.5 → bw_processing-2.0}/tests/test_unique_fields.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: bw_processing
|
|
3
|
-
Version:
|
|
3
|
+
Version: 2.0
|
|
4
4
|
Summary: Foo
|
|
5
5
|
Author-email: Chris Mutel <cmutel@gmail.com>
|
|
6
6
|
Maintainer-email: Chris Mutel <cmutel@gmail.com>
|
|
@@ -10,20 +10,18 @@ Project-URL: tracker, https://github.com/brightway-lca/bw_processing/issues
|
|
|
10
10
|
Classifier: Development Status :: 4 - Beta
|
|
11
11
|
Classifier: Programming Language :: Python
|
|
12
12
|
Classifier: Programming Language :: Python :: 3
|
|
13
|
-
Classifier: Programming Language :: Python :: 3.9
|
|
14
|
-
Classifier: Programming Language :: Python :: 3.10
|
|
15
13
|
Classifier: Programming Language :: Python :: 3.11
|
|
16
14
|
Classifier: Programming Language :: Python :: 3.12
|
|
17
15
|
Classifier: Natural Language :: English
|
|
18
16
|
Classifier: Operating System :: OS Independent
|
|
19
17
|
Classifier: Topic :: Scientific/Engineering
|
|
20
|
-
Requires-Python: >=3.
|
|
18
|
+
Requires-Python: >=3.11
|
|
21
19
|
Description-Content-Type: text/markdown
|
|
22
20
|
License-File: LICENSE
|
|
23
|
-
Requires-Dist: fsspec
|
|
21
|
+
Requires-Dist: fsspec>=2023.3.0
|
|
24
22
|
Requires-Dist: jsonschema>=4.0
|
|
25
23
|
Requires-Dist: morefs
|
|
26
|
-
Requires-Dist: numpy
|
|
24
|
+
Requires-Dist: numpy>=2.0
|
|
27
25
|
Requires-Dist: pandas
|
|
28
26
|
Provides-Extra: testing
|
|
29
27
|
Requires-Dist: bw_processing; extra == "testing"
|
|
@@ -211,12 +209,12 @@ print(data_obj.url)
|
|
|
211
209
|
|
|
212
210
|
### Scale arrays
|
|
213
211
|
|
|
214
|
-
Any resource group (persistent or dynamic, vector or array) can carry an optional `
|
|
212
|
+
Any resource group (persistent or dynamic, vector or array) can carry an optional `rescale_array`: a one-dimensional float array of the same length as `indices_array`. Each element is a multiplicative factor applied to the corresponding data value before it is inserted into the matrix. The factor is applied to both static and stochastically-sampled values. A value of `1.0` leaves the data unchanged.
|
|
215
213
|
|
|
216
214
|
Typical use cases:
|
|
217
215
|
|
|
218
|
-
* **Allocation factors** — when a process produces multiple products, the exchange amounts must be partitioned between them. Storing the allocation coefficients as a `
|
|
219
|
-
* **Unit conversions** — when source data is expressed in a unit that differs from the matrix convention, a constant conversion factor can be stored as a `
|
|
216
|
+
* **Allocation factors** — when a process produces multiple products, the exchange amounts must be partitioned between them. Storing the allocation coefficients as a `rescale_array` keeps them alongside the data they modify without requiring a separate processing step.
|
|
217
|
+
* **Unit conversions** — when source data is expressed in a unit that differs from the matrix convention, a constant conversion factor can be stored as a `rescale_array` rather than baked into every data value.
|
|
220
218
|
|
|
221
219
|
```python
|
|
222
220
|
import numpy as np
|
|
@@ -226,18 +224,45 @@ from bw_processing.constants import INDICES_DTYPE
|
|
|
226
224
|
dp = create_datapackage()
|
|
227
225
|
indices_array = np.array([(1, 4), (2, 5), (3, 6)], dtype=INDICES_DTYPE)
|
|
228
226
|
data_array = np.array([100.0, 200.0, 300.0])
|
|
229
|
-
|
|
227
|
+
rescale_array = np.array([0.6, 1.0, 0.4]) # e.g. allocation factors
|
|
230
228
|
|
|
231
229
|
dp.add_persistent_vector(
|
|
232
230
|
matrix="technosphere",
|
|
233
231
|
name="my-process",
|
|
234
232
|
indices_array=indices_array,
|
|
235
233
|
data_array=data_array,
|
|
236
|
-
|
|
234
|
+
rescale_array=rescale_array,
|
|
237
235
|
)
|
|
238
236
|
```
|
|
239
237
|
|
|
240
|
-
The stored resource has `kind="
|
|
238
|
+
The stored resource has `kind="rescale"` and can be retrieved via `dp.get_resource("my-process.rescale")`. The `rescale_array` must be a float dtype and is always stored as `float64`; passing an integer array raises `WrongDatatype`.
|
|
239
|
+
|
|
240
|
+
### Reference (production) exchanges
|
|
241
|
+
|
|
242
|
+
Any resource group can also carry an optional `reference_array`: a one-dimensional boolean array of the same length as `indices_array`. Where an element is `True`, that exchange is the **reference (production) exchange** for its activity/column.
|
|
243
|
+
|
|
244
|
+
The five structural heuristics in `bw_graph_tools` (matching ids, single non-flipped entry, single positive, single negative, unique product) cannot always identify the reference exchange — whenever an activity has more than one same-sign exchange and the products also appear in other columns, the choice is genuinely ambiguous. Only the modeller knows the answer. `reference_array` records it directly so downstream tools can read it instead of guessing.
|
|
245
|
+
|
|
246
|
+
```python
|
|
247
|
+
import numpy as np
|
|
248
|
+
from bw_processing import create_datapackage
|
|
249
|
+
from bw_processing.constants import INDICES_DTYPE
|
|
250
|
+
|
|
251
|
+
dp = create_datapackage()
|
|
252
|
+
indices_array = np.array([(1, 4), (2, 5), (3, 6)], dtype=INDICES_DTYPE)
|
|
253
|
+
data_array = np.array([1.0, 0.5, 2.0])
|
|
254
|
+
reference_array = np.array([True, False, False]) # first exchange is the reference
|
|
255
|
+
|
|
256
|
+
dp.add_persistent_vector(
|
|
257
|
+
matrix="technosphere",
|
|
258
|
+
name="my-process",
|
|
259
|
+
indices_array=indices_array,
|
|
260
|
+
data_array=data_array,
|
|
261
|
+
reference_array=reference_array,
|
|
262
|
+
)
|
|
263
|
+
```
|
|
264
|
+
|
|
265
|
+
The stored resource has `kind="reference"` and can be retrieved via `dp.get_resource("my-process.reference")`. It must be a boolean array; passing a non-boolean array raises `WrongDatatype`. To keep the common case cheap, the resource is written only when at least one entry is `True` — a group with no reference flags carries no `reference` resource. Using the high-level `MatrixEntry`/`ArrayEntry` API, set `reference=True` (or a boolean `reference` array) on the entries you want flagged.
|
|
241
266
|
|
|
242
267
|
### Parameter arrays for sensitivity analysis
|
|
243
268
|
|
|
@@ -168,12 +168,12 @@ print(data_obj.url)
|
|
|
168
168
|
|
|
169
169
|
### Scale arrays
|
|
170
170
|
|
|
171
|
-
Any resource group (persistent or dynamic, vector or array) can carry an optional `
|
|
171
|
+
Any resource group (persistent or dynamic, vector or array) can carry an optional `rescale_array`: a one-dimensional float array of the same length as `indices_array`. Each element is a multiplicative factor applied to the corresponding data value before it is inserted into the matrix. The factor is applied to both static and stochastically-sampled values. A value of `1.0` leaves the data unchanged.
|
|
172
172
|
|
|
173
173
|
Typical use cases:
|
|
174
174
|
|
|
175
|
-
* **Allocation factors** — when a process produces multiple products, the exchange amounts must be partitioned between them. Storing the allocation coefficients as a `
|
|
176
|
-
* **Unit conversions** — when source data is expressed in a unit that differs from the matrix convention, a constant conversion factor can be stored as a `
|
|
175
|
+
* **Allocation factors** — when a process produces multiple products, the exchange amounts must be partitioned between them. Storing the allocation coefficients as a `rescale_array` keeps them alongside the data they modify without requiring a separate processing step.
|
|
176
|
+
* **Unit conversions** — when source data is expressed in a unit that differs from the matrix convention, a constant conversion factor can be stored as a `rescale_array` rather than baked into every data value.
|
|
177
177
|
|
|
178
178
|
```python
|
|
179
179
|
import numpy as np
|
|
@@ -183,18 +183,45 @@ from bw_processing.constants import INDICES_DTYPE
|
|
|
183
183
|
dp = create_datapackage()
|
|
184
184
|
indices_array = np.array([(1, 4), (2, 5), (3, 6)], dtype=INDICES_DTYPE)
|
|
185
185
|
data_array = np.array([100.0, 200.0, 300.0])
|
|
186
|
-
|
|
186
|
+
rescale_array = np.array([0.6, 1.0, 0.4]) # e.g. allocation factors
|
|
187
187
|
|
|
188
188
|
dp.add_persistent_vector(
|
|
189
189
|
matrix="technosphere",
|
|
190
190
|
name="my-process",
|
|
191
191
|
indices_array=indices_array,
|
|
192
192
|
data_array=data_array,
|
|
193
|
-
|
|
193
|
+
rescale_array=rescale_array,
|
|
194
194
|
)
|
|
195
195
|
```
|
|
196
196
|
|
|
197
|
-
The stored resource has `kind="
|
|
197
|
+
The stored resource has `kind="rescale"` and can be retrieved via `dp.get_resource("my-process.rescale")`. The `rescale_array` must be a float dtype and is always stored as `float64`; passing an integer array raises `WrongDatatype`.
|
|
198
|
+
|
|
199
|
+
### Reference (production) exchanges
|
|
200
|
+
|
|
201
|
+
Any resource group can also carry an optional `reference_array`: a one-dimensional boolean array of the same length as `indices_array`. Where an element is `True`, that exchange is the **reference (production) exchange** for its activity/column.
|
|
202
|
+
|
|
203
|
+
The five structural heuristics in `bw_graph_tools` (matching ids, single non-flipped entry, single positive, single negative, unique product) cannot always identify the reference exchange — whenever an activity has more than one same-sign exchange and the products also appear in other columns, the choice is genuinely ambiguous. Only the modeller knows the answer. `reference_array` records it directly so downstream tools can read it instead of guessing.
|
|
204
|
+
|
|
205
|
+
```python
|
|
206
|
+
import numpy as np
|
|
207
|
+
from bw_processing import create_datapackage
|
|
208
|
+
from bw_processing.constants import INDICES_DTYPE
|
|
209
|
+
|
|
210
|
+
dp = create_datapackage()
|
|
211
|
+
indices_array = np.array([(1, 4), (2, 5), (3, 6)], dtype=INDICES_DTYPE)
|
|
212
|
+
data_array = np.array([1.0, 0.5, 2.0])
|
|
213
|
+
reference_array = np.array([True, False, False]) # first exchange is the reference
|
|
214
|
+
|
|
215
|
+
dp.add_persistent_vector(
|
|
216
|
+
matrix="technosphere",
|
|
217
|
+
name="my-process",
|
|
218
|
+
indices_array=indices_array,
|
|
219
|
+
data_array=data_array,
|
|
220
|
+
reference_array=reference_array,
|
|
221
|
+
)
|
|
222
|
+
```
|
|
223
|
+
|
|
224
|
+
The stored resource has `kind="reference"` and can be retrieved via `dp.get_resource("my-process.reference")`. It must be a boolean array; passing a non-boolean array raises `WrongDatatype`. To keep the common case cheap, the resource is written only when at least one entry is `True` — a group with no reference flags carries no `reference` resource. Using the high-level `MatrixEntry`/`ArrayEntry` API, set `reference=True` (or a boolean `reference` array) on the entries you want flagged.
|
|
198
225
|
|
|
199
226
|
### Parameter arrays for sensitivity analysis
|
|
200
227
|
|
|
@@ -19,24 +19,21 @@ classifiers = [
|
|
|
19
19
|
"Development Status :: 4 - Beta",
|
|
20
20
|
"Programming Language :: Python",
|
|
21
21
|
"Programming Language :: Python :: 3",
|
|
22
|
-
"Programming Language :: Python :: 3.9",
|
|
23
|
-
"Programming Language :: Python :: 3.10",
|
|
24
22
|
"Programming Language :: Python :: 3.11",
|
|
25
23
|
"Programming Language :: Python :: 3.12",
|
|
26
24
|
"Natural Language :: English",
|
|
27
25
|
"Operating System :: OS Independent",
|
|
28
26
|
"Topic :: Scientific/Engineering"
|
|
29
27
|
]
|
|
30
|
-
requires-python = ">=3.
|
|
28
|
+
requires-python = ">=3.11"
|
|
31
29
|
dependencies = [
|
|
32
30
|
# dependencies as strings with quotes, e.g. "foo"
|
|
33
31
|
# You can add version requirements
|
|
34
|
-
"fsspec",
|
|
32
|
+
"fsspec>=2023.3.0",
|
|
35
33
|
"jsonschema>=4.0",
|
|
36
34
|
"morefs",
|
|
37
|
-
"numpy",
|
|
35
|
+
"numpy>=2.0",
|
|
38
36
|
"pandas",
|
|
39
|
-
|
|
40
37
|
]
|
|
41
38
|
|
|
42
39
|
[project.urls]
|
|
@@ -4,7 +4,6 @@ __all__ = (
|
|
|
4
4
|
"as_unique_attributes",
|
|
5
5
|
"as_unique_attributes_dataframe",
|
|
6
6
|
"clean_datapackage_name",
|
|
7
|
-
"create_array",
|
|
8
7
|
"create_datapackage",
|
|
9
8
|
"create_datapackage_from_entries",
|
|
10
9
|
"create_structured_array",
|
|
@@ -35,10 +34,10 @@ __all__ = (
|
|
|
35
34
|
"UndefinedInterface",
|
|
36
35
|
)
|
|
37
36
|
|
|
38
|
-
__version__ = "
|
|
37
|
+
__version__ = "2.0"
|
|
39
38
|
|
|
40
39
|
|
|
41
|
-
from bw_processing.array_creation import
|
|
40
|
+
from bw_processing.array_creation import create_structured_array
|
|
42
41
|
from bw_processing.constants import DEFAULT_LICENSES, INDICES_DTYPE, UNCERTAINTY_DTYPE, MatrixSerializeFormat
|
|
43
42
|
from bw_processing.param_labels import (
|
|
44
43
|
AnyLabelSchema,
|
|
@@ -3,17 +3,6 @@ import itertools
|
|
|
3
3
|
import numpy as np
|
|
4
4
|
|
|
5
5
|
|
|
6
|
-
def peek(iterator):
|
|
7
|
-
iterator = iter(iterator)
|
|
8
|
-
first = next(iterator)
|
|
9
|
-
return first, itertools.chain([first], iterator)
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
def get_ncols(iterator):
|
|
13
|
-
a, b = peek(iterator)
|
|
14
|
-
return len(a), b
|
|
15
|
-
|
|
16
|
-
|
|
17
6
|
def chunked(iterable, chunk_size):
|
|
18
7
|
# Black magic, see https://stackoverflow.com/a/31185097
|
|
19
8
|
# and https://docs.python.org/3/library/functions.html#iter
|
|
@@ -21,8 +10,8 @@ def chunked(iterable, chunk_size):
|
|
|
21
10
|
return iter(lambda: list(itertools.islice(iterable, chunk_size)), [])
|
|
22
11
|
|
|
23
12
|
|
|
24
|
-
def create_chunked(iterable, dtype,
|
|
25
|
-
"""Create a numpy array from an iterable of indeterminate length.
|
|
13
|
+
def create_chunked(iterable, dtype, bucket_size=500):
|
|
14
|
+
"""Create a 1D numpy structured array from an iterable of indeterminate length.
|
|
26
15
|
|
|
27
16
|
Needed when we can't determine the length of the iterable ahead of time
|
|
28
17
|
(e.g. for a generator or a database cursor), so can't create the complete
|
|
@@ -31,21 +20,16 @@ def create_chunked(iterable, dtype, ncols=None, bucket_size=500):
|
|
|
31
20
|
Creates a list of arrays with ``bucket_size`` rows until ``iterable`` is
|
|
32
21
|
exhausted, then concatenates them along axis 0.
|
|
33
22
|
|
|
34
|
-
Pass ``ncols`` for a plain 2D array; omit it for a 1D structured array.
|
|
35
|
-
|
|
36
23
|
Args:
|
|
37
24
|
iterable: Iterable of data used to populate the array.
|
|
38
25
|
dtype: Numpy dtype of the created array.
|
|
39
|
-
ncols: Number of columns; if None, a 1D structured array is created.
|
|
40
26
|
bucket_size: Number of rows in each intermediate array.
|
|
41
27
|
|
|
42
28
|
Returns:
|
|
43
29
|
The created array. Returns a zero-length array if ``iterable`` has no data.
|
|
44
30
|
"""
|
|
45
|
-
bucket_shape = (bucket_size,) if ncols is None else (bucket_size, ncols)
|
|
46
|
-
empty_shape = (0,) if ncols is None else (0, ncols)
|
|
47
31
|
arrays = []
|
|
48
|
-
array = np.zeros(
|
|
32
|
+
array = np.zeros(bucket_size, dtype=dtype)
|
|
49
33
|
for chunk in chunked(iterable, bucket_size):
|
|
50
34
|
for i, row in enumerate(chunk):
|
|
51
35
|
array[i] = row
|
|
@@ -55,11 +39,11 @@ def create_chunked(iterable, dtype, ncols=None, bucket_size=500):
|
|
|
55
39
|
arrays.append(array[: i + 1].copy())
|
|
56
40
|
else:
|
|
57
41
|
arrays.append(array)
|
|
58
|
-
array = np.zeros(
|
|
42
|
+
array = np.zeros(bucket_size, dtype=dtype)
|
|
59
43
|
# Empty iterable - create zero-length array.
|
|
60
44
|
# Needed because we return iterators for SQL databases
|
|
61
45
|
# but don't know if e.g. sometimes a database has no biosphere exchanges.
|
|
62
|
-
return np.concatenate(arrays, axis=0) if arrays else np.zeros(
|
|
46
|
+
return np.concatenate(arrays, axis=0) if arrays else np.zeros(0, dtype=dtype)
|
|
63
47
|
|
|
64
48
|
|
|
65
49
|
def create_structured_array(iterable, dtype, nrows=None, sort=False, sort_fields=None):
|
|
@@ -73,10 +57,15 @@ def create_structured_array(iterable, dtype, nrows=None, sort=False, sort_fields
|
|
|
73
57
|
if not nrows:
|
|
74
58
|
nrows = len(iterable)
|
|
75
59
|
array = np.zeros(nrows, dtype=dtype)
|
|
60
|
+
count = 0
|
|
76
61
|
for i, row in enumerate(iterable):
|
|
77
62
|
if i > (nrows - 1):
|
|
78
63
|
raise ValueError("More rows than `nrows`")
|
|
79
64
|
array[i] = tuple(row)
|
|
65
|
+
count = i + 1
|
|
66
|
+
if count < nrows:
|
|
67
|
+
# Unfilled rows would be stored as zeros (row 0, col 0, amount 0)
|
|
68
|
+
raise ValueError(f"Fewer rows ({count}) than `nrows` ({nrows})")
|
|
80
69
|
|
|
81
70
|
else:
|
|
82
71
|
array = create_chunked(iterable, dtype)
|
|
@@ -90,33 +79,3 @@ def create_structured_array(iterable, dtype, nrows=None, sort=False, sort_fields
|
|
|
90
79
|
array.sort(order=order)
|
|
91
80
|
|
|
92
81
|
return array
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
def create_array(iterable, nrows=None, dtype=np.float32):
|
|
96
|
-
"""Create a numpy array data ``iterable``. Returns a filepath of a created file (if ``filepath`` is provided, or the array.
|
|
97
|
-
|
|
98
|
-
``iterable`` can be data already in memory, or a generator.
|
|
99
|
-
|
|
100
|
-
``nrows`` can be supplied, if known. If ``iterable`` has a length, it will be determined automatically. If ``nrows`` is not known, this function generates chunked arrays until ``iterable`` is exhausted, and concatenates them.
|
|
101
|
-
|
|
102
|
-
Either ``nrows`` or ``ncols`` must be specified."""
|
|
103
|
-
if isinstance(iterable, np.ndarray):
|
|
104
|
-
array = iterable.astype(dtype)
|
|
105
|
-
elif nrows or hasattr(iterable, "__len__"):
|
|
106
|
-
if not nrows:
|
|
107
|
-
nrows = len(iterable)
|
|
108
|
-
ncols, data = get_ncols(iterable)
|
|
109
|
-
array = np.zeros((nrows, ncols), dtype=dtype)
|
|
110
|
-
for i, row in enumerate(data):
|
|
111
|
-
if i > (nrows - 1):
|
|
112
|
-
raise ValueError("More rows than `nrows`")
|
|
113
|
-
array[i, :] = tuple(row)
|
|
114
|
-
|
|
115
|
-
else:
|
|
116
|
-
try:
|
|
117
|
-
ncols, data = get_ncols(iterable)
|
|
118
|
-
except StopIteration:
|
|
119
|
-
return np.zeros((0, 0), dtype=dtype)
|
|
120
|
-
array = create_chunked(data, dtype, ncols=ncols)
|
|
121
|
-
|
|
122
|
-
return array
|
|
@@ -14,11 +14,11 @@ MAX_SIGNED_64BIT_INT = 9223372036854775807
|
|
|
14
14
|
# but -687ms to append two columns.
|
|
15
15
|
UNCERTAINTY_DTYPE = [
|
|
16
16
|
("uncertainty_type", np.uint8),
|
|
17
|
-
("loc", np.
|
|
18
|
-
("scale", np.
|
|
19
|
-
("shape", np.
|
|
20
|
-
("minimum", np.
|
|
21
|
-
("maximum", np.
|
|
17
|
+
("loc", np.float64),
|
|
18
|
+
("scale", np.float64),
|
|
19
|
+
("shape", np.float64),
|
|
20
|
+
("minimum", np.float64),
|
|
21
|
+
("maximum", np.float64),
|
|
22
22
|
("negative", bool),
|
|
23
23
|
]
|
|
24
24
|
INDICES_DTYPE = [("row", np.int64), ("col", np.int64)]
|