bw-processing 1.6__tar.gz → 2.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {bw_processing-1.6/src/bw_processing.egg-info → bw_processing-2.0}/PKG-INFO +5 -7
- {bw_processing-1.6 → bw_processing-2.0}/README.md +1 -1
- {bw_processing-1.6 → bw_processing-2.0}/pyproject.toml +3 -6
- {bw_processing-1.6 → bw_processing-2.0}/src/bw_processing/__init__.py +2 -3
- {bw_processing-1.6 → bw_processing-2.0}/src/bw_processing/array_creation.py +10 -51
- {bw_processing-1.6 → bw_processing-2.0}/src/bw_processing/constants.py +5 -5
- {bw_processing-1.6 → bw_processing-2.0}/src/bw_processing/datapackage.py +112 -50
- {bw_processing-1.6 → bw_processing-2.0}/src/bw_processing/io_pyarrow_helpers.py +4 -1
- {bw_processing-1.6 → bw_processing-2.0}/src/bw_processing/matrix_entry.py +66 -12
- {bw_processing-1.6 → bw_processing-2.0}/src/bw_processing/merging.py +2 -2
- {bw_processing-1.6 → bw_processing-2.0}/src/bw_processing/utils.py +20 -6
- {bw_processing-1.6 → bw_processing-2.0/src/bw_processing.egg-info}/PKG-INFO +5 -7
- {bw_processing-1.6 → bw_processing-2.0}/src/bw_processing.egg-info/requires.txt +2 -2
- bw_processing-2.0/tests/test_array_creation.py +44 -0
- {bw_processing-1.6 → bw_processing-2.0}/tests/test_datapackage.py +101 -0
- {bw_processing-1.6 → bw_processing-2.0}/tests/test_filtered_datapackage.py +43 -0
- {bw_processing-1.6 → bw_processing-2.0}/tests/test_io_parquet_helpers.py +0 -8
- {bw_processing-1.6 → bw_processing-2.0}/tests/test_io_pyarrow_helpers.py +15 -0
- {bw_processing-1.6 → bw_processing-2.0}/tests/test_matrix_entry.py +222 -2
- {bw_processing-1.6 → bw_processing-2.0}/tests/test_merging.py +9 -0
- bw_processing-1.6/tests/test_array_creation.py +0 -97
- {bw_processing-1.6 → bw_processing-2.0}/LICENSE +0 -0
- {bw_processing-1.6 → bw_processing-2.0}/MANIFEST.in +0 -0
- {bw_processing-1.6 → bw_processing-2.0}/setup.cfg +0 -0
- {bw_processing-1.6 → bw_processing-2.0}/src/bw_processing/errors.py +0 -0
- {bw_processing-1.6 → bw_processing-2.0}/src/bw_processing/examples/__init__.py +0 -0
- {bw_processing-1.6 → bw_processing-2.0}/src/bw_processing/examples/datapackage_1/another name.indices.parquet +0 -0
- {bw_processing-1.6 → bw_processing-2.0}/src/bw_processing/examples/datapackage_1/sa-data-vector-from-dict.data.parquet +0 -0
- {bw_processing-1.6 → bw_processing-2.0}/src/bw_processing/examples/datapackage_1/sa-data-vector-from-dict.distributions.parquet +0 -0
- {bw_processing-1.6 → bw_processing-2.0}/src/bw_processing/examples/datapackage_1/sa-data-vector-from-dict.flip.parquet +0 -0
- {bw_processing-1.6 → bw_processing-2.0}/src/bw_processing/examples/datapackage_1/sa-data-vector-from-dict.indices.parquet +0 -0
- {bw_processing-1.6 → bw_processing-2.0}/src/bw_processing/examples/datapackage_1/some name.data.parquet +0 -0
- {bw_processing-1.6 → bw_processing-2.0}/src/bw_processing/examples/datapackage_1/some name.flip.parquet +0 -0
- {bw_processing-1.6 → bw_processing-2.0}/src/bw_processing/examples/datapackage_1/some name.indices.parquet +0 -0
- {bw_processing-1.6 → bw_processing-2.0}/src/bw_processing/examples/datapackage_2.zip +0 -0
- {bw_processing-1.6 → bw_processing-2.0}/src/bw_processing/examples/interfaces.py +0 -0
- {bw_processing-1.6 → bw_processing-2.0}/src/bw_processing/examples/parquet_files.py +0 -0
- {bw_processing-1.6 → bw_processing-2.0}/src/bw_processing/examples/simple.zip +0 -0
- {bw_processing-1.6 → bw_processing-2.0}/src/bw_processing/filesystem.py +0 -0
- {bw_processing-1.6 → bw_processing-2.0}/src/bw_processing/indexing.py +0 -0
- {bw_processing-1.6 → bw_processing-2.0}/src/bw_processing/io_helpers.py +0 -0
- {bw_processing-1.6 → bw_processing-2.0}/src/bw_processing/io_parquet_helpers.py +0 -0
- {bw_processing-1.6 → bw_processing-2.0}/src/bw_processing/param_labels.py +0 -0
- {bw_processing-1.6 → bw_processing-2.0}/src/bw_processing/proxies.py +0 -0
- {bw_processing-1.6 → bw_processing-2.0}/src/bw_processing/unique_fields.py +0 -0
- {bw_processing-1.6 → bw_processing-2.0}/src/bw_processing.egg-info/SOURCES.txt +0 -0
- {bw_processing-1.6 → bw_processing-2.0}/src/bw_processing.egg-info/dependency_links.txt +0 -0
- {bw_processing-1.6 → bw_processing-2.0}/src/bw_processing.egg-info/top_level.txt +0 -0
- {bw_processing-1.6 → bw_processing-2.0}/tests/test_filesystem.py +0 -0
- {bw_processing-1.6 → bw_processing-2.0}/tests/test_indexing.py +0 -0
- {bw_processing-1.6 → bw_processing-2.0}/tests/test_integration.py +0 -0
- {bw_processing-1.6 → bw_processing-2.0}/tests/test_interfaces.py +0 -0
- {bw_processing-1.6 → bw_processing-2.0}/tests/test_loading.py +0 -0
- {bw_processing-1.6 → bw_processing-2.0}/tests/test_params.py +0 -0
- {bw_processing-1.6 → bw_processing-2.0}/tests/test_proxies.py +0 -0
- {bw_processing-1.6 → bw_processing-2.0}/tests/test_unique_fields.py +0 -0
- {bw_processing-1.6 → bw_processing-2.0}/tests/test_utils.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: bw_processing
|
|
3
|
-
Version:
|
|
3
|
+
Version: 2.0
|
|
4
4
|
Summary: Foo
|
|
5
5
|
Author-email: Chris Mutel <cmutel@gmail.com>
|
|
6
6
|
Maintainer-email: Chris Mutel <cmutel@gmail.com>
|
|
@@ -10,20 +10,18 @@ Project-URL: tracker, https://github.com/brightway-lca/bw_processing/issues
|
|
|
10
10
|
Classifier: Development Status :: 4 - Beta
|
|
11
11
|
Classifier: Programming Language :: Python
|
|
12
12
|
Classifier: Programming Language :: Python :: 3
|
|
13
|
-
Classifier: Programming Language :: Python :: 3.9
|
|
14
|
-
Classifier: Programming Language :: Python :: 3.10
|
|
15
13
|
Classifier: Programming Language :: Python :: 3.11
|
|
16
14
|
Classifier: Programming Language :: Python :: 3.12
|
|
17
15
|
Classifier: Natural Language :: English
|
|
18
16
|
Classifier: Operating System :: OS Independent
|
|
19
17
|
Classifier: Topic :: Scientific/Engineering
|
|
20
|
-
Requires-Python: >=3.
|
|
18
|
+
Requires-Python: >=3.11
|
|
21
19
|
Description-Content-Type: text/markdown
|
|
22
20
|
License-File: LICENSE
|
|
23
|
-
Requires-Dist: fsspec
|
|
21
|
+
Requires-Dist: fsspec>=2023.3.0
|
|
24
22
|
Requires-Dist: jsonschema>=4.0
|
|
25
23
|
Requires-Dist: morefs
|
|
26
|
-
Requires-Dist: numpy
|
|
24
|
+
Requires-Dist: numpy>=2.0
|
|
27
25
|
Requires-Dist: pandas
|
|
28
26
|
Provides-Extra: testing
|
|
29
27
|
Requires-Dist: bw_processing; extra == "testing"
|
|
@@ -237,7 +235,7 @@ dp.add_persistent_vector(
|
|
|
237
235
|
)
|
|
238
236
|
```
|
|
239
237
|
|
|
240
|
-
The stored resource has `kind="rescale"` and can be retrieved via `dp.get_resource("my-process.rescale")`. The `rescale_array` must be a float dtype
|
|
238
|
+
The stored resource has `kind="rescale"` and can be retrieved via `dp.get_resource("my-process.rescale")`. The `rescale_array` must be a float dtype and is always stored as `float64`; passing an integer array raises `WrongDatatype`.
|
|
241
239
|
|
|
242
240
|
### Reference (production) exchanges
|
|
243
241
|
|
|
@@ -194,7 +194,7 @@ dp.add_persistent_vector(
|
|
|
194
194
|
)
|
|
195
195
|
```
|
|
196
196
|
|
|
197
|
-
The stored resource has `kind="rescale"` and can be retrieved via `dp.get_resource("my-process.rescale")`. The `rescale_array` must be a float dtype
|
|
197
|
+
The stored resource has `kind="rescale"` and can be retrieved via `dp.get_resource("my-process.rescale")`. The `rescale_array` must be a float dtype and is always stored as `float64`; passing an integer array raises `WrongDatatype`.
|
|
198
198
|
|
|
199
199
|
### Reference (production) exchanges
|
|
200
200
|
|
|
@@ -19,24 +19,21 @@ classifiers = [
|
|
|
19
19
|
"Development Status :: 4 - Beta",
|
|
20
20
|
"Programming Language :: Python",
|
|
21
21
|
"Programming Language :: Python :: 3",
|
|
22
|
-
"Programming Language :: Python :: 3.9",
|
|
23
|
-
"Programming Language :: Python :: 3.10",
|
|
24
22
|
"Programming Language :: Python :: 3.11",
|
|
25
23
|
"Programming Language :: Python :: 3.12",
|
|
26
24
|
"Natural Language :: English",
|
|
27
25
|
"Operating System :: OS Independent",
|
|
28
26
|
"Topic :: Scientific/Engineering"
|
|
29
27
|
]
|
|
30
|
-
requires-python = ">=3.
|
|
28
|
+
requires-python = ">=3.11"
|
|
31
29
|
dependencies = [
|
|
32
30
|
# dependencies as strings with quotes, e.g. "foo"
|
|
33
31
|
# You can add version requirements
|
|
34
|
-
"fsspec",
|
|
32
|
+
"fsspec>=2023.3.0",
|
|
35
33
|
"jsonschema>=4.0",
|
|
36
34
|
"morefs",
|
|
37
|
-
"numpy",
|
|
35
|
+
"numpy>=2.0",
|
|
38
36
|
"pandas",
|
|
39
|
-
|
|
40
37
|
]
|
|
41
38
|
|
|
42
39
|
[project.urls]
|
|
@@ -4,7 +4,6 @@ __all__ = (
|
|
|
4
4
|
"as_unique_attributes",
|
|
5
5
|
"as_unique_attributes_dataframe",
|
|
6
6
|
"clean_datapackage_name",
|
|
7
|
-
"create_array",
|
|
8
7
|
"create_datapackage",
|
|
9
8
|
"create_datapackage_from_entries",
|
|
10
9
|
"create_structured_array",
|
|
@@ -35,10 +34,10 @@ __all__ = (
|
|
|
35
34
|
"UndefinedInterface",
|
|
36
35
|
)
|
|
37
36
|
|
|
38
|
-
__version__ = "
|
|
37
|
+
__version__ = "2.0"
|
|
39
38
|
|
|
40
39
|
|
|
41
|
-
from bw_processing.array_creation import
|
|
40
|
+
from bw_processing.array_creation import create_structured_array
|
|
42
41
|
from bw_processing.constants import DEFAULT_LICENSES, INDICES_DTYPE, UNCERTAINTY_DTYPE, MatrixSerializeFormat
|
|
43
42
|
from bw_processing.param_labels import (
|
|
44
43
|
AnyLabelSchema,
|
|
@@ -3,17 +3,6 @@ import itertools
|
|
|
3
3
|
import numpy as np
|
|
4
4
|
|
|
5
5
|
|
|
6
|
-
def peek(iterator):
|
|
7
|
-
iterator = iter(iterator)
|
|
8
|
-
first = next(iterator)
|
|
9
|
-
return first, itertools.chain([first], iterator)
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
def get_ncols(iterator):
|
|
13
|
-
a, b = peek(iterator)
|
|
14
|
-
return len(a), b
|
|
15
|
-
|
|
16
|
-
|
|
17
6
|
def chunked(iterable, chunk_size):
|
|
18
7
|
# Black magic, see https://stackoverflow.com/a/31185097
|
|
19
8
|
# and https://docs.python.org/3/library/functions.html#iter
|
|
@@ -21,8 +10,8 @@ def chunked(iterable, chunk_size):
|
|
|
21
10
|
return iter(lambda: list(itertools.islice(iterable, chunk_size)), [])
|
|
22
11
|
|
|
23
12
|
|
|
24
|
-
def create_chunked(iterable, dtype,
|
|
25
|
-
"""Create a numpy array from an iterable of indeterminate length.
|
|
13
|
+
def create_chunked(iterable, dtype, bucket_size=500):
|
|
14
|
+
"""Create a 1D numpy structured array from an iterable of indeterminate length.
|
|
26
15
|
|
|
27
16
|
Needed when we can't determine the length of the iterable ahead of time
|
|
28
17
|
(e.g. for a generator or a database cursor), so can't create the complete
|
|
@@ -31,21 +20,16 @@ def create_chunked(iterable, dtype, ncols=None, bucket_size=500):
|
|
|
31
20
|
Creates a list of arrays with ``bucket_size`` rows until ``iterable`` is
|
|
32
21
|
exhausted, then concatenates them along axis 0.
|
|
33
22
|
|
|
34
|
-
Pass ``ncols`` for a plain 2D array; omit it for a 1D structured array.
|
|
35
|
-
|
|
36
23
|
Args:
|
|
37
24
|
iterable: Iterable of data used to populate the array.
|
|
38
25
|
dtype: Numpy dtype of the created array.
|
|
39
|
-
ncols: Number of columns; if None, a 1D structured array is created.
|
|
40
26
|
bucket_size: Number of rows in each intermediate array.
|
|
41
27
|
|
|
42
28
|
Returns:
|
|
43
29
|
The created array. Returns a zero-length array if ``iterable`` has no data.
|
|
44
30
|
"""
|
|
45
|
-
bucket_shape = (bucket_size,) if ncols is None else (bucket_size, ncols)
|
|
46
|
-
empty_shape = (0,) if ncols is None else (0, ncols)
|
|
47
31
|
arrays = []
|
|
48
|
-
array = np.zeros(
|
|
32
|
+
array = np.zeros(bucket_size, dtype=dtype)
|
|
49
33
|
for chunk in chunked(iterable, bucket_size):
|
|
50
34
|
for i, row in enumerate(chunk):
|
|
51
35
|
array[i] = row
|
|
@@ -55,11 +39,11 @@ def create_chunked(iterable, dtype, ncols=None, bucket_size=500):
|
|
|
55
39
|
arrays.append(array[: i + 1].copy())
|
|
56
40
|
else:
|
|
57
41
|
arrays.append(array)
|
|
58
|
-
array = np.zeros(
|
|
42
|
+
array = np.zeros(bucket_size, dtype=dtype)
|
|
59
43
|
# Empty iterable - create zero-length array.
|
|
60
44
|
# Needed because we return iterators for SQL databases
|
|
61
45
|
# but don't know if e.g. sometimes a database has no biosphere exchanges.
|
|
62
|
-
return np.concatenate(arrays, axis=0) if arrays else np.zeros(
|
|
46
|
+
return np.concatenate(arrays, axis=0) if arrays else np.zeros(0, dtype=dtype)
|
|
63
47
|
|
|
64
48
|
|
|
65
49
|
def create_structured_array(iterable, dtype, nrows=None, sort=False, sort_fields=None):
|
|
@@ -73,10 +57,15 @@ def create_structured_array(iterable, dtype, nrows=None, sort=False, sort_fields
|
|
|
73
57
|
if not nrows:
|
|
74
58
|
nrows = len(iterable)
|
|
75
59
|
array = np.zeros(nrows, dtype=dtype)
|
|
60
|
+
count = 0
|
|
76
61
|
for i, row in enumerate(iterable):
|
|
77
62
|
if i > (nrows - 1):
|
|
78
63
|
raise ValueError("More rows than `nrows`")
|
|
79
64
|
array[i] = tuple(row)
|
|
65
|
+
count = i + 1
|
|
66
|
+
if count < nrows:
|
|
67
|
+
# Unfilled rows would be stored as zeros (row 0, col 0, amount 0)
|
|
68
|
+
raise ValueError(f"Fewer rows ({count}) than `nrows` ({nrows})")
|
|
80
69
|
|
|
81
70
|
else:
|
|
82
71
|
array = create_chunked(iterable, dtype)
|
|
@@ -90,33 +79,3 @@ def create_structured_array(iterable, dtype, nrows=None, sort=False, sort_fields
|
|
|
90
79
|
array.sort(order=order)
|
|
91
80
|
|
|
92
81
|
return array
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
def create_array(iterable, nrows=None, dtype=np.float32):
|
|
96
|
-
"""Create a numpy array data ``iterable``. Returns a filepath of a created file (if ``filepath`` is provided, or the array.
|
|
97
|
-
|
|
98
|
-
``iterable`` can be data already in memory, or a generator.
|
|
99
|
-
|
|
100
|
-
``nrows`` can be supplied, if known. If ``iterable`` has a length, it will be determined automatically. If ``nrows`` is not known, this function generates chunked arrays until ``iterable`` is exhausted, and concatenates them.
|
|
101
|
-
|
|
102
|
-
Either ``nrows`` or ``ncols`` must be specified."""
|
|
103
|
-
if isinstance(iterable, np.ndarray):
|
|
104
|
-
array = iterable.astype(dtype)
|
|
105
|
-
elif nrows or hasattr(iterable, "__len__"):
|
|
106
|
-
if not nrows:
|
|
107
|
-
nrows = len(iterable)
|
|
108
|
-
ncols, data = get_ncols(iterable)
|
|
109
|
-
array = np.zeros((nrows, ncols), dtype=dtype)
|
|
110
|
-
for i, row in enumerate(data):
|
|
111
|
-
if i > (nrows - 1):
|
|
112
|
-
raise ValueError("More rows than `nrows`")
|
|
113
|
-
array[i, :] = tuple(row)
|
|
114
|
-
|
|
115
|
-
else:
|
|
116
|
-
try:
|
|
117
|
-
ncols, data = get_ncols(iterable)
|
|
118
|
-
except StopIteration:
|
|
119
|
-
return np.zeros((0, 0), dtype=dtype)
|
|
120
|
-
array = create_chunked(data, dtype, ncols=ncols)
|
|
121
|
-
|
|
122
|
-
return array
|
|
@@ -14,11 +14,11 @@ MAX_SIGNED_64BIT_INT = 9223372036854775807
|
|
|
14
14
|
# but -687ms to append two columns.
|
|
15
15
|
UNCERTAINTY_DTYPE = [
|
|
16
16
|
("uncertainty_type", np.uint8),
|
|
17
|
-
("loc", np.
|
|
18
|
-
("scale", np.
|
|
19
|
-
("shape", np.
|
|
20
|
-
("minimum", np.
|
|
21
|
-
("maximum", np.
|
|
17
|
+
("loc", np.float64),
|
|
18
|
+
("scale", np.float64),
|
|
19
|
+
("shape", np.float64),
|
|
20
|
+
("minimum", np.float64),
|
|
21
|
+
("maximum", np.float64),
|
|
22
22
|
("negative", bool),
|
|
23
23
|
]
|
|
24
24
|
INDICES_DTYPE = [("row", np.int64), ("col", np.int64)]
|
|
@@ -2,8 +2,9 @@ import datetime
|
|
|
2
2
|
import uuid
|
|
3
3
|
import warnings
|
|
4
4
|
from abc import ABC
|
|
5
|
+
from collections.abc import Sequence
|
|
5
6
|
from functools import partial
|
|
6
|
-
from typing import TYPE_CHECKING, Any, Dict, List, Optional, Union
|
|
7
|
+
from typing import TYPE_CHECKING, Any, Dict, Iterable, List, Optional, Union
|
|
7
8
|
|
|
8
9
|
if TYPE_CHECKING:
|
|
9
10
|
from bw_processing.matrix_entry import ArrayEntry, MatrixEntry
|
|
@@ -30,6 +31,7 @@ from bw_processing.constants import (
|
|
|
30
31
|
NUMPY_SERIALIZE_FORMAT_NAME,
|
|
31
32
|
PARQUET_SERIALIZE_FORMAT_EXTENSION,
|
|
32
33
|
PARQUET_SERIALIZE_FORMAT_NAME,
|
|
34
|
+
UNCERTAINTY_DTYPE,
|
|
33
35
|
MatrixSerializeFormat,
|
|
34
36
|
)
|
|
35
37
|
from bw_processing.errors import (
|
|
@@ -45,7 +47,14 @@ from bw_processing.filesystem import clean_datapackage_name
|
|
|
45
47
|
from bw_processing.io_helpers import file_reader, file_writer
|
|
46
48
|
from bw_processing.param_labels import AnyLabelSchema
|
|
47
49
|
from bw_processing.proxies import Proxy, UndefinedInterface
|
|
48
|
-
from bw_processing.utils import
|
|
50
|
+
from bw_processing.utils import (
|
|
51
|
+
check_name,
|
|
52
|
+
check_suffix,
|
|
53
|
+
load_bytes,
|
|
54
|
+
resolve_dict_iterator,
|
|
55
|
+
resolve_tuple_iterator,
|
|
56
|
+
utc_now,
|
|
57
|
+
)
|
|
49
58
|
|
|
50
59
|
|
|
51
60
|
class DatapackageBase(ABC):
|
|
@@ -68,7 +77,11 @@ class DatapackageBase(ABC):
|
|
|
68
77
|
return len(self.data)
|
|
69
78
|
|
|
70
79
|
def __contains__(self, key):
|
|
71
|
-
return key in {o["name"] for o in self.resources} or
|
|
80
|
+
return key in {o["name"] for o in self.resources} or self._has_group(key)
|
|
81
|
+
|
|
82
|
+
def _has_group(self, label: Any) -> bool:
|
|
83
|
+
"""Check for a resource group label without building the filtered ``groups``."""
|
|
84
|
+
return bool(label) and any(o.get("group") == label for o in self.resources)
|
|
72
85
|
|
|
73
86
|
resources = property(__get_resources, __set_resources)
|
|
74
87
|
|
|
@@ -78,14 +91,14 @@ class DatapackageBase(ABC):
|
|
|
78
91
|
|
|
79
92
|
Ignores resources which don't have group labels.
|
|
80
93
|
"""
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
94
|
+
# One pass over the resources: filtering once per label made this
|
|
95
|
+
# quadratic in the number of resources.
|
|
96
|
+
indices = {}
|
|
97
|
+
for i, resource in enumerate(self.resources):
|
|
98
|
+
if resource.get("group"):
|
|
99
|
+
indices.setdefault(resource["group"], []).append(i)
|
|
87
100
|
|
|
88
|
-
return {label: self.
|
|
101
|
+
return {label: self._filtered(lst) for label, lst in indices.items()}
|
|
89
102
|
|
|
90
103
|
def _get_index(self, name_or_index: Union[str, int]) -> int:
|
|
91
104
|
"""Get index of a resource by name or index.
|
|
@@ -144,7 +157,7 @@ class DatapackageBase(ABC):
|
|
|
144
157
|
if self._modified:
|
|
145
158
|
raise PotentialInconsistency("Datapackage is modified; save modifications or reload")
|
|
146
159
|
|
|
147
|
-
indices =
|
|
160
|
+
indices = {i for i, resource in enumerate(self.resources) if resource.get("group") == name}
|
|
148
161
|
|
|
149
162
|
for obj in (obj for i, obj in enumerate(self.resources) if i in indices):
|
|
150
163
|
try:
|
|
@@ -188,16 +201,9 @@ class DatapackageBase(ABC):
|
|
|
188
201
|
|
|
189
202
|
This method was introduced to allow for the efficient construction of matrices; each datapackage can have data for multiple matrices, and we can then create filtered datapackages which exclusively have data for the matrix of interest. As such, they should be considered read-only, though this is not enforced.
|
|
190
203
|
"""
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
fdp.metadata["resources"] = []
|
|
195
|
-
to_include = [i for i, resource in enumerate(self.resources) if resource.get(key) == value]
|
|
196
|
-
fdp.data = [o for i, o in enumerate(self.data) if i in to_include]
|
|
197
|
-
fdp.resources = [o for i, o in enumerate(self.resources) if i in to_include]
|
|
198
|
-
if hasattr(self, "indexer"):
|
|
199
|
-
fdp.indexer = self.indexer
|
|
200
|
-
return fdp
|
|
204
|
+
return self._filtered(
|
|
205
|
+
[i for i, resource in enumerate(self.resources) if resource.get(key) == value]
|
|
206
|
+
)
|
|
201
207
|
|
|
202
208
|
def exclude(self, filters: Dict[str, str]) -> "FilteredDatapackage":
|
|
203
209
|
"""Filter a datapackage to exclude resources matching a filter.
|
|
@@ -213,21 +219,33 @@ class DatapackageBase(ABC):
|
|
|
213
219
|
exclude_generic({"group': "some_group", "kind": "some_kind"})
|
|
214
220
|
|
|
215
221
|
"""
|
|
222
|
+
return self._filtered(
|
|
223
|
+
[
|
|
224
|
+
i
|
|
225
|
+
for i, resource in enumerate(self.resources)
|
|
226
|
+
if any(resource.get(key) != value for key, value in filters.items())
|
|
227
|
+
]
|
|
228
|
+
)
|
|
229
|
+
|
|
230
|
+
def _check_length_consistency(self) -> None:
|
|
231
|
+
if len(self.resources) != len(self.data):
|
|
232
|
+
raise LengthMismatch(
|
|
233
|
+
"Number of resources ({}) doesn't match number of data objects ({})".format(
|
|
234
|
+
len(self.resources), len(self.data)
|
|
235
|
+
)
|
|
236
|
+
)
|
|
237
|
+
|
|
238
|
+
def _filtered(self, indices: List[int]) -> "FilteredDatapackage":
|
|
239
|
+
"""Create a ``FilteredDatapackage`` with the resources at ``indices``, sharing their data."""
|
|
240
|
+
self._check_length_consistency()
|
|
216
241
|
fdp = FilteredDatapackage()
|
|
217
242
|
fdp.fs = self.fs
|
|
218
243
|
fdp.metadata = {k: v for k, v in self.metadata.items() if k != "resources"}
|
|
219
244
|
fdp.metadata["resources"] = []
|
|
220
|
-
|
|
245
|
+
fdp.data = [self.data[i] for i in indices]
|
|
246
|
+
fdp.resources = [self.resources[i] for i in indices]
|
|
221
247
|
if hasattr(self, "indexer"):
|
|
222
248
|
fdp.indexer = self.indexer
|
|
223
|
-
|
|
224
|
-
indices_to_include = [
|
|
225
|
-
i
|
|
226
|
-
for i, resource in enumerate(self.resources)
|
|
227
|
-
if any(resource.get(key) != value for key, value in filters.items())
|
|
228
|
-
]
|
|
229
|
-
fdp.data = [o for i, o in enumerate(self.data) if i in indices_to_include]
|
|
230
|
-
fdp.resources = [o for i, o in enumerate(self.resources) if i in indices_to_include]
|
|
231
249
|
return fdp
|
|
232
250
|
|
|
233
251
|
def _dehydrate_interfaces(self) -> None:
|
|
@@ -307,14 +325,6 @@ class Datapackage(DatapackageBase):
|
|
|
307
325
|
def __eq__(self, other):
|
|
308
326
|
return (id(self.fs), self.metadata) == (id(other.fs), other.metadata)
|
|
309
327
|
|
|
310
|
-
def _check_length_consistency(self) -> None:
|
|
311
|
-
if len(self.resources) != len(self.data):
|
|
312
|
-
raise LengthMismatch(
|
|
313
|
-
"Number of resources ({}) doesn't match number of data objects ({})".format(
|
|
314
|
-
len(self.resources), len(self.data)
|
|
315
|
-
)
|
|
316
|
-
)
|
|
317
|
-
|
|
318
328
|
def _load(
|
|
319
329
|
self, fs: AbstractFileSystem, mmap_mode: Optional[str] = None, proxy: bool = False
|
|
320
330
|
) -> None:
|
|
@@ -396,7 +406,7 @@ class Datapackage(DatapackageBase):
|
|
|
396
406
|
A distributions resource is considered trivial when every row has an
|
|
397
407
|
``uncertainty_type`` of 0 (no uncertainty) or 1 (unknown uncertainty)
|
|
398
408
|
*and* the ``loc`` values are equal to the corresponding data-array
|
|
399
|
-
amounts
|
|
409
|
+
amounts. Such a resource duplicates what the
|
|
400
410
|
data array already encodes, so it is deleted from the filesystem and
|
|
401
411
|
removed from ``self.data`` / ``self.resources``.
|
|
402
412
|
"""
|
|
@@ -430,7 +440,7 @@ class Datapackage(DatapackageBase):
|
|
|
430
440
|
data_arr = data_arr()
|
|
431
441
|
|
|
432
442
|
all_trivial = np.all(np.isin(dist_arr["uncertainty_type"], _NO_UNCERTAINTY_IDS))
|
|
433
|
-
loc_matches = np.array_equal(dist_arr["loc"], data_arr
|
|
443
|
+
loc_matches = np.array_equal(dist_arr["loc"], data_arr)
|
|
434
444
|
|
|
435
445
|
if all_trivial and loc_matches:
|
|
436
446
|
to_remove.append(idx)
|
|
@@ -505,7 +515,31 @@ class Datapackage(DatapackageBase):
|
|
|
505
515
|
"""Create a persistant vector from an iterator. Uses the utility function ``resolve_dict_iterator``.
|
|
506
516
|
|
|
507
517
|
This is the **only array creation method which produces sorted arrays**."""
|
|
518
|
+
# Check the name before consuming the iterator, which might not be
|
|
519
|
+
# restartable.
|
|
508
520
|
name = self._prepare_name(name)
|
|
521
|
+
self._add_persistent_vector_from_resolved(
|
|
522
|
+
resolve_dict_iterator(dict_iterator, nrows),
|
|
523
|
+
matrix=matrix,
|
|
524
|
+
name=name,
|
|
525
|
+
matrix_serialize_format_type=matrix_serialize_format_type,
|
|
526
|
+
**kwargs,
|
|
527
|
+
)
|
|
528
|
+
|
|
529
|
+
def _add_persistent_vector_from_resolved(
|
|
530
|
+
self,
|
|
531
|
+
resolved: tuple,
|
|
532
|
+
*,
|
|
533
|
+
matrix: str,
|
|
534
|
+
name: str,
|
|
535
|
+
matrix_serialize_format_type: Optional[MatrixSerializeFormat] = None,
|
|
536
|
+
**kwargs,
|
|
537
|
+
) -> None:
|
|
538
|
+
"""Add a persistent vector from the output of ``resolve_dict_iterator``
|
|
539
|
+
or ``resolve_tuple_iterator``.
|
|
540
|
+
|
|
541
|
+
``name`` must already have been checked with ``_prepare_name``, so
|
|
542
|
+
that a bad name is rejected before the iterator is consumed."""
|
|
509
543
|
(
|
|
510
544
|
data_array,
|
|
511
545
|
indices_array,
|
|
@@ -513,7 +547,7 @@ class Datapackage(DatapackageBase):
|
|
|
513
547
|
flip_array,
|
|
514
548
|
rescale_array,
|
|
515
549
|
reference_array,
|
|
516
|
-
) =
|
|
550
|
+
) = resolved
|
|
517
551
|
self.add_persistent_vector(
|
|
518
552
|
matrix=matrix,
|
|
519
553
|
name=name,
|
|
@@ -532,10 +566,11 @@ class Datapackage(DatapackageBase):
|
|
|
532
566
|
self,
|
|
533
567
|
*,
|
|
534
568
|
matrix: str,
|
|
535
|
-
entries:
|
|
569
|
+
entries: Iterable["MatrixEntry"],
|
|
536
570
|
name: Optional[str] = None,
|
|
571
|
+
nrows: Optional[int] = None,
|
|
537
572
|
) -> None:
|
|
538
|
-
"""Add matrix data from
|
|
573
|
+
"""Add matrix data from :class:`MatrixEntry` objects.
|
|
539
574
|
|
|
540
575
|
High-level convenience method that does not require working directly
|
|
541
576
|
with NumPy arrays. If any entry has a ``rescale`` value other than
|
|
@@ -543,16 +578,32 @@ class Datapackage(DatapackageBase):
|
|
|
543
578
|
If any entry has ``reference=True``, the reference flags are stored as
|
|
544
579
|
a ``reference_array`` resource (``kind="reference"``).
|
|
545
580
|
|
|
581
|
+
``entries`` can be a list or any iterable, such as a generator
|
|
582
|
+
streaming rows from a database cursor. Entries are converted with
|
|
583
|
+
:meth:`.MatrixEntry.as_tuple`, skipping the dict round trip of
|
|
584
|
+
:meth:`add_persistent_vector_from_iterator`.
|
|
585
|
+
|
|
546
586
|
Args:
|
|
547
587
|
matrix: Name of the target matrix (e.g. ``"technosphere"``).
|
|
548
|
-
entries:
|
|
588
|
+
entries: Iterable of :class:`.MatrixEntry` instances.
|
|
549
589
|
name: Optional resource group name; auto-generated if omitted.
|
|
590
|
+
nrows: Exact number of entries, if known and ``entries`` is not a
|
|
591
|
+
sequence. Lets the array be allocated in one step; otherwise
|
|
592
|
+
it is built in chunks. Raises ``ValueError`` if ``entries``
|
|
593
|
+
yields more or fewer than ``nrows`` entries.
|
|
550
594
|
"""
|
|
551
|
-
|
|
595
|
+
# Only take the length of real sequences. Other objects with
|
|
596
|
+
# ``__len__`` can be lazy: ``len()`` runs the whole query on an ORM
|
|
597
|
+
# query object, and raises for a ``tqdm`` wrapper with no total.
|
|
598
|
+
if nrows is None and isinstance(entries, Sequence):
|
|
599
|
+
nrows = len(entries)
|
|
600
|
+
# Check the name before consuming the iterator, which might not be
|
|
601
|
+
# restartable.
|
|
602
|
+
name = self._prepare_name(name)
|
|
603
|
+
self._add_persistent_vector_from_resolved(
|
|
604
|
+
resolve_tuple_iterator((e.as_tuple() for e in entries), nrows),
|
|
552
605
|
matrix=matrix,
|
|
553
606
|
name=name,
|
|
554
|
-
dict_iterator=(e.as_dict() for e in entries),
|
|
555
|
-
nrows=len(entries),
|
|
556
607
|
)
|
|
557
608
|
|
|
558
609
|
def add_array_entries(
|
|
@@ -674,6 +725,15 @@ class Datapackage(DatapackageBase):
|
|
|
674
725
|
)
|
|
675
726
|
if distributions_array is not None:
|
|
676
727
|
distributions_array = load_bytes(distributions_array)
|
|
728
|
+
if distributions_array.dtype != UNCERTAINTY_DTYPE:
|
|
729
|
+
# Always store float fields as float64, e.g. for arrays built
|
|
730
|
+
# with the float32 fields used before version 1.7
|
|
731
|
+
if distributions_array.dtype.names != np.dtype(UNCERTAINTY_DTYPE).names:
|
|
732
|
+
raise WrongDatatype(
|
|
733
|
+
"`distributions_array` dtype is {}, but must have the fields of "
|
|
734
|
+
"`UNCERTAINTY_DTYPE`".format(distributions_array.dtype)
|
|
735
|
+
)
|
|
736
|
+
distributions_array = distributions_array.astype(UNCERTAINTY_DTYPE)
|
|
677
737
|
# If no uncertainty, don't need to store it
|
|
678
738
|
if (distributions_array["uncertainty_type"] < 2).sum() < len(distributions_array):
|
|
679
739
|
if distributions_array.shape != indices_array.shape:
|
|
@@ -1007,7 +1067,9 @@ class Datapackage(DatapackageBase):
|
|
|
1007
1067
|
raise WrongDatatype(
|
|
1008
1068
|
"`rescale_array` dtype is {}, but must be a float dtype".format(rescale_array.dtype)
|
|
1009
1069
|
)
|
|
1010
|
-
|
|
1070
|
+
# Always store rescale factors as float64, same as `ArrayEntry.rescale`
|
|
1071
|
+
rescale_array = rescale_array.astype(np.float64, copy=False)
|
|
1072
|
+
if rescale_array.shape != indices_array.shape:
|
|
1011
1073
|
raise ShapeMismatch(
|
|
1012
1074
|
"`rescale_array` shape ({}) doesn't match `indices_array` ({}).".format(
|
|
1013
1075
|
rescale_array.shape, indices_array.shape
|
|
@@ -1537,7 +1599,7 @@ class Datapackage(DatapackageBase):
|
|
|
1537
1599
|
assert isinstance(dataframe, pd.DataFrame)
|
|
1538
1600
|
assert isinstance(valid_for, list)
|
|
1539
1601
|
|
|
1540
|
-
assert all(
|
|
1602
|
+
assert all(self._has_group(x) for x, y in valid_for)
|
|
1541
1603
|
|
|
1542
1604
|
name = self._prepare_name(name)
|
|
1543
1605
|
self._prepare_modifications()
|
|
@@ -1587,7 +1649,7 @@ class Datapackage(DatapackageBase):
|
|
|
1587
1649
|
|
|
1588
1650
|
"""
|
|
1589
1651
|
assert isinstance(valid_for, str)
|
|
1590
|
-
assert
|
|
1652
|
+
assert self._has_group(valid_for)
|
|
1591
1653
|
|
|
1592
1654
|
self._prepare_modifications()
|
|
1593
1655
|
|
|
@@ -181,7 +181,10 @@ def numpy_distributions_vector_to_pyarrow_distributions_vector_table(
|
|
|
181
181
|
"""
|
|
182
182
|
assert isinstance(arr, np.ndarray)
|
|
183
183
|
assert arr.ndim == 1
|
|
184
|
-
|
|
184
|
+
if arr.dtype != UNCERTAINTY_DTYPE:
|
|
185
|
+
# Arrays built with the float32 fields used before version 1.7
|
|
186
|
+
assert arr.dtype.names == np.dtype(UNCERTAINTY_DTYPE).names
|
|
187
|
+
arr = arr.astype(UNCERTAINTY_DTYPE)
|
|
185
188
|
|
|
186
189
|
from_dict = {}
|
|
187
190
|
for i in range(NBR_UNCERTAINTY_FIELDS):
|