bw-processing 1.6__tar.gz → 2.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (57) hide show
  1. {bw_processing-1.6/src/bw_processing.egg-info → bw_processing-2.0}/PKG-INFO +5 -7
  2. {bw_processing-1.6 → bw_processing-2.0}/README.md +1 -1
  3. {bw_processing-1.6 → bw_processing-2.0}/pyproject.toml +3 -6
  4. {bw_processing-1.6 → bw_processing-2.0}/src/bw_processing/__init__.py +2 -3
  5. {bw_processing-1.6 → bw_processing-2.0}/src/bw_processing/array_creation.py +10 -51
  6. {bw_processing-1.6 → bw_processing-2.0}/src/bw_processing/constants.py +5 -5
  7. {bw_processing-1.6 → bw_processing-2.0}/src/bw_processing/datapackage.py +112 -50
  8. {bw_processing-1.6 → bw_processing-2.0}/src/bw_processing/io_pyarrow_helpers.py +4 -1
  9. {bw_processing-1.6 → bw_processing-2.0}/src/bw_processing/matrix_entry.py +66 -12
  10. {bw_processing-1.6 → bw_processing-2.0}/src/bw_processing/merging.py +2 -2
  11. {bw_processing-1.6 → bw_processing-2.0}/src/bw_processing/utils.py +20 -6
  12. {bw_processing-1.6 → bw_processing-2.0/src/bw_processing.egg-info}/PKG-INFO +5 -7
  13. {bw_processing-1.6 → bw_processing-2.0}/src/bw_processing.egg-info/requires.txt +2 -2
  14. bw_processing-2.0/tests/test_array_creation.py +44 -0
  15. {bw_processing-1.6 → bw_processing-2.0}/tests/test_datapackage.py +101 -0
  16. {bw_processing-1.6 → bw_processing-2.0}/tests/test_filtered_datapackage.py +43 -0
  17. {bw_processing-1.6 → bw_processing-2.0}/tests/test_io_parquet_helpers.py +0 -8
  18. {bw_processing-1.6 → bw_processing-2.0}/tests/test_io_pyarrow_helpers.py +15 -0
  19. {bw_processing-1.6 → bw_processing-2.0}/tests/test_matrix_entry.py +222 -2
  20. {bw_processing-1.6 → bw_processing-2.0}/tests/test_merging.py +9 -0
  21. bw_processing-1.6/tests/test_array_creation.py +0 -97
  22. {bw_processing-1.6 → bw_processing-2.0}/LICENSE +0 -0
  23. {bw_processing-1.6 → bw_processing-2.0}/MANIFEST.in +0 -0
  24. {bw_processing-1.6 → bw_processing-2.0}/setup.cfg +0 -0
  25. {bw_processing-1.6 → bw_processing-2.0}/src/bw_processing/errors.py +0 -0
  26. {bw_processing-1.6 → bw_processing-2.0}/src/bw_processing/examples/__init__.py +0 -0
  27. {bw_processing-1.6 → bw_processing-2.0}/src/bw_processing/examples/datapackage_1/another name.indices.parquet +0 -0
  28. {bw_processing-1.6 → bw_processing-2.0}/src/bw_processing/examples/datapackage_1/sa-data-vector-from-dict.data.parquet +0 -0
  29. {bw_processing-1.6 → bw_processing-2.0}/src/bw_processing/examples/datapackage_1/sa-data-vector-from-dict.distributions.parquet +0 -0
  30. {bw_processing-1.6 → bw_processing-2.0}/src/bw_processing/examples/datapackage_1/sa-data-vector-from-dict.flip.parquet +0 -0
  31. {bw_processing-1.6 → bw_processing-2.0}/src/bw_processing/examples/datapackage_1/sa-data-vector-from-dict.indices.parquet +0 -0
  32. {bw_processing-1.6 → bw_processing-2.0}/src/bw_processing/examples/datapackage_1/some name.data.parquet +0 -0
  33. {bw_processing-1.6 → bw_processing-2.0}/src/bw_processing/examples/datapackage_1/some name.flip.parquet +0 -0
  34. {bw_processing-1.6 → bw_processing-2.0}/src/bw_processing/examples/datapackage_1/some name.indices.parquet +0 -0
  35. {bw_processing-1.6 → bw_processing-2.0}/src/bw_processing/examples/datapackage_2.zip +0 -0
  36. {bw_processing-1.6 → bw_processing-2.0}/src/bw_processing/examples/interfaces.py +0 -0
  37. {bw_processing-1.6 → bw_processing-2.0}/src/bw_processing/examples/parquet_files.py +0 -0
  38. {bw_processing-1.6 → bw_processing-2.0}/src/bw_processing/examples/simple.zip +0 -0
  39. {bw_processing-1.6 → bw_processing-2.0}/src/bw_processing/filesystem.py +0 -0
  40. {bw_processing-1.6 → bw_processing-2.0}/src/bw_processing/indexing.py +0 -0
  41. {bw_processing-1.6 → bw_processing-2.0}/src/bw_processing/io_helpers.py +0 -0
  42. {bw_processing-1.6 → bw_processing-2.0}/src/bw_processing/io_parquet_helpers.py +0 -0
  43. {bw_processing-1.6 → bw_processing-2.0}/src/bw_processing/param_labels.py +0 -0
  44. {bw_processing-1.6 → bw_processing-2.0}/src/bw_processing/proxies.py +0 -0
  45. {bw_processing-1.6 → bw_processing-2.0}/src/bw_processing/unique_fields.py +0 -0
  46. {bw_processing-1.6 → bw_processing-2.0}/src/bw_processing.egg-info/SOURCES.txt +0 -0
  47. {bw_processing-1.6 → bw_processing-2.0}/src/bw_processing.egg-info/dependency_links.txt +0 -0
  48. {bw_processing-1.6 → bw_processing-2.0}/src/bw_processing.egg-info/top_level.txt +0 -0
  49. {bw_processing-1.6 → bw_processing-2.0}/tests/test_filesystem.py +0 -0
  50. {bw_processing-1.6 → bw_processing-2.0}/tests/test_indexing.py +0 -0
  51. {bw_processing-1.6 → bw_processing-2.0}/tests/test_integration.py +0 -0
  52. {bw_processing-1.6 → bw_processing-2.0}/tests/test_interfaces.py +0 -0
  53. {bw_processing-1.6 → bw_processing-2.0}/tests/test_loading.py +0 -0
  54. {bw_processing-1.6 → bw_processing-2.0}/tests/test_params.py +0 -0
  55. {bw_processing-1.6 → bw_processing-2.0}/tests/test_proxies.py +0 -0
  56. {bw_processing-1.6 → bw_processing-2.0}/tests/test_unique_fields.py +0 -0
  57. {bw_processing-1.6 → bw_processing-2.0}/tests/test_utils.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: bw_processing
3
- Version: 1.6
3
+ Version: 2.0
4
4
  Summary: Foo
5
5
  Author-email: Chris Mutel <cmutel@gmail.com>
6
6
  Maintainer-email: Chris Mutel <cmutel@gmail.com>
@@ -10,20 +10,18 @@ Project-URL: tracker, https://github.com/brightway-lca/bw_processing/issues
10
10
  Classifier: Development Status :: 4 - Beta
11
11
  Classifier: Programming Language :: Python
12
12
  Classifier: Programming Language :: Python :: 3
13
- Classifier: Programming Language :: Python :: 3.9
14
- Classifier: Programming Language :: Python :: 3.10
15
13
  Classifier: Programming Language :: Python :: 3.11
16
14
  Classifier: Programming Language :: Python :: 3.12
17
15
  Classifier: Natural Language :: English
18
16
  Classifier: Operating System :: OS Independent
19
17
  Classifier: Topic :: Scientific/Engineering
20
- Requires-Python: >=3.9
18
+ Requires-Python: >=3.11
21
19
  Description-Content-Type: text/markdown
22
20
  License-File: LICENSE
23
- Requires-Dist: fsspec
21
+ Requires-Dist: fsspec>=2023.3.0
24
22
  Requires-Dist: jsonschema>=4.0
25
23
  Requires-Dist: morefs
26
- Requires-Dist: numpy
24
+ Requires-Dist: numpy>=2.0
27
25
  Requires-Dist: pandas
28
26
  Provides-Extra: testing
29
27
  Requires-Dist: bw_processing; extra == "testing"
@@ -237,7 +235,7 @@ dp.add_persistent_vector(
237
235
  )
238
236
  ```
239
237
 
240
- The stored resource has `kind="rescale"` and can be retrieved via `dp.get_resource("my-process.rescale")`. The `rescale_array` must be a float dtype (`float32` or `float64`); passing an integer array raises `WrongDatatype`.
238
+ The stored resource has `kind="rescale"` and can be retrieved via `dp.get_resource("my-process.rescale")`. The `rescale_array` must be a float dtype and is always stored as `float64`; passing an integer array raises `WrongDatatype`.
241
239
 
242
240
  ### Reference (production) exchanges
243
241
 
@@ -194,7 +194,7 @@ dp.add_persistent_vector(
194
194
  )
195
195
  ```
196
196
 
197
- The stored resource has `kind="rescale"` and can be retrieved via `dp.get_resource("my-process.rescale")`. The `rescale_array` must be a float dtype (`float32` or `float64`); passing an integer array raises `WrongDatatype`.
197
+ The stored resource has `kind="rescale"` and can be retrieved via `dp.get_resource("my-process.rescale")`. The `rescale_array` must be a float dtype and is always stored as `float64`; passing an integer array raises `WrongDatatype`.
198
198
 
199
199
  ### Reference (production) exchanges
200
200
 
@@ -19,24 +19,21 @@ classifiers = [
19
19
  "Development Status :: 4 - Beta",
20
20
  "Programming Language :: Python",
21
21
  "Programming Language :: Python :: 3",
22
- "Programming Language :: Python :: 3.9",
23
- "Programming Language :: Python :: 3.10",
24
22
  "Programming Language :: Python :: 3.11",
25
23
  "Programming Language :: Python :: 3.12",
26
24
  "Natural Language :: English",
27
25
  "Operating System :: OS Independent",
28
26
  "Topic :: Scientific/Engineering"
29
27
  ]
30
- requires-python = ">=3.9"
28
+ requires-python = ">=3.11"
31
29
  dependencies = [
32
30
  # dependencies as strings with quotes, e.g. "foo"
33
31
  # You can add version requirements
34
- "fsspec",
32
+ "fsspec>=2023.3.0",
35
33
  "jsonschema>=4.0",
36
34
  "morefs",
37
- "numpy",
35
+ "numpy>=2.0",
38
36
  "pandas",
39
-
40
37
  ]
41
38
 
42
39
  [project.urls]
@@ -4,7 +4,6 @@ __all__ = (
4
4
  "as_unique_attributes",
5
5
  "as_unique_attributes_dataframe",
6
6
  "clean_datapackage_name",
7
- "create_array",
8
7
  "create_datapackage",
9
8
  "create_datapackage_from_entries",
10
9
  "create_structured_array",
@@ -35,10 +34,10 @@ __all__ = (
35
34
  "UndefinedInterface",
36
35
  )
37
36
 
38
- __version__ = "1.6"
37
+ __version__ = "2.0"
39
38
 
40
39
 
41
- from bw_processing.array_creation import create_array, create_structured_array
40
+ from bw_processing.array_creation import create_structured_array
42
41
  from bw_processing.constants import DEFAULT_LICENSES, INDICES_DTYPE, UNCERTAINTY_DTYPE, MatrixSerializeFormat
43
42
  from bw_processing.param_labels import (
44
43
  AnyLabelSchema,
@@ -3,17 +3,6 @@ import itertools
3
3
  import numpy as np
4
4
 
5
5
 
6
- def peek(iterator):
7
- iterator = iter(iterator)
8
- first = next(iterator)
9
- return first, itertools.chain([first], iterator)
10
-
11
-
12
- def get_ncols(iterator):
13
- a, b = peek(iterator)
14
- return len(a), b
15
-
16
-
17
6
  def chunked(iterable, chunk_size):
18
7
  # Black magic, see https://stackoverflow.com/a/31185097
19
8
  # and https://docs.python.org/3/library/functions.html#iter
@@ -21,8 +10,8 @@ def chunked(iterable, chunk_size):
21
10
  return iter(lambda: list(itertools.islice(iterable, chunk_size)), [])
22
11
 
23
12
 
24
- def create_chunked(iterable, dtype, ncols=None, bucket_size=500):
25
- """Create a numpy array from an iterable of indeterminate length.
13
+ def create_chunked(iterable, dtype, bucket_size=500):
14
+ """Create a 1D numpy structured array from an iterable of indeterminate length.
26
15
 
27
16
  Needed when we can't determine the length of the iterable ahead of time
28
17
  (e.g. for a generator or a database cursor), so can't create the complete
@@ -31,21 +20,16 @@ def create_chunked(iterable, dtype, ncols=None, bucket_size=500):
31
20
  Creates a list of arrays with ``bucket_size`` rows until ``iterable`` is
32
21
  exhausted, then concatenates them along axis 0.
33
22
 
34
- Pass ``ncols`` for a plain 2D array; omit it for a 1D structured array.
35
-
36
23
  Args:
37
24
  iterable: Iterable of data used to populate the array.
38
25
  dtype: Numpy dtype of the created array.
39
- ncols: Number of columns; if None, a 1D structured array is created.
40
26
  bucket_size: Number of rows in each intermediate array.
41
27
 
42
28
  Returns:
43
29
  The created array. Returns a zero-length array if ``iterable`` has no data.
44
30
  """
45
- bucket_shape = (bucket_size,) if ncols is None else (bucket_size, ncols)
46
- empty_shape = (0,) if ncols is None else (0, ncols)
47
31
  arrays = []
48
- array = np.zeros(bucket_shape, dtype=dtype)
32
+ array = np.zeros(bucket_size, dtype=dtype)
49
33
  for chunk in chunked(iterable, bucket_size):
50
34
  for i, row in enumerate(chunk):
51
35
  array[i] = row
@@ -55,11 +39,11 @@ def create_chunked(iterable, dtype, ncols=None, bucket_size=500):
55
39
  arrays.append(array[: i + 1].copy())
56
40
  else:
57
41
  arrays.append(array)
58
- array = np.zeros(bucket_shape, dtype=dtype)
42
+ array = np.zeros(bucket_size, dtype=dtype)
59
43
  # Empty iterable - create zero-length array.
60
44
  # Needed because we return iterators for SQL databases
61
45
  # but don't know if e.g. sometimes a database has no biosphere exchanges.
62
- return np.concatenate(arrays, axis=0) if arrays else np.zeros(empty_shape, dtype=dtype)
46
+ return np.concatenate(arrays, axis=0) if arrays else np.zeros(0, dtype=dtype)
63
47
 
64
48
 
65
49
  def create_structured_array(iterable, dtype, nrows=None, sort=False, sort_fields=None):
@@ -73,10 +57,15 @@ def create_structured_array(iterable, dtype, nrows=None, sort=False, sort_fields
73
57
  if not nrows:
74
58
  nrows = len(iterable)
75
59
  array = np.zeros(nrows, dtype=dtype)
60
+ count = 0
76
61
  for i, row in enumerate(iterable):
77
62
  if i > (nrows - 1):
78
63
  raise ValueError("More rows than `nrows`")
79
64
  array[i] = tuple(row)
65
+ count = i + 1
66
+ if count < nrows:
67
+ # Unfilled rows would be stored as zeros (row 0, col 0, amount 0)
68
+ raise ValueError(f"Fewer rows ({count}) than `nrows` ({nrows})")
80
69
 
81
70
  else:
82
71
  array = create_chunked(iterable, dtype)
@@ -90,33 +79,3 @@ def create_structured_array(iterable, dtype, nrows=None, sort=False, sort_fields
90
79
  array.sort(order=order)
91
80
 
92
81
  return array
93
-
94
-
95
- def create_array(iterable, nrows=None, dtype=np.float32):
96
- """Create a numpy array data ``iterable``. Returns a filepath of a created file (if ``filepath`` is provided, or the array.
97
-
98
- ``iterable`` can be data already in memory, or a generator.
99
-
100
- ``nrows`` can be supplied, if known. If ``iterable`` has a length, it will be determined automatically. If ``nrows`` is not known, this function generates chunked arrays until ``iterable`` is exhausted, and concatenates them.
101
-
102
- Either ``nrows`` or ``ncols`` must be specified."""
103
- if isinstance(iterable, np.ndarray):
104
- array = iterable.astype(dtype)
105
- elif nrows or hasattr(iterable, "__len__"):
106
- if not nrows:
107
- nrows = len(iterable)
108
- ncols, data = get_ncols(iterable)
109
- array = np.zeros((nrows, ncols), dtype=dtype)
110
- for i, row in enumerate(data):
111
- if i > (nrows - 1):
112
- raise ValueError("More rows than `nrows`")
113
- array[i, :] = tuple(row)
114
-
115
- else:
116
- try:
117
- ncols, data = get_ncols(iterable)
118
- except StopIteration:
119
- return np.zeros((0, 0), dtype=dtype)
120
- array = create_chunked(data, dtype, ncols=ncols)
121
-
122
- return array
@@ -14,11 +14,11 @@ MAX_SIGNED_64BIT_INT = 9223372036854775807
14
14
  # but -687ms to append two columns.
15
15
  UNCERTAINTY_DTYPE = [
16
16
  ("uncertainty_type", np.uint8),
17
- ("loc", np.float32),
18
- ("scale", np.float32),
19
- ("shape", np.float32),
20
- ("minimum", np.float32),
21
- ("maximum", np.float32),
17
+ ("loc", np.float64),
18
+ ("scale", np.float64),
19
+ ("shape", np.float64),
20
+ ("minimum", np.float64),
21
+ ("maximum", np.float64),
22
22
  ("negative", bool),
23
23
  ]
24
24
  INDICES_DTYPE = [("row", np.int64), ("col", np.int64)]
@@ -2,8 +2,9 @@ import datetime
2
2
  import uuid
3
3
  import warnings
4
4
  from abc import ABC
5
+ from collections.abc import Sequence
5
6
  from functools import partial
6
- from typing import TYPE_CHECKING, Any, Dict, List, Optional, Union
7
+ from typing import TYPE_CHECKING, Any, Dict, Iterable, List, Optional, Union
7
8
 
8
9
  if TYPE_CHECKING:
9
10
  from bw_processing.matrix_entry import ArrayEntry, MatrixEntry
@@ -30,6 +31,7 @@ from bw_processing.constants import (
30
31
  NUMPY_SERIALIZE_FORMAT_NAME,
31
32
  PARQUET_SERIALIZE_FORMAT_EXTENSION,
32
33
  PARQUET_SERIALIZE_FORMAT_NAME,
34
+ UNCERTAINTY_DTYPE,
33
35
  MatrixSerializeFormat,
34
36
  )
35
37
  from bw_processing.errors import (
@@ -45,7 +47,14 @@ from bw_processing.filesystem import clean_datapackage_name
45
47
  from bw_processing.io_helpers import file_reader, file_writer
46
48
  from bw_processing.param_labels import AnyLabelSchema
47
49
  from bw_processing.proxies import Proxy, UndefinedInterface
48
- from bw_processing.utils import check_name, check_suffix, load_bytes, resolve_dict_iterator, utc_now
50
+ from bw_processing.utils import (
51
+ check_name,
52
+ check_suffix,
53
+ load_bytes,
54
+ resolve_dict_iterator,
55
+ resolve_tuple_iterator,
56
+ utc_now,
57
+ )
49
58
 
50
59
 
51
60
  class DatapackageBase(ABC):
@@ -68,7 +77,11 @@ class DatapackageBase(ABC):
68
77
  return len(self.data)
69
78
 
70
79
  def __contains__(self, key):
71
- return key in {o["name"] for o in self.resources} or key in self.groups
80
+ return key in {o["name"] for o in self.resources} or self._has_group(key)
81
+
82
+ def _has_group(self, label: Any) -> bool:
83
+ """Check for a resource group label without building the filtered ``groups``."""
84
+ return bool(label) and any(o.get("group") == label for o in self.resources)
72
85
 
73
86
  resources = property(__get_resources, __set_resources)
74
87
 
@@ -78,14 +91,14 @@ class DatapackageBase(ABC):
78
91
 
79
92
  Ignores resources which don't have group labels.
80
93
  """
81
- labels = []
82
- for resource in self.resources:
83
- if not resource.get("group"):
84
- continue
85
- elif resource["group"] not in labels:
86
- labels.append(resource["group"])
94
+ # One pass over the resources: filtering once per label made this
95
+ # quadratic in the number of resources.
96
+ indices = {}
97
+ for i, resource in enumerate(self.resources):
98
+ if resource.get("group"):
99
+ indices.setdefault(resource["group"], []).append(i)
87
100
 
88
- return {label: self.filter_by_attribute("group", label) for label in labels}
101
+ return {label: self._filtered(lst) for label, lst in indices.items()}
89
102
 
90
103
  def _get_index(self, name_or_index: Union[str, int]) -> int:
91
104
  """Get index of a resource by name or index.
@@ -144,7 +157,7 @@ class DatapackageBase(ABC):
144
157
  if self._modified:
145
158
  raise PotentialInconsistency("Datapackage is modified; save modifications or reload")
146
159
 
147
- indices = [i for i, resource in enumerate(self.resources) if resource.get("group") == name]
160
+ indices = {i for i, resource in enumerate(self.resources) if resource.get("group") == name}
148
161
 
149
162
  for obj in (obj for i, obj in enumerate(self.resources) if i in indices):
150
163
  try:
@@ -188,16 +201,9 @@ class DatapackageBase(ABC):
188
201
 
189
202
  This method was introduced to allow for the efficient construction of matrices; each datapackage can have data for multiple matrices, and we can then create filtered datapackages which exclusively have data for the matrix of interest. As such, they should be considered read-only, though this is not enforced.
190
203
  """
191
- fdp = FilteredDatapackage()
192
- fdp.fs = self.fs
193
- fdp.metadata = {k: v for k, v in self.metadata.items() if k != "resources"}
194
- fdp.metadata["resources"] = []
195
- to_include = [i for i, resource in enumerate(self.resources) if resource.get(key) == value]
196
- fdp.data = [o for i, o in enumerate(self.data) if i in to_include]
197
- fdp.resources = [o for i, o in enumerate(self.resources) if i in to_include]
198
- if hasattr(self, "indexer"):
199
- fdp.indexer = self.indexer
200
- return fdp
204
+ return self._filtered(
205
+ [i for i, resource in enumerate(self.resources) if resource.get(key) == value]
206
+ )
201
207
 
202
208
  def exclude(self, filters: Dict[str, str]) -> "FilteredDatapackage":
203
209
  """Filter a datapackage to exclude resources matching a filter.
@@ -213,21 +219,33 @@ class DatapackageBase(ABC):
213
219
  exclude_generic({"group': "some_group", "kind": "some_kind"})
214
220
 
215
221
  """
222
+ return self._filtered(
223
+ [
224
+ i
225
+ for i, resource in enumerate(self.resources)
226
+ if any(resource.get(key) != value for key, value in filters.items())
227
+ ]
228
+ )
229
+
230
+ def _check_length_consistency(self) -> None:
231
+ if len(self.resources) != len(self.data):
232
+ raise LengthMismatch(
233
+ "Number of resources ({}) doesn't match number of data objects ({})".format(
234
+ len(self.resources), len(self.data)
235
+ )
236
+ )
237
+
238
+ def _filtered(self, indices: List[int]) -> "FilteredDatapackage":
239
+ """Create a ``FilteredDatapackage`` with the resources at ``indices``, sharing their data."""
240
+ self._check_length_consistency()
216
241
  fdp = FilteredDatapackage()
217
242
  fdp.fs = self.fs
218
243
  fdp.metadata = {k: v for k, v in self.metadata.items() if k != "resources"}
219
244
  fdp.metadata["resources"] = []
220
-
245
+ fdp.data = [self.data[i] for i in indices]
246
+ fdp.resources = [self.resources[i] for i in indices]
221
247
  if hasattr(self, "indexer"):
222
248
  fdp.indexer = self.indexer
223
-
224
- indices_to_include = [
225
- i
226
- for i, resource in enumerate(self.resources)
227
- if any(resource.get(key) != value for key, value in filters.items())
228
- ]
229
- fdp.data = [o for i, o in enumerate(self.data) if i in indices_to_include]
230
- fdp.resources = [o for i, o in enumerate(self.resources) if i in indices_to_include]
231
249
  return fdp
232
250
 
233
251
  def _dehydrate_interfaces(self) -> None:
@@ -307,14 +325,6 @@ class Datapackage(DatapackageBase):
307
325
  def __eq__(self, other):
308
326
  return (id(self.fs), self.metadata) == (id(other.fs), other.metadata)
309
327
 
310
- def _check_length_consistency(self) -> None:
311
- if len(self.resources) != len(self.data):
312
- raise LengthMismatch(
313
- "Number of resources ({}) doesn't match number of data objects ({})".format(
314
- len(self.resources), len(self.data)
315
- )
316
- )
317
-
318
328
  def _load(
319
329
  self, fs: AbstractFileSystem, mmap_mode: Optional[str] = None, proxy: bool = False
320
330
  ) -> None:
@@ -396,7 +406,7 @@ class Datapackage(DatapackageBase):
396
406
  A distributions resource is considered trivial when every row has an
397
407
  ``uncertainty_type`` of 0 (no uncertainty) or 1 (unknown uncertainty)
398
408
  *and* the ``loc`` values are equal to the corresponding data-array
399
- amounts (at float32 precision). Such a resource duplicates what the
409
+ amounts. Such a resource duplicates what the
400
410
  data array already encodes, so it is deleted from the filesystem and
401
411
  removed from ``self.data`` / ``self.resources``.
402
412
  """
@@ -430,7 +440,7 @@ class Datapackage(DatapackageBase):
430
440
  data_arr = data_arr()
431
441
 
432
442
  all_trivial = np.all(np.isin(dist_arr["uncertainty_type"], _NO_UNCERTAINTY_IDS))
433
- loc_matches = np.array_equal(dist_arr["loc"], data_arr.astype(np.float32))
443
+ loc_matches = np.array_equal(dist_arr["loc"], data_arr)
434
444
 
435
445
  if all_trivial and loc_matches:
436
446
  to_remove.append(idx)
@@ -505,7 +515,31 @@ class Datapackage(DatapackageBase):
505
515
  """Create a persistant vector from an iterator. Uses the utility function ``resolve_dict_iterator``.
506
516
 
507
517
  This is the **only array creation method which produces sorted arrays**."""
518
+ # Check the name before consuming the iterator, which might not be
519
+ # restartable.
508
520
  name = self._prepare_name(name)
521
+ self._add_persistent_vector_from_resolved(
522
+ resolve_dict_iterator(dict_iterator, nrows),
523
+ matrix=matrix,
524
+ name=name,
525
+ matrix_serialize_format_type=matrix_serialize_format_type,
526
+ **kwargs,
527
+ )
528
+
529
+ def _add_persistent_vector_from_resolved(
530
+ self,
531
+ resolved: tuple,
532
+ *,
533
+ matrix: str,
534
+ name: str,
535
+ matrix_serialize_format_type: Optional[MatrixSerializeFormat] = None,
536
+ **kwargs,
537
+ ) -> None:
538
+ """Add a persistent vector from the output of ``resolve_dict_iterator``
539
+ or ``resolve_tuple_iterator``.
540
+
541
+ ``name`` must already have been checked with ``_prepare_name``, so
542
+ that a bad name is rejected before the iterator is consumed."""
509
543
  (
510
544
  data_array,
511
545
  indices_array,
@@ -513,7 +547,7 @@ class Datapackage(DatapackageBase):
513
547
  flip_array,
514
548
  rescale_array,
515
549
  reference_array,
516
- ) = resolve_dict_iterator(dict_iterator, nrows)
550
+ ) = resolved
517
551
  self.add_persistent_vector(
518
552
  matrix=matrix,
519
553
  name=name,
@@ -532,10 +566,11 @@ class Datapackage(DatapackageBase):
532
566
  self,
533
567
  *,
534
568
  matrix: str,
535
- entries: list["MatrixEntry"],
569
+ entries: Iterable["MatrixEntry"],
536
570
  name: Optional[str] = None,
571
+ nrows: Optional[int] = None,
537
572
  ) -> None:
538
- """Add matrix data from a list of :class:`MatrixEntry` objects.
573
+ """Add matrix data from :class:`MatrixEntry` objects.
539
574
 
540
575
  High-level convenience method that does not require working directly
541
576
  with NumPy arrays. If any entry has a ``rescale`` value other than
@@ -543,16 +578,32 @@ class Datapackage(DatapackageBase):
543
578
  If any entry has ``reference=True``, the reference flags are stored as
544
579
  a ``reference_array`` resource (``kind="reference"``).
545
580
 
581
+ ``entries`` can be a list or any iterable, such as a generator
582
+ streaming rows from a database cursor. Entries are converted with
583
+ :meth:`.MatrixEntry.as_tuple`, skipping the dict round trip of
584
+ :meth:`add_persistent_vector_from_iterator`.
585
+
546
586
  Args:
547
587
  matrix: Name of the target matrix (e.g. ``"technosphere"``).
548
- entries: List of :class:`.MatrixEntry` instances.
588
+ entries: Iterable of :class:`.MatrixEntry` instances.
549
589
  name: Optional resource group name; auto-generated if omitted.
590
+ nrows: Exact number of entries, if known and ``entries`` is not a
591
+ sequence. Lets the array be allocated in one step; otherwise
592
+ it is built in chunks. Raises ``ValueError`` if ``entries``
593
+ yields more or fewer than ``nrows`` entries.
550
594
  """
551
- self.add_persistent_vector_from_iterator(
595
+ # Only take the length of real sequences. Other objects with
596
+ # ``__len__`` can be lazy: ``len()`` runs the whole query on an ORM
597
+ # query object, and raises for a ``tqdm`` wrapper with no total.
598
+ if nrows is None and isinstance(entries, Sequence):
599
+ nrows = len(entries)
600
+ # Check the name before consuming the iterator, which might not be
601
+ # restartable.
602
+ name = self._prepare_name(name)
603
+ self._add_persistent_vector_from_resolved(
604
+ resolve_tuple_iterator((e.as_tuple() for e in entries), nrows),
552
605
  matrix=matrix,
553
606
  name=name,
554
- dict_iterator=(e.as_dict() for e in entries),
555
- nrows=len(entries),
556
607
  )
557
608
 
558
609
  def add_array_entries(
@@ -674,6 +725,15 @@ class Datapackage(DatapackageBase):
674
725
  )
675
726
  if distributions_array is not None:
676
727
  distributions_array = load_bytes(distributions_array)
728
+ if distributions_array.dtype != UNCERTAINTY_DTYPE:
729
+ # Always store float fields as float64, e.g. for arrays built
730
+ # with the float32 fields used before version 1.7
731
+ if distributions_array.dtype.names != np.dtype(UNCERTAINTY_DTYPE).names:
732
+ raise WrongDatatype(
733
+ "`distributions_array` dtype is {}, but must have the fields of "
734
+ "`UNCERTAINTY_DTYPE`".format(distributions_array.dtype)
735
+ )
736
+ distributions_array = distributions_array.astype(UNCERTAINTY_DTYPE)
677
737
  # If no uncertainty, don't need to store it
678
738
  if (distributions_array["uncertainty_type"] < 2).sum() < len(distributions_array):
679
739
  if distributions_array.shape != indices_array.shape:
@@ -1007,7 +1067,9 @@ class Datapackage(DatapackageBase):
1007
1067
  raise WrongDatatype(
1008
1068
  "`rescale_array` dtype is {}, but must be a float dtype".format(rescale_array.dtype)
1009
1069
  )
1010
- elif rescale_array.shape != indices_array.shape:
1070
+ # Always store rescale factors as float64, same as `ArrayEntry.rescale`
1071
+ rescale_array = rescale_array.astype(np.float64, copy=False)
1072
+ if rescale_array.shape != indices_array.shape:
1011
1073
  raise ShapeMismatch(
1012
1074
  "`rescale_array` shape ({}) doesn't match `indices_array` ({}).".format(
1013
1075
  rescale_array.shape, indices_array.shape
@@ -1537,7 +1599,7 @@ class Datapackage(DatapackageBase):
1537
1599
  assert isinstance(dataframe, pd.DataFrame)
1538
1600
  assert isinstance(valid_for, list)
1539
1601
 
1540
- assert all(x in self.groups for x, y in valid_for)
1602
+ assert all(self._has_group(x) for x, y in valid_for)
1541
1603
 
1542
1604
  name = self._prepare_name(name)
1543
1605
  self._prepare_modifications()
@@ -1587,7 +1649,7 @@ class Datapackage(DatapackageBase):
1587
1649
 
1588
1650
  """
1589
1651
  assert isinstance(valid_for, str)
1590
- assert valid_for in self.groups
1652
+ assert self._has_group(valid_for)
1591
1653
 
1592
1654
  self._prepare_modifications()
1593
1655
 
@@ -181,7 +181,10 @@ def numpy_distributions_vector_to_pyarrow_distributions_vector_table(
181
181
  """
182
182
  assert isinstance(arr, np.ndarray)
183
183
  assert arr.ndim == 1
184
- assert arr.dtype == UNCERTAINTY_DTYPE
184
+ if arr.dtype != UNCERTAINTY_DTYPE:
185
+ # Arrays built with the float32 fields used before version 1.7
186
+ assert arr.dtype.names == np.dtype(UNCERTAINTY_DTYPE).names
187
+ arr = arr.astype(UNCERTAINTY_DTYPE)
185
188
 
186
189
  from_dict = {}
187
190
  for i in range(NBR_UNCERTAINTY_FIELDS):