matrix-utils 0.6.3__tar.gz → 0.7.0__tar.gz

{matrix_utils-0.6.3 → matrix_utils-0.7.0}/CHANGELOG.md

@@ -1,5 +1,9 @@
 # Changelog
 
+### [0.7.0] - 2026-02-27
+
+* Merge [#33](https://github.com/brightway-lca/matrix_utils/pull/33): Speed up large-index mapping by avoiding repeated unique and using fast integer dedupicator. Thanks @romainsacchi!
+
 ### [0.6.3] - 2026-02-23
 
 * Fix [#32](https://github.com/brightway-lca/matrix_utils/issues/32): Replace deprecated `.A1` property on scipy sparse matrices

{matrix_utils-0.6.3 → matrix_utils-0.7.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: matrix_utils
-Version: 0.6.3
+Version: 0.7.0
 Summary: Tools to create matrices from data packages
 Author-email: Chris Mutel <cmutel@gmail.com>
 Maintainer-email: Chris Mutel <cmutel@gmail.com>

{matrix_utils-0.6.3 → matrix_utils-0.7.0}/matrix_utils/__init__.py

@@ -11,7 +11,7 @@ __all__ = (
     "SparseMatrixDict",
 )
 
-__version__ = "0.6.3"
+__version__ = "0.7.0"
 
 from matrix_utils.array_mapper import ArrayMapper
 from matrix_utils.indexers import CombinatorialIndexer, Proxy, RandomIndexer, SequentialIndexer

{matrix_utils-0.6.3 → matrix_utils-0.7.0}/matrix_utils/array_mapper.py

@@ -1,4 +1,5 @@
 import numpy as np
+import pandas as pd
 from scipy import sparse
 
 from matrix_utils.errors import EmptyArray
@@ -25,10 +26,21 @@ class ArrayMapper:
     """  # NOQA: E501
 
     def __init__(self, *, array: np.ndarray, sparse_cutoff: int = 50000, empty_ok: bool = False):
+
+
         self._check_input_array(array)
 
-        # Even if already unique, this only adds ~2ms for 100.000 elements
-        self.array = np.unique(array)
+        # `np.unique` can be very slow on large, unsorted integer arrays.
+        # Use hash-based `pandas.unique` first, then sort only the unique values.
+        # Fall back to NumPy if anything unexpected occurs.
+        if array.dtype.kind in {"i", "u"} and array.shape[0] > 100_000:
+            try:
+                self.array = np.sort(pd.unique(array))
+            except (TypeError, ValueError):
+                self.array = np.unique(array)
+        else:
+            self.array = np.unique(array)
+
         self.empty_ok = empty_ok
 
         if self.array.shape == (0,):

{matrix_utils-0.6.3 → matrix_utils-0.7.0}/matrix_utils/mapped_matrix.py

@@ -115,20 +115,36 @@ class MappedMatrix:
         self.add_indexers(indexer_override, seed_override)
 
         try:
-            self.row_mapper = row_mapper or ArrayMapper(
-                array=safe_concatenate_indices(
-                    [obj.unique_row_indices_for_mapping() for obj in self.groups], empty_ok
-                ),
-                empty_ok=empty_ok,
-            )
-            if not diagonal:
-                self.col_mapper = col_mapper or ArrayMapper(
-                    array=safe_concatenate_indices(
-                        [obj.unique_col_indices_for_mapping() for obj in self.groups],
-                        empty_ok,
-                    ),
+            if row_mapper is None:
+                row_arrays = []
+                for obj in self.groups:
+                    arr = obj.row_indices_for_mapping()
+                    row_arrays.append(arr)
+
+                row_values = safe_concatenate_indices(row_arrays, empty_ok)
+
+                self.row_mapper = ArrayMapper(
+                    array=row_values,
                     empty_ok=empty_ok,
                 )
+            else:
+                self.row_mapper = row_mapper
+
+            if not diagonal:
+                if col_mapper is None:
+                    col_arrays = []
+                    for obj in self.groups:
+                        arr = obj.col_indices_for_mapping()
+                        col_arrays.append(arr)
+
+                    col_values = safe_concatenate_indices(col_arrays, empty_ok)
+
+                    self.col_mapper = ArrayMapper(
+                        array=col_values,
+                        empty_ok=empty_ok,
+                    )
+                else:
+                    self.col_mapper = col_mapper
         except AllArraysEmpty:
             handle_all_arrays_empty(
                 packages=self.packages, matrix_label=self.matrix_label, identifier=identifier
@@ -234,7 +250,7 @@ class MappedMatrix:
                     obj.add_indexer(indexer=package.indexer)
             else:
                 package.indexer = RandomIndexer(
-                    seed=seed_override or package.metadata["seed"],
+                    seed=seed_override if seed_override is not None else package.metadata["seed"],
                     max_value=package.get_max_index_value(),
                 )
                 for obj in resources:

{matrix_utils-0.6.3 → matrix_utils-0.7.0}/matrix_utils/resource_group.py

@@ -69,7 +69,9 @@ class ResourceGroup:
         self.custom_filter = custom_filter
         self.transpose = transpose
         self.vector = self.is_vector()
-        self.seed = seed_override or self.package.metadata.get("seed")
+        self.seed = (
+            seed_override if seed_override is not None else self.package.metadata.get("seed")
+        )
 
         if custom_filter is not None:
             self.custom_filter_mask = custom_filter(self.get_indices_data())
@@ -218,11 +220,29 @@ class ResourceGroup:
 
     def unique_row_indices_for_mapping(self):
         """Return array of unique indices that respect aggregation policy"""
-        return np.unique(mask_array(self.get_indices_data()["row"], self.custom_filter_mask))
+        data = mask_array(self.get_indices_data()["row"], self.custom_filter_mask)
+        # Structured-array field views can be non-contiguous; materialize contiguous memory first
+        # to avoid pathological performance in `np.unique`.
+        return np.unique(np.ascontiguousarray(data))
 
     def unique_col_indices_for_mapping(self):
         """Return array of unique indices that respect aggregation policy"""
-        return np.unique(mask_array(self.get_indices_data()["col"], self.custom_filter_mask))
+        data = mask_array(self.get_indices_data()["col"], self.custom_filter_mask)
+        # Structured-array field views can be non-contiguous; materialize contiguous memory first
+        # to avoid pathological performance in `np.unique`.
+        return np.unique(np.ascontiguousarray(data))
+
+    def row_indices_for_mapping(self):
+        """Return raw row indices for mapping, with custom filter applied if present."""
+        data = mask_array(self.get_indices_data()["row"], self.custom_filter_mask)
+        # Force contiguous memory to improve downstream sorting/uniquing performance.
+        return np.ascontiguousarray(data)
+
+    def col_indices_for_mapping(self):
+        """Return raw column indices for mapping, with custom filter applied if present."""
+        data = mask_array(self.get_indices_data()["col"], self.custom_filter_mask)
+        # Force contiguous memory to improve downstream sorting/uniquing performance.
+        return np.ascontiguousarray(data)
 
     def add_indexer(self, indexer: Indexer):
         self.indexer = indexer

{matrix_utils-0.6.3 → matrix_utils-0.7.0}/matrix_utils.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: matrix_utils
-Version: 0.6.3
+Version: 0.7.0
 Summary: Tools to create matrices from data packages
 Author-email: Chris Mutel <cmutel@gmail.com>
 Maintainer-email: Chris Mutel <cmutel@gmail.com>

{matrix_utils-0.6.3 → matrix_utils-0.7.0}/tests/array_mapper.py

@@ -1,6 +1,7 @@
 import numpy as np
 import pytest
 
+import matrix_utils.array_mapper as array_mapper_module
 from matrix_utils.array_mapper import ArrayMapper
 from matrix_utils.errors import EmptyArray
 
@@ -78,3 +79,27 @@ def test_empty_array_ok():
     am = ArrayMapper(array=np.array([], dtype=int), empty_ok=True)
     assert am.array.shape == (0,)
     assert am.empty_input
+
+
+def test_empty_mapper_maps_to_missing_values():
+    am = ArrayMapper(array=np.array([], dtype=int), empty_ok=True)
+    result = am.map_array(np.array([0, 42], dtype=int))
+    assert np.array_equal(result, np.array([-1, -1]))
+
+
+def test_reverse_dict():
+    am = ArrayMapper(array=np.array([10, 20, 10, 30]))
+    assert am.reverse_dict() == {0: 10, 1: 20, 2: 30}
+
+
+def test_large_integer_array_pandas_fallback(monkeypatch):
+    # Trigger the large integer optimization branch and force fallback
+    inpt = np.tile(np.array([5, 2, 7, 2, 5], dtype=np.int64), 25_000)
+
+    def raise_type_error(_):
+        raise TypeError("boom")
+
+    monkeypatch.setattr(array_mapper_module.pd, "unique", raise_type_error)
+
+    am = ArrayMapper(array=inpt)
+    assert np.array_equal(am.array, np.array([2, 5, 7]))

{matrix_utils-0.6.3 → matrix_utils-0.7.0}/tests/mapped_matrix.py

@@ -842,3 +842,29 @@ def test_empty_combinatorial_datapackage():
     )
     for _ in range(10):
         next(mm)
+
+
+def test_input_indexer_vector_raises_on_unsupported_type():
+    class BadIndexer:
+        index = "not-an-index"
+
+        def __next__(self):
+            return self
+
+    s = bwp.create_datapackage(sequential=True)
+    s.add_persistent_vector(
+        matrix="foo",
+        data_array=np.arange(3),
+        indices_array=np.array([(0, 0), (1, 1), (0, 1)], dtype=bwp.INDICES_DTYPE),
+    )
+
+    mm = MappedMatrix(
+        packages=[s],
+        matrix="foo",
+        use_arrays=False,
+        use_distributions=False,
+        indexer_override=BadIndexer(),
+    )
+
+    with pytest.raises(ValueError, match="Can't understand indexer value"):
+        mm.input_indexer_vector()