matrix-utils 0.4.3__tar.gz → 0.6__tar.gz

{matrix_utils-0.4.3 → matrix_utils-0.6}/CHANGELOG.md

@@ -1,5 +1,13 @@
 # Changelog
 
+### [0.6] - 2024-11-01
+
+* New array mapping implementation using sparse matrices which is ~100x faster and allows for very large input values
+
+### [0.5] - 2024-08-20
+
+* Default is now 64 bit integers for indexing; compatible with `bw_processing` 1.0
+
 ### [0.4.3] - 2024-06-04
 
 * Allow `MappedMatrixDict` to take extra `kwargs` for compatiblity with custom matrix classes

{matrix_utils-0.4.3 → matrix_utils-0.6}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: matrix_utils
-Version: 0.4.3
+Version: 0.6
 Summary: Tools to create matrices from data packages
 Author-email: Chris Mutel <cmutel@gmail.com>
 Maintainer-email: Chris Mutel <cmutel@gmail.com>
@@ -26,9 +26,9 @@ Requires-Python: >=3.9
 Description-Content-Type: text/markdown
 License-File: LICENSE
 Requires-Dist: numpy
-Requires-Dist: scipy
+Requires-Dist: scipy>=1.10
 Requires-Dist: pandas
-Requires-Dist: bw_processing
+Requires-Dist: bw_processing>=1.0
 Requires-Dist: stats_arrays
 Provides-Extra: testing
 Requires-Dist: matrix_utils; extra == "testing"

{matrix_utils-0.4.3 → matrix_utils-0.6}/dev/Mapping speed tests.ipynb

@@ -9,7 +9,8 @@
    "source": [
     "import itertools\n",
     "import numpy as np\n",
-    "from scipy import sparse"
+    "from scipy import sparse\n",
+    "import matrix_utils as mu"
    ]
   },
   {
@@ -60,20 +61,120 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 3,
+   "execution_count": 14,
    "id": "wired-lesbian",
    "metadata": {},
    "outputs": [],
    "source": [
     "def index_with_indexarray(array_from):\n",
-    "    # Twice as fast as index_with_searchsorted\n",
-    "    unique = np.unique(array_from)\n",
-    "    values = np.arange(unique.max() + 1)\n",
+    "    array = np.unique(array_from)\n",
+    "    \n",
+    "    if array_from.shape == (0,):\n",
+    "        if empty_ok:\n",
+    "            max_value = 0\n",
+    "        else:\n",
+    "            raise EmptyArray(\"Empty array can't be used to map values\")\n",
+    "    else:\n",
+    "        max_value = int(array.max())\n",
+    "    index_array = np.zeros(max_value + 1) - 1\n",
+    "    index_array[array] = np.arange(len(array))\n",
     "\n",
-    "    index_array = np.zeros_like(unique) - 1\n",
-    "    index_array[unique] = values\n",
+    "    result = np.zeros_like(array_from) - 1\n",
+    "    mask = array_from <= max_value\n",
+    "    result[mask] = index_array[array_from[mask]]"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 42,
+   "id": "b24eddaa-76e1-4bcc-8db9-4e648f564f13",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def index_with_sparse_matrix(array_from):\n",
+    "    rows = np.unique(array_from)\n",
+    "    cols = np.zeros_like(rows)\n",
+    "    values = np.arange(1, len(rows) + 1)\n",
+    "    matrix = sparse.coo_matrix((values, (rows, cols)), (rows[-1] + 1, 1)).tocsr()\n",
     "\n",
-    "    return index_array[array_from]"
+    "    return (matrix[array_from, 0]).tocoo().data"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 43,
+   "id": "b6a818cc-00fb-457f-b91d-14044c028efd",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "array([1, 3, 1, 2, 0])"
+      ]
+     },
+     "execution_count": 43,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "array_from = np.array([4, 19000, 4, 8, 1])\n",
+    "index_with_sparse_matrix(array_from)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 11,
+   "id": "d5486625-8bab-4500-8167-66f5a8f851f3",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "<3488834974658x1 sparse array of type '<class 'numpy.int64'>'\n",
+       "\twith 4 stored elements in COOrdinate format>"
+      ]
+     },
+     "execution_count": 11,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "inpt = np.array([1288834974657, 2288834974657, 3488834974657, 3288834974657])\n",
+    "array = np.unique(inpt)\n",
+    "max_value = array[-1]\n",
+    "max_index = len(array) - 1\n",
+    "matrix = sparse.coo_array(\n",
+    "    (np.arange(1, len(array) + 1), (array, np.zeros_like(array))),\n",
+    "    (max_value + 1, 1),\n",
+    ") #.todok()\n",
+    "matrix"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 12,
+   "id": "562a1c34-32b8-45fc-bf31-519d9b0a83f0",
+   "metadata": {},
+   "outputs": [
+    {
+     "ename": "TypeError",
+     "evalue": "'coo_array' object is not subscriptable",
+     "output_type": "error",
+     "traceback": [
+      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
+      "\u001b[0;31mTypeError\u001b[0m                                 Traceback (most recent call last)",
+      "Cell \u001b[0;32mIn[12], line 4\u001b[0m\n\u001b[1;32m      2\u001b[0m result \u001b[38;5;241m=\u001b[39m np\u001b[38;5;241m.\u001b[39mzeros_like(inpt) \u001b[38;5;241m-\u001b[39m \u001b[38;5;241m1\u001b[39m\n\u001b[1;32m      3\u001b[0m mask \u001b[38;5;241m=\u001b[39m inpt \u001b[38;5;241m<\u001b[39m\u001b[38;5;241m=\u001b[39m max_value\n\u001b[0;32m----> 4\u001b[0m (\u001b[43mmatrix\u001b[49m\u001b[43m[\u001b[49m\u001b[43minpt\u001b[49m\u001b[43m[\u001b[49m\u001b[43mmask\u001b[49m\u001b[43m]\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mnp\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mzeros_like\u001b[49m\u001b[43m(\u001b[49m\u001b[43minpt\u001b[49m\u001b[43m[\u001b[49m\u001b[43mmask\u001b[49m\u001b[43m]\u001b[49m\u001b[43m)\u001b[49m\u001b[43m]\u001b[49m)\n",
+      "\u001b[0;31mTypeError\u001b[0m: 'coo_array' object is not subscriptable"
+     ]
+    }
+   ],
+   "source": [
+    "inpt = np.array([1288834974657, 2288834974657, 348883974657])\n",
+    "result = np.zeros_like(inpt) - 1\n",
+    "mask = inpt <= max_value\n",
+    "(matrix[inpt[mask], np.zeros_like(inpt[mask])])"
    ]
   },
   {
@@ -113,7 +214,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 5,
+   "execution_count": 2,
    "id": "little-marshall",
    "metadata": {},
    "outputs": [],
@@ -163,36 +264,43 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 8,
-   "id": "parliamentary-period",
+   "execution_count": 15,
+   "id": "84f97b1b-eb9a-498b-bf24-979be3212fd4",
    "metadata": {},
-   "outputs": [],
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "12.5 ms ± 126 µs per loop (mean ± std. dev. of 7 runs, 100 loops each)\n"
+     ]
+    }
+   ],
    "source": [
-    "_, mapping = index_with_searchsorted(arr)\n",
-    "arr2 = input_array()"
+    "%timeit index_with_indexarray(arr)"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 9,
-   "id": "laughing-selection",
+   "execution_count": 44,
+   "id": "7491e454-ce31-4132-90c9-0d37f79e92c8",
    "metadata": {},
    "outputs": [
     {
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "5.29 ms ± 498 µs per loop (mean ± std. dev. of 7 runs, 100 loops each)\n"
+      "121 µs ± 352 ns per loop (mean ± std. dev. of 7 runs, 10,000 loops each)\n"
      ]
     }
    ],
    "source": [
-    "%timeit index_with_arrays(arr2, np.zeros_like(arr2), mapping)"
+    "%timeit index_with_sparse_matrix(array_from)"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 10,
+   "execution_count": 9,
    "id": "systematic-shopping",
    "metadata": {},
    "outputs": [],
@@ -290,7 +398,7 @@
  ],
  "metadata": {
   "kernelspec": {
-   "display_name": "Python 3",
+   "display_name": "Python 3 (ipykernel)",
    "language": "python",
    "name": "python3"
   },
@@ -304,7 +412,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.7.9"
+   "version": "3.10.14"
   }
  },
  "nbformat": 4,

{matrix_utils-0.4.3 → matrix_utils-0.6}/matrix_utils/__init__.py

@@ -11,7 +11,7 @@ __all__ = (
     "SparseMatrixDict",
 )
 
-__version__ = "0.4.3"
+__version__ = "0.6"
 
 from .array_mapper import ArrayMapper
 from .indexers import CombinatorialIndexer, Proxy, RandomIndexer, SequentialIndexer

{matrix_utils-0.4.3 → matrix_utils-0.6}/matrix_utils/array_mapper.py

@@ -1,4 +1,5 @@
 import numpy as np
+from scipy import sparse
 
 from .errors import EmptyArray
 
@@ -23,28 +24,31 @@ class ArrayMapper:
 
     """  # NOQA: E501
 
-    def __init__(self, *, array: np.ndarray, sparse_cutoff: float = 0.1, empty_ok: bool = False):
+    def __init__(self, *, array: np.ndarray, sparse_cutoff: int = 50000, empty_ok: bool = False):
         self._check_input_array(array)
 
         # Even if already unique, this only adds ~2ms for 100.000 elements
         self.array = np.unique(array)
+        self.empty_ok = empty_ok
 
-        # TODO
-        # Sparse matrices could be used if the number of values present is much less
-        # than the number of possible values, given the (min, max) range.
-        # The default code will generate a complete mapping for the (min, max)
-        # interval, which can use too much memory in certain cases.
-        # self.use_sparse = len(self.keys) / self.keys.max() <= sparse_cutoff:
-
-        if array.shape == (0,):
-            if empty_ok:
+        if self.array.shape == (0,):
+            if self.empty_ok:
+                self.empty_input = True
                 self.max_value = 0
+                self.max_index = 0
+                return
             else:
                 raise EmptyArray("Empty array can't be used to map values")
         else:
-            self.max_value = int(self.array.max())
-        self.index_array = np.zeros(self.max_value + 1) - 1
-        self.index_array[self.array] = np.arange(len(self.array))
+            self.empty_input = False
+            self.max_value = self.array[-1]
+            self.max_index = len(self.array) - 1
+
+        # Zero serves as a missing value, so start at one
+        self.matrix = sparse.coo_matrix(
+            (np.arange(1, len(self.array) + 1), (self.array, np.zeros_like(self.array))),
+            (self.max_value + 1, 1),
+        ).tocsc()
 
     def __len__(self):
         return self.array.shape[0]
@@ -61,16 +65,25 @@ class ArrayMapper:
         if array.shape == (0,):
             # Empty array
             return array.copy()
+        elif self.empty_input:
+            if self.empty_ok:
+                # Return all with missing flag
+                return np.zeros_like(array) - 1
+            else:
+                raise EmptyArray("Can't map with empty input array")
 
         result = np.zeros_like(array) - 1
         mask = array <= self.max_value
-        result[mask] = self.index_array[array[mask]]
+        # https://numpy.org/doc/stable/reference/generated/numpy.matrix.A1.html
+        result[mask] = (self.matrix[array[mask], np.zeros_like(array[mask])]).A1 - 1
         return result
 
     def to_dict(self) -> dict:
         """Turn the mapping arrays into a Python dict. This is only useful for
         human examination, the normal implementation uses Numpy functions on the
         arrays directly."""
+        if self.empty_input:
+            return {}
         return {int(x): int(y) for x, y in zip(self.array, self.map_array(self.array))}
 
     def reverse_dict(self) -> dict:

{matrix_utils-0.4.3 → matrix_utils-0.6}/matrix_utils/indexers.py

@@ -1,10 +1,12 @@
 from itertools import product
 from typing import List, Union
 
+from numpy import int32, int64
 from numpy.random import PCG64, Generator
 
 # Max signed 32 bit integer, compatible with Windows
 MAX_SIGNED_32BIT_INT = 2147483647
+MAX_SIGNED_64BIT_INT = 9223372036854775807
 
 
 class Indexer:
@@ -18,15 +20,21 @@ class RandomIndexer(Generator, Indexer):
 
     Returns indices for a sample array.
 
+    Because we are creating Python indices, and Python starts indices with zero, the value
+    `max_value` is never reached - the maximum generated index is `max_value - 1`.
+
     max_value: Number of columns in the array for which a column index is returned.
     seed: Seed for RNG. Optional."""
 
-    def __init__(self, seed: Union[int, None] = None):
+    def __init__(self, seed: Union[int, None] = None, max_value: int = MAX_SIGNED_64BIT_INT):
         self.seed = seed
+        self.max_value = max_value
         self.reset()
 
     def __next__(self):
-        self.index = self.integers(0, MAX_SIGNED_32BIT_INT)
+        self.index = self.integers(
+            0, self.max_value, dtype=int32 if self.max_value == MAX_SIGNED_32BIT_INT else int64
+        )
         return self.index
 
     def reset(self):

{matrix_utils-0.4.3 → matrix_utils-0.6}/matrix_utils/mapped_matrix.py

@@ -137,12 +137,12 @@ class MappedMatrix:
         col_indices = safe_concatenate_indices([obj.col_matrix for obj in self.groups], empty_ok)
 
         if diagonal:
-            x = int(self.row_mapper.index_array.max() + 1)
+            x = 0 if self.row_mapper.empty_input else self.row_mapper.max_index + 1
             dimensions = (x, x)
         else:
             dimensions = (
-                int(self.row_mapper.index_array.max() + 1),
-                int(self.col_mapper.index_array.max() + 1),
+                0 if self.row_mapper.empty_input else self.row_mapper.max_index + 1,
+                0 if self.col_mapper.empty_input else self.col_mapper.max_index + 1,
             )
 
         self.matrix = sparse.coo_matrix(
@@ -178,6 +178,9 @@ class MappedMatrix:
             self.rebuild_matrix()
 
     def rebuild_matrix(self) -> None:
+        # In theory, setting this to zero instead of multiplying it would
+        # save some nanoseconds, but it led to weird Numpy dtype errors and
+        # wasn't worth thinking about too much for very little gain.
         self.matrix.data *= 0
         for group in self.groups:
             row, col, data = group.calculate()
@@ -211,7 +214,10 @@ class MappedMatrix:
                 for obj in resources:
                     obj.add_indexer(indexer=package.indexer)
             else:
-                package.indexer = RandomIndexer(seed=seed_override or package.metadata["seed"])
+                package.indexer = RandomIndexer(
+                    seed=seed_override or package.metadata["seed"],
+                    max_value=package.get_max_index_value(),
+                )
                 for obj in resources:
                     obj.add_indexer(indexer=package.indexer)
 
@@ -270,7 +276,7 @@ class MappedMatrix:
 
         array = np.zeros(len(given), dtype=UNCERTAINTY_DTYPE)
         for field in FIELDS:
-            array[field] = np.NaN
+            array[field] = np.nan
         array["loc"] = given
         array["uncertainty_type"] = uncertainty_type
         return array

{matrix_utils-0.4.3 → matrix_utils-0.6}/matrix_utils/mapped_matrix_dict.py

@@ -26,7 +26,7 @@ class SparseMatrixDict(dict):
             return SparseMatrixDict(
                 {unroll(a, b): c @ d.matrix for a, c in self.items() for b, d in other.items()}
             )
-        elif sparse.base.issparse(other):
+        elif sparse.issparse(other):
             return SparseMatrixDict({a: b @ other for a, b in self.items()})
         else:
             raise TypeError(f"Can't understand matrix multiplication for type {type(other)}")
@@ -180,7 +180,7 @@ class MappedMatrixDict(Mapping):
 
         Note that the mapped matrix dict must come first, i.e. `MappedMatrixDict @ other`.
         """
-        if sparse.base.issparse(other):
+        if sparse.issparse(other):
             return SparseMatrixDict(
                 [(key, value.matrix @ other) for key, value in self.matrices.items()]
             )

{matrix_utils-0.4.3 → matrix_utils-0.6}/matrix_utils/resource_group.py

@@ -120,7 +120,9 @@ class ResourceGroup:
         """The source data for the indices array."""
         indices = self.get_resource_by_suffix("indices")
         if self.transpose:
-            indices = indices.astype([("col", np.int32), ("row", np.int32)], copy=False)
+            indices = indices.astype(
+                [("col", indices.dtype[0]), ("row", indices.dtype[0])], copy=False
+            )
         return indices
 
     def get_resource_by_suffix(self, suffix: str) -> Any:

{matrix_utils-0.4.3 → matrix_utils-0.6}/matrix_utils.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: matrix_utils
-Version: 0.4.3
+Version: 0.6
 Summary: Tools to create matrices from data packages
 Author-email: Chris Mutel <cmutel@gmail.com>
 Maintainer-email: Chris Mutel <cmutel@gmail.com>
@@ -26,9 +26,9 @@ Requires-Python: >=3.9
 Description-Content-Type: text/markdown
 License-File: LICENSE
 Requires-Dist: numpy
-Requires-Dist: scipy
+Requires-Dist: scipy>=1.10
 Requires-Dist: pandas
-Requires-Dist: bw_processing
+Requires-Dist: bw_processing>=1.0
 Requires-Dist: stats_arrays
 Provides-Extra: testing
 Requires-Dist: matrix_utils; extra == "testing"

{matrix_utils-0.4.3 → matrix_utils-0.6}/matrix_utils.egg-info/requires.txt

@@ -1,7 +1,7 @@
 numpy
-scipy
+scipy>=1.10
 pandas
-bw_processing
+bw_processing>=1.0
 stats_arrays
 
 [dev]

{matrix_utils-0.4.3 → matrix_utils-0.6}/pyproject.toml

@@ -37,9 +37,9 @@ dependencies = [
     # dependencies as strings with quotes, e.g. "foo"
     # You can add version requirements like "foo>2.0"
     "numpy",
-    "scipy",
+    "scipy>=1.10",
     "pandas",
-    "bw_processing",
+    "bw_processing>=1.0",
     "stats_arrays",
 ]
 

{matrix_utils-0.4.3 → matrix_utils-0.6}/tests/array_mapper.py

@@ -12,9 +12,17 @@ def test_initial_setup():
     assert np.allclose(am.map_array(inpt), expected)
 
 
+def test_with_large_values():
+    inpt = np.array([1288834974657, 2288834974657, 3488834974657, 3288834974657])
+    am = ArrayMapper(array=inpt)
+    given = np.array([1288834974657, 228883474657, 3288834974657, 3488834974657])
+    expected = np.array([0, -1, 2, 3])
+    assert np.allclose(am.map_array(given), expected)
+
+
 def test_float_indices_raises_error():
     inpt = np.array([1, 2, 3, 6.0])
-    with pytest.raises(IndexError):
+    with pytest.raises(TypeError):
         ArrayMapper(array=inpt)
 
 
@@ -69,4 +77,4 @@ def test_empty_array_error():
 def test_empty_array_ok():
     am = ArrayMapper(array=np.array([], dtype=int), empty_ok=True)
     assert am.array.shape == (0,)
-    assert am.index_array.shape == (1,)
+    assert am.empty_input

{matrix_utils-0.4.3 → matrix_utils-0.6}/tests/fixtures/basic.py

@@ -2,63 +2,75 @@ import bw_processing as bwp
 import numpy as np
 
 
-def basic_mm(**kwargs):
+def switch_dtype(dp: bwp.Datapackage, indices_32bit: bool) -> tuple:
+    if indices_32bit:
+        del dp.metadata["64_bit_indices"]
+        return [("row", np.int32), ("col", np.int32)]
+    else:
+        return bwp.INDICES_DTYPE
+
+
+def basic_mm(indices_32bit: bool = False, **kwargs):
     dp = bwp.create_datapackage(**kwargs)
+    dtype = switch_dtype(dp, indices_32bit)
     dp.add_persistent_vector(
         matrix="foo",
         name="vector",
-        indices_array=np.array([(0, 0), (2, 1), (4, 2), (8, 3)], dtype=bwp.INDICES_DTYPE),
+        indices_array=np.array([(0, 0), (2, 1), (4, 2), (8, 3)], dtype=dtype),
         data_array=np.array([1, 2.3, 4, 25]),
     )
     dp.add_persistent_vector(
         matrix="foo",
         name="vector2",
-        indices_array=np.array([(10, 10), (12, 9), (14, 8), (18, 7)], dtype=bwp.INDICES_DTYPE),
+        indices_array=np.array([(10, 10), (12, 9), (14, 8), (18, 7)], dtype=dtype),
         data_array=np.array([11, 12.3, 14, 125]),
     )
     dp.add_persistent_array(
         matrix="foo",
         name="array",
-        indices_array=np.array([(1, 0), (2, 1), (5, 1), (8, 1)], dtype=bwp.INDICES_DTYPE),
+        indices_array=np.array([(1, 0), (2, 1), (5, 1), (8, 1)], dtype=dtype),
         data_array=np.array([[1, 2.3, 4, 25]]).T,
     )
     return dp
 
 
-def overlapping(**kwargs):
+def overlapping(indices_32bit: bool = False, **kwargs):
     dp = bwp.create_datapackage(**kwargs)
+    dtype = switch_dtype(dp, indices_32bit)
     dp.add_persistent_vector(
         matrix="foo",
         name="vector",
-        indices_array=np.array([(0, 0), (2, 1), (4, 2), (8, 3)], dtype=bwp.INDICES_DTYPE),
+        indices_array=np.array([(0, 0), (2, 1), (4, 2), (8, 3)], dtype=dtype),
         data_array=np.array([1, 2.3, 4, 25]),
     )
     dp.add_persistent_vector(
         matrix="foo",
         name="vector2",
-        indices_array=np.array([(0, 0), (12, 9), (2, 1), (18, 7)], dtype=bwp.INDICES_DTYPE),
+        indices_array=np.array([(0, 0), (12, 9), (2, 1), (18, 7)], dtype=dtype),
         data_array=np.array([11, 12.3, 14, 125]),
     )
     return dp
 
 
-def aggregation(**kwargs):
+def aggregation(indices_32bit: bool = False, **kwargs):
     dp = bwp.create_datapackage(**kwargs)
+    dtype = switch_dtype(dp, indices_32bit)
     dp.add_persistent_vector(
         matrix="foo",
         name="vector",
-        indices_array=np.array([(0, 0), (2, 1), (4, 2), (4, 2), (8, 3)], dtype=bwp.INDICES_DTYPE),
+        indices_array=np.array([(0, 0), (2, 1), (4, 2), (4, 2), (8, 3)], dtype=dtype),
         data_array=np.array([1, 2.3, 4, 17, 25]),
     )
     return dp
 
 
-def diagonal(**kwargs):
+def diagonal(indices_32bit: bool = False, **kwargs):
     dp = bwp.create_datapackage(**kwargs)
+    dtype = switch_dtype(dp, indices_32bit)
     dp.add_persistent_vector(
         matrix="foo",
         name="vector",
-        indices_array=np.array([(0, 1), (1, 1), (2, 0), (3, 1)], dtype=bwp.INDICES_DTYPE),
+        indices_array=np.array([(0, 1), (1, 1), (2, 0), (3, 1)], dtype=dtype),
         data_array=np.array([1, 2.3, 4, 25]),
         flip_array=np.array([0, 1, 0, 0], dtype=bool),
     )

{matrix_utils-0.4.3 → matrix_utils-0.6}/tests/fixtures/create_static_fixtures.py

@@ -157,10 +157,10 @@ def create_sensitivity_fixtures():
     indices_array = np.array([(12, 10), (12, 11), (12, 11), (12, 12)], dtype=INDICES_DTYPE)
     distributions_array = np.zeros((4,), dtype=UNCERTAINTY_DTYPE)
     distributions_array["uncertainty_type"] = (4, 4, 0, 4)
-    distributions_array["scale"] = np.NaN
-    distributions_array["shape"] = np.NaN
-    distributions_array["minimum"] = (0.5, 1.5, np.NaN, 2.5)
-    distributions_array["maximum"] = (1.5, 2.5, np.NaN, 3.5)
+    distributions_array["scale"] = np.nan
+    distributions_array["shape"] = np.nan
+    distributions_array["minimum"] = (0.5, 1.5, np.nan, 2.5)
+    distributions_array["maximum"] = (1.5, 2.5, np.nan, 3.5)
     distributions_array["loc"] = (1, 2, 1, 3)
     distributions_array["negative"] = (False, False, True, True)
     dp.add_persistent_vector(

{matrix_utils-0.4.3 → matrix_utils-0.6}/tests/indexers.py

@@ -1,6 +1,7 @@
 import numpy as np
 
 from matrix_utils import CombinatorialIndexer, Proxy, RandomIndexer, SequentialIndexer
+from matrix_utils.indexers import MAX_SIGNED_32BIT_INT, MAX_SIGNED_64BIT_INT
 
 
 def test_random():
@@ -12,6 +13,17 @@ def test_random():
     assert not np.allclose([next(a) for _ in range(10)], [next(b) for _ in range(10)])
 
 
+def test_indexing_max_value():
+    a = RandomIndexer(42, max_value=10)
+    values = [next(a) for _ in range(10000)]
+    assert max(values) == 10 - 1
+
+    a = RandomIndexer(42)
+    values = [next(a) for _ in range(10000)]
+    assert max(values) < MAX_SIGNED_64BIT_INT
+    assert max(values) > MAX_SIGNED_32BIT_INT
+
+
 def test_random_reset():
     a = RandomIndexer(42)
     given = [next(a) for _ in range(10)]

{matrix_utils-0.4.3 → matrix_utils-0.6}/tests/integration.py

@@ -1,13 +1,26 @@
 import bw_processing as bwp
 import numpy as np
+import pytest
 from fixtures import aggregation, basic_mm, diagonal, overlapping
 
 from matrix_utils import MappedMatrix
 
 
-def test_basic_matrix_construction():
+@pytest.mark.parametrize("smaller", [True, False])
+def test_basic_resource_group_indices_dtype(smaller):
+    dp = basic_mm(indices_32bit=smaller)
+
+    dtype = np.int32 if smaller else np.int64
+
+    for name in ("vector", "vector2", "array"):
+        assert dp.get_resource(f"{name}.indices")[0]["row"].dtype == dtype
+        assert dp.get_resource(f"{name}.indices")[0]["col"].dtype == dtype
+
+
+@pytest.mark.parametrize("smaller", [True, False])
+def test_basic_matrix_construction(smaller):
     mm = MappedMatrix(
-        packages=[basic_mm()],
+        packages=[basic_mm(indices_32bit=smaller)],
         matrix="foo",
         use_arrays=False,
         use_distributions=False,
@@ -21,9 +34,10 @@ def test_basic_matrix_construction():
     assert np.allclose(matrix.data, data)
 
 
-def test_matrix_construction_transpose():
+@pytest.mark.parametrize("smaller", [True, False])
+def test_matrix_construction_transpose(smaller):
     mm = MappedMatrix(
-        packages=[diagonal()],
+        packages=[diagonal(indices_32bit=smaller)],
         matrix="foo",
         use_arrays=False,
         use_distributions=False,
@@ -39,7 +53,7 @@ def test_matrix_construction_transpose():
     assert np.allclose(matrix.data, data)
 
     mm = MappedMatrix(
-        packages=[diagonal()],
+        packages=[diagonal(indices_32bit=smaller)],
         matrix="foo",
         use_arrays=False,
         use_distributions=False,
@@ -47,9 +61,14 @@ def test_matrix_construction_transpose():
     assert mm.matrix.shape == (4, 2)
 
 
-def test_matrix_construction_overlapping_substitution():
+@pytest.mark.parametrize("smaller", [True, False])
+def test_matrix_construction_overlapping_substitution(smaller):
     mm = MappedMatrix(
-        packages=[overlapping(sum_intra_duplicates=True, sum_inter_duplicates=False)],
+        packages=[
+            overlapping(
+                indices_32bit=smaller, sum_intra_duplicates=True, sum_inter_duplicates=False
+            )
+        ],
         matrix="foo",
         use_arrays=False,
         use_distributions=False,
@@ -64,9 +83,14 @@ def test_matrix_construction_overlapping_substitution():
     assert np.allclose(matrix.data, data)
 
 
-def test_matrix_construction_overlapping_sum():
+@pytest.mark.parametrize("smaller", [True, False])
+def test_matrix_construction_overlapping_sum(smaller):
     mm = MappedMatrix(
-        packages=[overlapping(sum_intra_duplicates=False, sum_inter_duplicates=True)],
+        packages=[
+            overlapping(
+                indices_32bit=smaller, sum_intra_duplicates=False, sum_inter_duplicates=True
+            )
+        ],
         matrix="foo",
         use_arrays=False,
         use_distributions=False,
@@ -76,16 +100,15 @@ def test_matrix_construction_overlapping_sum():
     col = np.array([0, 1, 2, 3, 5, 4])
     data = np.array([12, 16.3, 4, 25, 12.3, 125])
     matrix = mm.matrix.tocoo()
-    print(matrix)
-    print(matrix.data)
     assert np.allclose(matrix.row, row)
     assert np.allclose(matrix.col, col)
     assert np.allclose(matrix.data, data)
 
 
-def test_matrix_construction_internal_aggregation():
+@pytest.mark.parametrize("smaller", [True, False])
+def test_matrix_construction_internal_aggregation(smaller):
     mm = MappedMatrix(
-        packages=[aggregation()],
+        packages=[aggregation(indices_32bit=smaller)],
         matrix="foo",
         use_arrays=False,
         use_distributions=False,
@@ -99,9 +122,10 @@ def test_matrix_construction_internal_aggregation():
     assert np.allclose(matrix.data, data)
 
 
-def test_matrix_construction_no_internal_aggregation():
+@pytest.mark.parametrize("smaller", [True, False])
+def test_matrix_construction_no_internal_aggregation(smaller):
     mm = MappedMatrix(
-        packages=[aggregation(sum_intra_duplicates=False)],
+        packages=[aggregation(indices_32bit=smaller, sum_intra_duplicates=False)],
         matrix="foo",
         use_arrays=False,
         use_distributions=False,

{matrix_utils-0.4.3 → matrix_utils-0.6}/tests/mapped_matrix.py

@@ -614,18 +614,18 @@ def test_input_uncertainties(sensitivity_dps):
 
     expected = np.array(
         [
-            (99, magic % 100, np.NaN, np.NaN, np.NaN, np.NaN, False),
-            (99, magic % 100, np.NaN, np.NaN, np.NaN, np.NaN, False),
-            (99, magic % 100, np.NaN, np.NaN, np.NaN, np.NaN, False),
-            (98, -8.5, 1.118034, np.NaN, np.NaN, np.NaN, False),
-            (98, 9.5, 1.118034, np.NaN, np.NaN, np.NaN, False),
-            (4, 1, np.NaN, np.NaN, 0.5, 1.5, False),
-            (4, 2, np.NaN, np.NaN, 1.5, 2.5, False),
-            (0, 1, np.NaN, np.NaN, np.NaN, np.NaN, True),
-            (4, 3, np.NaN, np.NaN, 2.5, 3.5, True),
-            (99, 1, np.NaN, np.NaN, np.NaN, np.NaN, False),
-            (99, 2, np.NaN, np.NaN, np.NaN, np.NaN, False),
-            (99, 3, np.NaN, np.NaN, np.NaN, np.NaN, False),
+            (99, magic % 100, np.nan, np.nan, np.nan, np.nan, False),
+            (99, magic % 100, np.nan, np.nan, np.nan, np.nan, False),
+            (99, magic % 100, np.nan, np.nan, np.nan, np.nan, False),
+            (98, -8.5, 1.118034, np.nan, np.nan, np.nan, False),
+            (98, 9.5, 1.118034, np.nan, np.nan, np.nan, False),
+            (4, 1, np.nan, np.nan, 0.5, 1.5, False),
+            (4, 2, np.nan, np.nan, 1.5, 2.5, False),
+            (0, 1, np.nan, np.nan, np.nan, np.nan, True),
+            (4, 3, np.nan, np.nan, 2.5, 3.5, True),
+            (99, 1, np.nan, np.nan, np.nan, np.nan, False),
+            (99, 2, np.nan, np.nan, np.nan, np.nan, False),
+            (99, 3, np.nan, np.nan, np.nan, np.nan, False),
         ],
         dtype=bwp.UNCERTAINTY_DTYPE,
     )
@@ -647,18 +647,18 @@ def test_input_uncertainties_limit_samples(sensitivity_dps):
 
     expected = np.array(
         [
-            (99, magic % 100, np.NaN, np.NaN, np.NaN, np.NaN, False),
-            (99, magic % 100, np.NaN, np.NaN, np.NaN, np.NaN, False),
-            (99, magic % 100, np.NaN, np.NaN, np.NaN, np.NaN, False),
-            (98, -8, 0.8164966, np.NaN, np.NaN, np.NaN, False),
-            (98, 9, 0.8164966, np.NaN, np.NaN, np.NaN, False),
-            (4, 1, np.NaN, np.NaN, 0.5, 1.5, False),
-            (4, 2, np.NaN, np.NaN, 1.5, 2.5, False),
-            (0, 1, np.NaN, np.NaN, np.NaN, np.NaN, True),
-            (4, 3, np.NaN, np.NaN, 2.5, 3.5, True),
-            (99, 1, np.NaN, np.NaN, np.NaN, np.NaN, False),
-            (99, 2, np.NaN, np.NaN, np.NaN, np.NaN, False),
-            (99, 3, np.NaN, np.NaN, np.NaN, np.NaN, False),
+            (99, magic % 100, np.nan, np.nan, np.nan, np.nan, False),
+            (99, magic % 100, np.nan, np.nan, np.nan, np.nan, False),
+            (99, magic % 100, np.nan, np.nan, np.nan, np.nan, False),
+            (98, -8, 0.8164966, np.nan, np.nan, np.nan, False),
+            (98, 9, 0.8164966, np.nan, np.nan, np.nan, False),
+            (4, 1, np.nan, np.nan, 0.5, 1.5, False),
+            (4, 2, np.nan, np.nan, 1.5, 2.5, False),
+            (0, 1, np.nan, np.nan, np.nan, np.nan, True),
+            (4, 3, np.nan, np.nan, 2.5, 3.5, True),
+            (99, 1, np.nan, np.nan, np.nan, np.nan, False),
+            (99, 2, np.nan, np.nan, np.nan, np.nan, False),
+            (99, 3, np.nan, np.nan, np.nan, np.nan, False),
         ],
         dtype=bwp.UNCERTAINTY_DTYPE,
     )
@@ -680,18 +680,18 @@ def test_input_uncertainties_no_distributions(sensitivity_dps):
 
     expected = np.array(
         [
-            (99, magic % 100, np.NaN, np.NaN, np.NaN, np.NaN, False),
-            (99, magic % 100, np.NaN, np.NaN, np.NaN, np.NaN, False),
-            (99, magic % 100, np.NaN, np.NaN, np.NaN, np.NaN, False),
-            (98, -8.5, 1.118034, np.NaN, np.NaN, np.NaN, False),
-            (98, 9.5, 1.118034, np.NaN, np.NaN, np.NaN, False),
-            (0, 1, np.NaN, np.NaN, np.NaN, np.NaN, False),
-            (0, 2, np.NaN, np.NaN, np.NaN, np.NaN, False),
-            (0, -1, np.NaN, np.NaN, np.NaN, np.NaN, False),
-            (0, -3, np.NaN, np.NaN, np.NaN, np.NaN, False),
-            (99, 1, np.NaN, np.NaN, np.NaN, np.NaN, False),
-            (99, 2, np.NaN, np.NaN, np.NaN, np.NaN, False),
-            (99, 3, np.NaN, np.NaN, np.NaN, np.NaN, False),
+            (99, magic % 100, np.nan, np.nan, np.nan, np.nan, False),
+            (99, magic % 100, np.nan, np.nan, np.nan, np.nan, False),
+            (99, magic % 100, np.nan, np.nan, np.nan, np.nan, False),
+            (98, -8.5, 1.118034, np.nan, np.nan, np.nan, False),
+            (98, 9.5, 1.118034, np.nan, np.nan, np.nan, False),
+            (0, 1, np.nan, np.nan, np.nan, np.nan, False),
+            (0, 2, np.nan, np.nan, np.nan, np.nan, False),
+            (0, -1, np.nan, np.nan, np.nan, np.nan, False),
+            (0, -3, np.nan, np.nan, np.nan, np.nan, False),
+            (99, 1, np.nan, np.nan, np.nan, np.nan, False),
+            (99, 2, np.nan, np.nan, np.nan, np.nan, False),
+            (99, 3, np.nan, np.nan, np.nan, np.nan, False),
         ],
         dtype=bwp.UNCERTAINTY_DTYPE,
     )
@@ -736,7 +736,6 @@ def test_input_index_vector(sensitivity_dps):
         (942484272, 942484272, 1, 1),
     ]
     for row in expected:
-        print(mm.input_indexer_vector())
         assert np.allclose(mm.input_indexer_vector(), row)
         try:
             next(mm)

{matrix_utils-0.4.3 → matrix_utils-0.6}/tests/monte_carlo.py

@@ -170,8 +170,6 @@ def test_distributions_reproducible():
         results[:, :, i] = mm.matrix.toarray()
 
     given = results.sum(axis=0).sum(axis=0).ravel()
-    print(given.shape)
-    print(given)
     expected = np.array(
         [
             21.06909828,

{matrix_utils-0.4.3 → matrix_utils-0.6}/tests/ordering.py

@@ -21,7 +21,6 @@ def test_ordering():
     ]
     for dp in dps:
         dp.rehydrate_interface("w-fourth", Interface())
-        print(list(dp.groups))
 
     mm = MappedMatrix(packages=dps, matrix="matrix-a")
     assert [grp.label for grp in mm.groups] == [

{matrix_utils-0.4.3 → matrix_utils-0.6}/tests/utils.py

@@ -44,7 +44,7 @@ def create_dp(vector, array, distributions):
             indices_array=np.array([(0, 0)], dtype=bwp.INDICES_DTYPE),
             distributions_array=np.array(
                 [
-                    (4, 0.5, np.NaN, np.NaN, 0.2, 0.8, False),
+                    (4, 0.5, np.nan, np.nan, 0.2, 0.8, False),
                 ],
                 dtype=bwp.UNCERTAINTY_DTYPE,
             ),