fastremap 1.15.2__tar.gz → 1.16.0__tar.gz

{fastremap-1.15.2 → fastremap-1.16.0}/ChangeLog

@@ -1,6 +1,21 @@
 CHANGES
 =======
 
+1.16.0
+------
+
+* test: add a more stringent random test
+* fix(unique): unique\_via\_sort returns proper data width
+* fix(unique): axis=0, ndim=2 works for higher data widths
+* fix(unique): edge case for uint64 range
+* fix(unique): handle floating point
+* test: make sure fastremap.unique matches np.unique
+* test: sanity check that axis=0 works
+* perf: accelerate special case of np.unique(x, axis=0) where x.ndim=2
+* feat: add indices
+* feat+test: add widen\_dtype and narrow\_dtype
+* docs: show how to use "indices"
+
 1.15.2
 ------
 

{fastremap-1.15.2/fastremap.egg-info → fastremap-1.16.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: fastremap
-Version: 1.15.2
+Version: 1.16.0
 Summary: Remap, mask, renumber, unique, and in-place transposition of 3D labeled images. Point cloud too.
 Home-page: https://github.com/seung-lab/fastremap/
 Author: William Silversmith
@@ -42,12 +42,18 @@ Renumber and relabel Numpy arrays at C++ speed and physically convert rectangula
 import fastremap
 
 uniq, cts = fastremap.unique(labels, return_counts=True) # may be much faster than np.unique
+
+idxs = fastremap.indices(labels, 1231) # important for huge arrays
+
 labels, remapping = fastremap.renumber(labels, in_place=True) # relabel values from 1 and refit data type
 ptc = fastremap.point_cloud(labels) # dict of coordinates by label
 
 labels = fastremap.refit(labels) # resize the data type of the array to fit extrema
 labels = fastremap.refit(labels, value=-35) # resize the data type to fit the value provided
 
+wider_dtype = fastremap.widen_dtype(np.uint32) # np.uint64
+narrower_dtype = fastremap.narrow_dtype(np.uint32) # np.uint16
+
 # remap all occurances of 1 -> 2
 labels = fastremap.remap(labels, { 1: 2 }, preserve_missing_labels=True, in_place=True)
 
@@ -78,8 +84,11 @@ binaries = fastremap.tobytes(labels, (64,64,64), order="F")
 ## All Available Functions 
 - **unique:** Faster implementation of `np.unique`.
 - **renumber:** Relabel array from 1 to N which can often use smaller datatypes.
+- **indices:** Optimized search for matching values.
 - **remap:** Custom relabeling of values in an array from a dictionary.
 - **refit:** Resize the data type of an array to the smallest that can contain the most extreme values in it.
+- **narrow_dtype:** Find the next sized up dtype. e.g. uint16 -> uint32
+- **widen_dtype:** Find the next sized down dtype. e.g. uint16 -> uint8
 - **mask:** Zero out labels in an array specified by a given list.
 - **mask_except**: Zero out all labels except those specified in a given list.
 - **component_map**: Extract an int-to-int dictionary mapping of labels from one image containing component labels to another parent labels.  

{fastremap-1.15.2 → fastremap-1.16.0}/README.md

@@ -8,12 +8,18 @@ Renumber and relabel Numpy arrays at C++ speed and physically convert rectangula
 import fastremap
 
 uniq, cts = fastremap.unique(labels, return_counts=True) # may be much faster than np.unique
+
+idxs = fastremap.indices(labels, 1231) # important for huge arrays
+
 labels, remapping = fastremap.renumber(labels, in_place=True) # relabel values from 1 and refit data type
 ptc = fastremap.point_cloud(labels) # dict of coordinates by label
 
 labels = fastremap.refit(labels) # resize the data type of the array to fit extrema
 labels = fastremap.refit(labels, value=-35) # resize the data type to fit the value provided
 
+wider_dtype = fastremap.widen_dtype(np.uint32) # np.uint64
+narrower_dtype = fastremap.narrow_dtype(np.uint32) # np.uint16
+
 # remap all occurances of 1 -> 2
 labels = fastremap.remap(labels, { 1: 2 }, preserve_missing_labels=True, in_place=True)
 
@@ -44,8 +50,11 @@ binaries = fastremap.tobytes(labels, (64,64,64), order="F")
 ## All Available Functions 
 - **unique:** Faster implementation of `np.unique`.
 - **renumber:** Relabel array from 1 to N which can often use smaller datatypes.
+- **indices:** Optimized search for matching values.
 - **remap:** Custom relabeling of values in an array from a dictionary.
 - **refit:** Resize the data type of an array to the smallest that can contain the most extreme values in it.
+- **narrow_dtype:** Find the next sized up dtype. e.g. uint16 -> uint32
+- **widen_dtype:** Find the next sized down dtype. e.g. uint16 -> uint8
 - **mask:** Zero out labels in an array specified by a given list.
 - **mask_except**: Zero out all labels except those specified in a given list.
 - **component_map**: Extract an int-to-int dictionary mapping of labels from one image containing component labels to another parent labels.  

{fastremap-1.15.2 → fastremap-1.16.0}/automated_test.py

@@ -397,6 +397,29 @@ def test_minmax():
   assert minval == np.min(volume)
   assert maxval == np.max(volume)
 
+@pytest.mark.parametrize("dtype", DTYPES)
+def test_unique_axis_0(dtype):
+  arr = np.array([
+    [0,1],
+    [0,2],
+    [0,3],
+    [0,2],
+    [1,2],
+    [2,3],
+  ], dtype=dtype)
+
+  res = fastremap.unique(arr, axis=0)
+  ans = np.unique(arr, axis=0)
+
+  assert np.all(res == ans)
+
+def test_unique_axis_0_random():
+  arr = np.random.randint(0,100000, size=[1000000,2], dtype=np.uint32)
+  a1 = fastremap.unique(arr, axis=0)
+  a2 = np.unique(arr, axis=0)
+  assert np.all(a1 == a2)
+
+
 @pytest.mark.parametrize("order", [ "C", "F" ])
 def test_unique(order):
   def reorder(arr):
@@ -610,9 +633,55 @@ def test_tobytes_misaligned(order):
   except ValueError:
     pass
 
-
-
-
-
+def test_narrow_dtype():
+  assert fastremap.narrow_dtype(np.uint64) == np.uint32
+  assert fastremap.narrow_dtype(np.uint32) == np.uint16
+  assert fastremap.narrow_dtype(np.uint16) == np.uint8
+  assert fastremap.narrow_dtype(np.uint8) == np.uint8
+
+  assert fastremap.narrow_dtype(np.int64) == np.int32
+  assert fastremap.narrow_dtype(np.int32) == np.int16
+  assert fastremap.narrow_dtype(np.int16) == np.int8
+  assert fastremap.narrow_dtype(np.int8) == np.int8
+
+  assert fastremap.narrow_dtype(np.uint64, exotics=True) == np.uint32
+  assert fastremap.narrow_dtype(np.uint32, exotics=True) == np.uint16
+  assert fastremap.narrow_dtype(np.uint16, exotics=True) == np.uint8
+  assert fastremap.narrow_dtype(np.uint8, exotics=True) == np.uint8
+
+  assert fastremap.narrow_dtype(np.int64, exotics=True) == np.int32
+  assert fastremap.narrow_dtype(np.int32, exotics=True) == np.int16
+  assert fastremap.narrow_dtype(np.int16, exotics=True) == np.int8
+  assert fastremap.narrow_dtype(np.int8, exotics=True) == np.int8
+
+  assert fastremap.narrow_dtype(np.float64) == np.float32
+  assert fastremap.narrow_dtype(np.float32) == np.float32
+
+  assert fastremap.narrow_dtype(np.float64, exotics=True) == np.float32
+  assert fastremap.narrow_dtype(np.float32, exotics=True) == np.float16
+
+  assert fastremap.narrow_dtype(np.complex128) == np.complex64
+  assert fastremap.narrow_dtype(np.complex64) == np.complex64
+
+def test_widen_dtype():
+  assert fastremap.widen_dtype(np.uint64) == np.uint64
+  assert fastremap.widen_dtype(np.uint32) == np.uint64
+  assert fastremap.widen_dtype(np.uint16) == np.uint32
+  assert fastremap.widen_dtype(np.uint8) == np.uint16
+
+  assert fastremap.widen_dtype(np.int64) == np.int64
+  assert fastremap.widen_dtype(np.int32) == np.int64
+  assert fastremap.widen_dtype(np.int16) == np.int32
+  assert fastremap.widen_dtype(np.int8) == np.int16
+
+  assert fastremap.widen_dtype(np.float64, exotics=True) == np.longdouble
+  assert fastremap.widen_dtype(np.float64) == np.float64
+  assert fastremap.widen_dtype(np.float32) == np.float64
+  assert fastremap.widen_dtype(np.float16) == np.float32
+
+  assert fastremap.widen_dtype(np.complex64) == np.complex64
+  assert fastremap.widen_dtype(np.complex64, exotics=True) == np.complex128
+  assert fastremap.widen_dtype(np.complex128, exotics=True) == np.clongdouble
+  assert fastremap.widen_dtype(np.clongdouble, exotics=True) == np.clongdouble
 
 

{fastremap-1.15.2 → fastremap-1.16.0/fastremap.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: fastremap
-Version: 1.15.2
+Version: 1.16.0
 Summary: Remap, mask, renumber, unique, and in-place transposition of 3D labeled images. Point cloud too.
 Home-page: https://github.com/seung-lab/fastremap/
 Author: William Silversmith
@@ -42,12 +42,18 @@ Renumber and relabel Numpy arrays at C++ speed and physically convert rectangula
 import fastremap
 
 uniq, cts = fastremap.unique(labels, return_counts=True) # may be much faster than np.unique
+
+idxs = fastremap.indices(labels, 1231) # important for huge arrays
+
 labels, remapping = fastremap.renumber(labels, in_place=True) # relabel values from 1 and refit data type
 ptc = fastremap.point_cloud(labels) # dict of coordinates by label
 
 labels = fastremap.refit(labels) # resize the data type of the array to fit extrema
 labels = fastremap.refit(labels, value=-35) # resize the data type to fit the value provided
 
+wider_dtype = fastremap.widen_dtype(np.uint32) # np.uint64
+narrower_dtype = fastremap.narrow_dtype(np.uint32) # np.uint16
+
 # remap all occurances of 1 -> 2
 labels = fastremap.remap(labels, { 1: 2 }, preserve_missing_labels=True, in_place=True)
 
@@ -78,8 +84,11 @@ binaries = fastremap.tobytes(labels, (64,64,64), order="F")
 ## All Available Functions 
 - **unique:** Faster implementation of `np.unique`.
 - **renumber:** Relabel array from 1 to N which can often use smaller datatypes.
+- **indices:** Optimized search for matching values.
 - **remap:** Custom relabeling of values in an array from a dictionary.
 - **refit:** Resize the data type of an array to the smallest that can contain the most extreme values in it.
+- **narrow_dtype:** Find the next sized up dtype. e.g. uint16 -> uint32
+- **widen_dtype:** Find the next sized down dtype. e.g. uint16 -> uint8
 - **mask:** Zero out labels in an array specified by a given list.
 - **mask_except**: Zero out all labels except those specified in a given list.
 - **component_map**: Extract an int-to-int dictionary mapping of labels from one image containing component labels to another parent labels.  

fastremap-1.16.0/fastremap.egg-info/pbr.json

@@ -0,0 +1 @@
+{"git_version": "a7c2749", "is_release": true}

{fastremap-1.15.2 → fastremap-1.16.0}/src/fastremap.pyx

@@ -12,7 +12,7 @@ constrained environments when format shifting.
 
 Author: William Silversmith
 Affiliation: Seung Lab, Princeton Neuroscience Institute
-Date: August 2018 - May 2022
+Date: August 2018 - May 2025
 """
 from typing import Sequence, List
 cimport cython
@@ -101,6 +101,23 @@ def match_array_orders(*arrs, order="K"):
   else:
     return [ np.asfortranarray(arr) for arr in arrs ]
 
+@cython.boundscheck(False)
+@cython.wraparound(False)  # turn off negative index wrapping for entire function
+@cython.nonecheck(False)
+def indices(cnp.ndarray[NUMBER, cast=True, ndim=1] arr, NUMBER value):
+  """
+  Search through an array and identify the indices where value matches the array.
+  """
+  cdef vector[uint64_t] all_indices
+  cdef uint64_t i = 0
+  cdef uint64_t size = arr.size
+
+  for i in range(size):
+    if arr[i] == value:
+      all_indices.push_back(i)
+
+  return np.asarray(all_indices, dtype=np.uint64)
+
 def renumber(arr, start=1, preserve_zero=True, in_place=False):
   """
   renumber(arr, start=1, preserve_zero=True, in_place=False)
@@ -339,6 +356,83 @@ def fit_dtype(dtype, value, exotics=False):
     dtype, value
   ))
 
+def widen_dtype(dtype, exotics:bool = False):
+  """
+  Widen the given dtype to the next size
+  of the same type. For example, 
+  int8 -> int16 or uint32 -> uint64
+
+  64-bit types will map to themselves.
+
+  Return: upgraded dtype
+  """
+  dtype = np.dtype(dtype)
+
+  if np.issubdtype(dtype, np.floating):
+    sequence = [ np.float16, np.float32, np.float64 ] 
+    if exotics:
+      sequence += [ np.longdouble ]
+  elif np.issubdtype(dtype, np.unsignedinteger):
+    sequence = [ np.uint8, np.uint16, np.uint32, np.uint64 ]
+  elif np.issubdtype(dtype, np.complexfloating):
+    sequence = [ np.complex64 ]
+    if exotics:
+      sequence += [ np.complex128, np.clongdouble ]
+  elif np.issubdtype(dtype, np.integer):
+    sequence = [ np.int8, np.int16, np.int32, np.int64 ]
+  elif np.issubdtype(dtype, (np.intp, np.uintp)):
+    return dtype
+  elif exotics:
+    raise ValueError(
+      f"Unsupported dtype: {dtype}\n"
+    )    
+  else:
+    raise ValueError(
+      f"Unsupported dtype: {dtype}\n"
+      f"Only standard floats, integers, and complex types are supported."
+      f"For additional types (e.g. long double, complex128, clongdouble), enable exotics."
+    )
+
+  idx = sequence.index(dtype)
+  return sequence[min(idx+1, len(sequence) - 1)]
+
+def narrow_dtype(dtype, exotics:bool = False):
+  """
+  Widen the given dtype to the next size
+  of the same type. For example, 
+  int16 -> int8 or uint64 -> uint32
+
+  8-bit types will map to themselves.
+
+  exotics: include float16
+
+  Return: upgraded dtype
+  """
+  dtype = np.dtype(dtype)
+  if dtype.itemsize == 1:
+    return dtype
+
+  if np.issubdtype(dtype, np.floating):
+    sequence = [ np.float32, np.float64, np.longdouble ] 
+    if exotics:
+      sequence = [ np.float16 ] + sequence
+  elif np.issubdtype(dtype, np.unsignedinteger):
+    sequence = [ np.uint8, np.uint16, np.uint32, np.uint64 ]
+  elif np.issubdtype(dtype, np.complexfloating):
+    sequence = [ np.complex64, np.complex128, np.clongdouble ]
+  elif np.issubdtype(dtype, np.integer):
+    sequence = [ np.int8, np.int16, np.int32, np.int64 ]
+  elif np.issubdtype(dtype, (np.intp, np.uintp)):
+    return dtype
+  else:
+    raise ValueError(
+      f"Unsupported dtype: {dtype}\n"
+      f"Only standard floats, integers, and complex types are supported."
+    )
+
+  idx = sequence.index(dtype)
+  return sequence[max(idx-1, 0)]
+
 def mask(arr, labels, in_place=False, value=0):
   """
   mask(arr, labels, in_place=False, value=0)
@@ -777,19 +871,24 @@ def unique(labels, return_index=False, return_inverse=False, return_counts=False
   if not isinstance(labels, np.ndarray):
     labels = np.array(labels)
 
-  if not np.issubdtype(labels.dtype, np.integer):
-    raise TypeError("fastremap.unique only supports integer types.")
-
   # These flags are currently unsupported so call uncle and
   # use the standard implementation instead.
-  if axis is not None:
-    return np.unique(
-      labels, 
-      return_index=return_index, 
-      return_inverse=return_inverse, 
-      return_counts=return_counts, 
-      axis=axis
-    )
+  if (axis is not None) or (not np.issubdtype(labels.dtype, np.integer)):
+    if (
+      axis == 0
+      and (labels.ndim == 2 and np.dtype(labels.dtype).itemsize < 8)
+      and not (return_index or return_inverse or return_counts)
+      and labels.flags.c_contiguous
+    ):
+      return two_axis_unique(labels)
+    else:
+      return np.unique(
+        labels, 
+        return_index=return_index, 
+        return_inverse=return_inverse, 
+        return_counts=return_counts, 
+        axis=axis
+      )
 
   cdef size_t voxels = labels.size
 
@@ -799,12 +898,10 @@ def unique(labels, return_index=False, return_inverse=False, return_counts=False
   labels_orig = labels
   labels = reshape(labels, (voxels,))
 
-  cdef int64_t max_label
-  cdef int64_t min_label
+  max_label = 0
+  min_label = 0
   if voxels > 0:
     min_label, max_label = minmax(labels)
-  else:
-    min_label, max_label = (0, 0)
 
   def c_order_index(arr):
     if len(shape) > 1 and fortran_order:
@@ -819,9 +916,9 @@ def unique(labels, return_index=False, return_inverse=False, return_counts=False
     counts = np.array([], dtype=np.uint32)
     index = np.array([], dtype=np.uint64)
     inverse = np.array([], dtype=np.uintp)
-  elif min_label >= 0 and max_label < <int64_t>voxels:
+  elif min_label >= 0 and max_label < int(voxels):
     uniq, index, counts, inverse = unique_via_array(labels, max_label, return_index=return_index, return_inverse=return_inverse)
-  elif (max_label - min_label) <= <int64_t>voxels:
+  elif (max_label - min_label) <= int(voxels):
     uniq, index, counts, inverse = unique_via_shifted_array(labels, min_label, max_label, return_index=return_index, return_inverse=return_inverse)
   elif float(pixel_pairs(labels)) / float(voxels) > 0.66:
     uniq, index, counts, inverse = unique_via_renumber(labels, return_index=return_index, return_inverse=return_inverse)
@@ -845,6 +942,23 @@ def unique(labels, return_index=False, return_inverse=False, return_counts=False
     return tuple(results)
   return uniq
 
+def two_axis_unique(labels):
+  """
+  Faster replacement for np.unique(labels, axis=0)
+  when ndim = 2 and the dtype can be widened.
+
+  This special case is useful for sorting edge lists.
+  """
+  dtype = labels.dtype
+  wide_dtype = widen_dtype(dtype)
+
+  labels = labels[:, [1,0]].reshape(-1, order="C")
+  labels = labels.view(wide_dtype)
+  labels = unique(labels)
+  N = len(labels)
+  labels = labels.view(dtype).reshape((N, 2), order="C")
+  return labels[:,[1,0]]
+
 def unique_via_shifted_array(labels, min_label=None, max_label=None, return_index=False, return_inverse=False):
   if min_label is None or max_label is None:
     min_label, max_label = minmax(labels)
@@ -895,9 +1009,10 @@ def unique_via_sort(cnp.ndarray[ALLINT, ndim=1] labels):
   uniq.push_back(cur)
   counts.push_back(accum)
 
+  dtype = labels.dtype
   del labels
 
-  return np.array(uniq), np.array(counts)
+  return np.array(uniq, dtype=dtype), np.array(counts, dtype=np.uint64)
 
 @cython.boundscheck(False)
 @cython.wraparound(False)  # turn off negative index wrapping for entire function

fastremap-1.15.2/fastremap.egg-info/pbr.json

@@ -1 +0,0 @@
-{"git_version": "b766561", "is_release": true}