pelutils 3.8.4__tar.gz → 3.9.0__tar.gz

{pelutils-3.8.4 → pelutils-3.9.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: pelutils
-Version: 3.8.4
+Version: 3.9.0
 Summary: The Swiss army knife of Python projects.
 Home-page: https://github.com/peleiden/pelutils
 Download-URL: https://pypi.org/project/pelutils/
@@ -275,7 +275,7 @@ log2.configure("path/to/save/log2.log")
 
 A few different numpy types are defined for the convenience of not having to remember what data types your arrays are - and also to satisfy your nasty type checkers.
 ```py
-from pelutils.types import AnyArray, FloatArray, IntArray
+from pelutils.types import AnyArray, BoolArray, BytesArray, ComplexArray, FloatArray, IntArray, ObjectArray, StringArray
 
 
 def function_which_takes_np_types(

{pelutils-3.8.4 → pelutils-3.9.0}/README.md

@@ -250,7 +250,7 @@ log2.configure("path/to/save/log2.log")
 
 A few different numpy types are defined for the convenience of not having to remember what data types your arrays are - and also to satisfy your nasty type checkers.
 ```py
-from pelutils.types import AnyArray, FloatArray, IntArray
+from pelutils.types import AnyArray, BoolArray, BytesArray, ComplexArray, FloatArray, IntArray, ObjectArray, StringArray
 
 
 def function_which_takes_np_types(

{pelutils-3.8.4 → pelutils-3.9.0}/pelutils/__init__.py

@@ -24,7 +24,6 @@ import numpy as np
 import psutil
 
 try:
-    # torch is only to be installed if pelutils[ds] has been installed
     import torch
 
     _has_torch = True
@@ -315,6 +314,7 @@ from .ticktock import *  # noqa: F403
 TT: TickTock  # noqa: F405
 from .datastorage import *  # noqa: F403
 from .datastorage2 import *  # noqa: F403
+from .ds import unique
 from .format import *  # noqa: F403
 from .jsonl import *  # noqa: F403
 from .pretty_json import *  # noqa: F403

{pelutils-3.8.4 → pelutils-3.9.0}/pelutils/__version__.py

@@ -1,2 +1,2 @@
 # Do not put anything else in this file
-__version__ = "3.8.4"
+__version__ = "3.9.0"

pelutils-3.9.0/pelutils/_utils.py

@@ -0,0 +1,46 @@
+import sys
+from typing import Any
+
+
+def isinstance_by_name(obj: Any, module: str, qualname: str) -> bool:  # pyright: ignore[reportExplicitAny]
+    """Check whether ``obj`` is an instance of a class without importing its module.
+
+    This is useful for libraries with optional dependencies: if the dependency
+    has not been imported anywhere in the current process, ``obj`` cannot
+    possibly be an instance of one of its classes, so we can skip the check
+    without forcing an import.
+
+    Parameters
+    ----------
+    obj
+        The object whose type is being checked.
+    module
+        The fully qualified name of the module that defines the target class
+        (e.g. ``"pandas"`` or ``"numpy"``).
+    qualname
+        The attribute name of the class within ``module`` (e.g. ``"Series"``
+        or ``"ndarray"``). Dotted paths are not supported.
+
+    Returns
+    -------
+    bool
+        ``True`` if ``module`` is already loaded in :data:`sys.modules` and
+        ``obj`` is an instance of ``module.qualname``; ``False`` otherwise
+        (including when the module is not loaded or does not expose the
+        named attribute).
+
+    Examples
+    --------
+    >>> import pandas as pd
+    >>> _isinstance_by_name(pd.Series([1, 2, 3]), "pandas", "Series")
+    True
+    >>> _isinstance_by_name([1, 2, 3], "pandas", "Series")
+    False
+    """
+    mod = sys.modules.get(module)
+    if mod is None:
+        return False
+    cls = getattr(mod, qualname, None)
+    if not isinstance(cls, type):
+        return False
+    return isinstance(obj, cls)

{pelutils-3.8.4 → pelutils-3.9.0}/pelutils/datastorage2/_pretty_json.py

@@ -9,6 +9,13 @@ import json
 import pickle
 from typing import Any
 
+try:
+    import torch
+
+    _has_torch = True
+except ModuleNotFoundError:
+    _has_torch = False
+
 # Sentinel prefix so consumers can detect pickled blobs.
 _PICKLE_PREFIX = "__pickled_b64__"
 
@@ -45,6 +52,10 @@ def _get_qualified_type_name(obj: object) -> str:
 
 def _pickle_encode(value: object) -> str:
     """Pickle *value*, base64-encode the bytes, and return a prefixed string."""
+    if _has_torch and isinstance(value, torch.Tensor):  # pyright: ignore[reportPossiblyUnboundVariable]
+        # If a view of torch Tensor is given, the whole tensor is pickled, not just the view
+        # Therefore, it is cloned first to ensure that only the view itself is pickled
+        value = value.clone()
     raw = pickle.dumps(value, protocol=pickle.HIGHEST_PROTOCOL)
     encoded = base64.b64encode(raw).decode("ascii")
     return f"{_PICKLE_PREFIX}:{_get_qualified_type_name(value)}:{encoded}"

{pelutils-3.8.4 → pelutils-3.9.0}/pelutils/ds/__init__.py

@@ -5,7 +5,8 @@ from typing import TypeVar, Union, cast
 
 import numpy as np
 
-from pelutils.types import AnyArray, FloatArray, IntArray
+from pelutils._utils import isinstance_by_name
+from pelutils.types import AnyArray, BoolArray, BytesArray, FloatArray, IntArray, StringArray
 
 try:
     import torch
@@ -18,27 +19,38 @@ import _pelutils_c as _c
 
 import pelutils._c as _c_utils
 
-ArrayOrTensor = TypeVar("ArrayOrTensor", bound=Union[AnyArray, torch.Tensor])
+ArrayOrTensor = Union["BoolArray | BytesArray | FloatArray | IntArray | StringArray | torch.Tensor"]
 
 
-def unique(
-    array: ArrayOrTensor,
+def unique(  # noqa: PLR0912
+    array: ...,
     *,
     return_index: bool = False,
     return_inverse: bool = False,
     return_counts: bool = False,
     axis: int = 0,
 ) -> ArrayOrTensor | tuple[ArrayOrTensor, ...]:
-    """Similar to np.unique, but in linear time and returns unsorted."""
-    if not array.size:
-        raise ValueError("Array must be non-empty")
+    """Return unique elements in a given numpy array (or torch tensor or pandas series).
 
+    This function works very similar to np.unique, but it runs in linear time, making it significantly faster
+    for large arrays. The returned unique elements are unsorted, however.
+    """
     is_tensor = False
     if _has_torch and isinstance(array, torch.Tensor):  # pyright: ignore[reportPossiblyUnboundVariable]
         is_tensor = True
         np_array = array.numpy()
-    else:
+    elif isinstance_by_name(array, "pandas", "Series"):
+        np_array = array.values
+    elif isinstance(array, np.ndarray):
+        if np.issubdtype(array.dtype, np.object_):
+            raise TypeError(f"Unsupported array dtype {array.dtype}")
         np_array = array
+    else:
+        raise TypeError(f"Unsupported array type {type(array)}, must be numpy array, torch tensor, or pandas dataframe.")
+
+    if np_array.ndim == 0:
+        raise ValueError("unique does not work for shape-less arrays")
+
     np_array = cast(AnyArray, np_array)
     del array  # Prevent reuse - underlying tensor already referenced by np_array, which should be used from here
 
@@ -82,7 +94,7 @@ def unique(
     if _has_torch and is_tensor:
         ret = [torch.from_numpy(x) for x in ret]  # pyright: ignore[reportPossiblyUnboundVariable]
 
-    return tuple(ret) if len(ret) > 1 else ret[0]  # pyright: ignore[reportReturnType]
+    return tuple(ret) if len(ret) > 1 else ret[0]
 
 
 def tensor_bytes(x: AnyArray | torch.Tensor) -> int:

{pelutils-3.8.4 → pelutils-3.9.0}/pelutils/types.py

@@ -5,5 +5,10 @@ import numpy.typing as npt
 
 # Common numpy types
 AnyArray = npt.NDArray[Any]  # pyright: ignore[reportExplicitAny]
+BoolArray = npt.NDArray[np.bool_]
+BytesArray = npt.NDArray[np.bytes_]
+ComplexArray = npt.NDArray[np.complexfloating[Any]]  # pyright: ignore[reportExplicitAny, reportInvalidTypeArguments]
 FloatArray = npt.NDArray[np.floating[Any]]  # pyright: ignore[reportExplicitAny]
 IntArray = npt.NDArray[np.integer[Any]]  # pyright: ignore[reportExplicitAny]
+ObjectArray = npt.NDArray[np.object_]
+StringArray = npt.NDArray[np.str_]

{pelutils-3.8.4 → pelutils-3.9.0}/pelutils.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: pelutils
-Version: 3.8.4
+Version: 3.9.0
 Summary: The Swiss army knife of Python projects.
 Home-page: https://github.com/peleiden/pelutils
 Download-URL: https://pypi.org/project/pelutils/
@@ -275,7 +275,7 @@ log2.configure("path/to/save/log2.log")
 
 A few different numpy types are defined for the convenience of not having to remember what data types your arrays are - and also to satisfy your nasty type checkers.
 ```py
-from pelutils.types import AnyArray, FloatArray, IntArray
+from pelutils.types import AnyArray, BoolArray, BytesArray, ComplexArray, FloatArray, IntArray, ObjectArray, StringArray
 
 
 def function_which_takes_np_types(

{pelutils-3.8.4 → pelutils-3.9.0}/pelutils.egg-info/SOURCES.txt

@@ -3,6 +3,7 @@ README.md
 setup.py
 pelutils/__init__.py
 pelutils/__version__.py
+pelutils/_utils.py
 pelutils/datastorage.py
 pelutils/format.py
 pelutils/jsonl.py
@@ -42,4 +43,5 @@ tests/test_parser.py
 tests/test_pretty_json.py
 tests/test_tests.py
 tests/test_ticktock.py
-tests/test_types.py
+tests/test_types.py
+tests/test_utils.py

pelutils-3.9.0/tests/test_types.py

@@ -0,0 +1,37 @@
+import numpy as np
+
+from pelutils import AnyArray, BoolArray, ComplexArray, FloatArray, IntArray, StringArray
+from pelutils.types import BytesArray, ObjectArray
+
+
+def function_which_takes_np_types(  # noqa: PLR0913
+    any_array: AnyArray,
+    bool_array: BoolArray,
+    bytes_array: BytesArray,
+    complex_array: ComplexArray,
+    float_array: FloatArray,
+    int_array: IntArray,
+    object_array: ObjectArray,
+    string_array: StringArray,
+) -> ComplexArray:
+    assert np.issubdtype(object_array.dtype, np.object_)
+    assert not np.issubdtype(bytes_array.dtype, np.object_)
+    assert not np.issubdtype(string_array.dtype, np.object_)
+    assert not np.issubdtype(float_array.dtype, np.object_)
+    return any_array + bool_array + complex_array + float_array + int_array + len(bytes_array) + len(string_array)
+
+
+def test_types():
+    """There isn't much to test.
+
+    Just make sure that using the types doesn't break anything silly like function declarations.
+    """
+    any_array = np.arange(5, dtype=np.float16)
+    bool_array = np.arange(5).astype(bool)
+    bytes_array = np.array([x.to_bytes(x, "little") for x in list(range(5))])
+    complex_array = 1j * np.arange(5)
+    float_array = np.arange(5, dtype=np.float32)
+    int_array = np.arange(5)
+    string_array = np.array([str(x) for x in list(range(5))])
+    object_array = np.array([str, "", 1, 2, object, object()])
+    function_which_takes_np_types(any_array, bool_array, bytes_array, complex_array, float_array, int_array, object_array, string_array)

pelutils-3.9.0/tests/test_utils.py

@@ -0,0 +1,13 @@
+import pandas as pd
+from numpy import array, ndarray
+
+from pelutils._utils import isinstance_by_name
+
+
+def test_isinstance_by_name():
+    assert not isinstance_by_name(array([1, 2, 3]), "np", "ndarray")
+    assert isinstance_by_name(array([1, 2, 3]), "numpy", "ndarray")
+    assert isinstance_by_name(pd.DataFrame({"a": [1]}), "pandas", "DataFrame")
+    assert isinstance_by_name(pd.Series([1]), "pandas", "Series")
+    assert not isinstance_by_name(pd.Series([1]), "pd", "Series")
+    assert not isinstance_by_name(ndarray, "np", "ndarray")

pelutils-3.8.4/tests/test_types.py

@@ -1,22 +0,0 @@
-import numpy as np
-
-from pelutils.types import AnyArray, FloatArray, IntArray
-
-
-def function_which_takes_np_types(
-    any_array: AnyArray,
-    float_array: FloatArray,
-    int_array: IntArray,
-) -> FloatArray:
-    return any_array + float_array + int_array
-
-
-def test_types():
-    """There isn't much to test.
-
-    Just make sure that using the types doesn't break anything silly like function declarations.
-    """
-    any_array = np.arange(5, dtype=np.float16)
-    float_array = np.arange(5, dtype=np.float32)
-    int_array = np.arange(5)
-    function_which_takes_np_types(any_array, float_array, int_array)