lazysort 1.0.0.dev0__py3-none-any.whl

lazysort/__init__.py

@@ -0,0 +1,9 @@
+"""
+>>> from lazysort import LazySort
+>>> ls = LazySort([1, 44, 32, 1, 434, 22, 11, 8, 66, 10], work_objs_limit=5)
+>>> ls[4]
+11
+"""
+
+from .main import LazySort as LazySort
+from .clusters import Cluster as Cluster

lazysort/clusters.py

@@ -0,0 +1,167 @@
+from collections.abc import Iterable, Iterator
+from typing import TypeVar, Generic
+import dataclasses
+
+_V = TypeVar('_V')
+
+
+@dataclasses.dataclass
+class Cluster(Generic[_V]):
+    """
+    A cluster of values.
+
+    :param val_min: Minimum value in the cluster.
+    :param val_max: Maximum value in the cluster.
+    :param src_idx_from: The index at which the first value of the cluster is located in a source sequence.
+    :param src_idx_to: The index at which the last value of the cluster is located in a source sequence.
+    :param idx: The index at which the cluster begins in a sorted sequence.
+    :param size: Number of values in the cluster.
+    :param next: The next cluster in the chain.
+    """
+    val_min: _V
+    val_max: _V
+    src_idx_from: int
+    src_idx_to: int
+    idx: int = -1
+    size: int = 1
+    next: 'Cluster[_V] | None' = dataclasses.field(default=None, compare=False, repr=False)
+
+    @classmethod
+    def new(cls, val: _V, src_idx: int = 0) -> 'Cluster[_V]':
+        """
+        Create a new cluster of values.
+
+        :param val: A value the cluster will contain.
+        :param src_idx: The index at which the value is located in a source sequence.
+        :return: A new cluster.
+        """
+        return cls(val_min=val, val_max=val, src_idx_from=src_idx, src_idx_to=src_idx + 1)
+
+    def add_value(self, val: _V, src_idx: int) -> None:
+        """
+        Add a new value to the cluster.
+
+        :param val: A new value the cluster will contain.
+        :param src_idx: The index at which the value is located in a source sequence.
+        """
+        self.size += 1
+        if val < self.val_min:
+            self.val_min = val
+        elif val > self.val_max:
+            self.val_max = val
+        if src_idx + 1 > self.src_idx_to:
+            self.src_idx_to = src_idx + 1
+        elif src_idx < self.src_idx_from:
+            self.src_idx_from = src_idx
+
+    def is_value_contained(self, val: _V) -> bool:
+        """
+        Return True if the cluster contains a value.
+
+        :param val: A value.
+        """
+        return self.val_min <= val <= self.val_max
+
+    def is_homogeneous(self) -> bool:
+        """
+        Return True if the cluster contains only identical values.
+        """
+        return self.val_min == self.val_max
+
+    def find_by_index(self, idx: int) -> 'Cluster[_V]':
+        """
+        Find a cluster in the chain by idx attribute.
+
+        :param idx: An index of an item in a sorted sequence.
+        """
+        cluster: Cluster[_V] = self
+        while not cluster.is_index_contained(idx):
+            if cluster.next is None:
+                raise ValueError(f'A cluster with the item index "{idx}" not found.')
+            cluster = cluster.next
+        return cluster
+
+    def is_index_contained(self, idx: int) -> bool:
+        """
+        Return True if the cluster contains an item with the given index.
+
+        :param idx: An index of an item in a sorted sequence.
+        """
+        return self.idx <= idx < self.idx + self.size
+
+    def last(self) -> 'Cluster[_V]':
+        """
+        Return the last cluster in the chain.
+        """
+        cluster: Cluster[_V] = self
+        while cluster.next is not None:
+            cluster = cluster.next
+        return cluster
+
+    def join_next(self) -> None:
+        """
+        Join the current cluster with the next one.
+        """
+        if self.next is None:
+            raise ValueError('No next cluster to join.')
+        self.size += self.next.size
+        self.val_max = max(self.val_max, self.next.val_max)
+        self.val_min = min(self.val_min, self.next.val_min)
+        self.src_idx_from = min(self.src_idx_from, self.next.src_idx_from)
+        self.src_idx_to = max(self.src_idx_to, self.next.src_idx_to)
+        self.next = self.next.next
+
+    def replace(self, old: 'Cluster[_V]', new: 'Cluster[_V]') -> 'Cluster[_V]':
+        """
+        Replace the old cluster with the new one.
+
+        :param old: The cluster that will be replaced.
+        :param new: The cluster that will replace the old one.
+        :return: A head of the new cluster chain.
+        """
+        if self is old:
+            if self.next is not None:
+                new.last().next = self.next
+            return new
+        cluster: Cluster[_V] = self
+        while cluster.next is not None:
+            if cluster.next is old:
+                cluster.next = new
+                if old.next is not None:
+                    new.last().next = old.next
+                return self
+            cluster = cluster.next
+        raise LookupError('Cluster not found.')
+
+    def __iter__(self) -> Iterator['Cluster']:
+        cluster: Cluster[_V] = self
+        yield cluster
+        while cluster.next is not None:
+            cluster = cluster.next
+            yield cluster
+
+    def __len__(self) -> int:
+        length: int = 1
+        cluster: Cluster[_V] = self
+        while cluster.next is not None:
+            length += 1
+            cluster = cluster.next
+        return length
+
+
+def bond_all(clusters: Iterable[Cluster[_V]]) -> Cluster[_V]:
+    """
+    Bond a list of clusters.
+
+    :param clusters: List of clusters.
+    :return: Chain of clusters.
+    """
+    if not clusters:
+        raise ValueError('No clusters to bond.')
+    cluster_iter: Iterator[Cluster[_V]] = iter(clusters)
+    cluster: Cluster[_V] = next(cluster_iter)
+    head = cluster
+    for next_group in cluster_iter:
+        cluster.next = next_group
+        cluster = next_group
+    return head

lazysort/main.py

@@ -0,0 +1,244 @@
+from collections.abc import Iterable, Callable, Iterator
+from typing import TypeVar, Generic, overload
+
+from .clusters import Cluster, bond_all
+from .protocols import RandomAccessSource, SortFunction
+
+_T = TypeVar('_T')
+_V = TypeVar('_V')
+
+
+class LazySort(Generic[_T, _V]):
+    clusters: Cluster[_V]
+    work_cluster: Cluster[_V]
+    work_items: tuple[_T, ...]
+    _seq: RandomAccessSource[_T]
+    _seq_length: int
+    _clusters_limit: int
+    _key_fn: Callable[[_T], _V] | None
+    _sort_fn: SortFunction[_T, _V]
+    _reverse: bool
+    _work_items_limit: int
+
+    def __init__(
+            self,
+            seq: RandomAccessSource[_T],
+            work_items_limit: int,
+            clusters_limit: int | None = None,
+            key_fn: Callable[[_T], _V] | None = None,
+            sort_fn: SortFunction[_T, _V] = sorted,
+            reverse: bool = False,
+    ):
+        """
+        Splits a sequence into parts as needed and sorts them.
+
+        sort_fn must take arguments: iterable_obj as Iterable, key as Callable, reverse as bool.
+
+        :param seq: Sequence of items.
+        :param work_items_limit: Limit of items loaded into memory for sorting.
+        :param clusters_limit: Limit of clusters. It must be greater than 5.
+        :param key_fn: A function that takes an item and returns the value to sort by.
+        :param sort_fn: A sorting function.
+        :param reverse: Sort in reverse order. Passed to the sort function as is.
+        """
+        self._seq = seq
+        self._seq_length = len(seq)
+        self.work_items = tuple()
+        self._work_items_limit = work_items_limit
+        if clusters_limit is None:
+            self._clusters_limit = max(len(seq) // (work_items_limit // 2), 6)
+        else:
+            if clusters_limit < 6:
+                raise ValueError('Clusters limit must be greater than 5.')
+            self._clusters_limit = clusters_limit
+        self._key_fn = key_fn
+        self._sort_fn = sort_fn
+        self._reverse = reverse
+        if self._key_fn is None:
+            values: Iterable[_V] = self._seq
+        else:
+            values: Iterable[_V] = map(self._key_fn, self._seq)
+        self.clusters = self._determine_clusters(values=values, limit=self._clusters_limit, reverse=self._reverse)
+
+    def __iter__(self) -> Iterator[_T]:
+        for idx in range(self._seq_length):
+            yield self[idx]
+
+    def __len__(self) -> int:
+        return self._seq_length
+
+    def __repr__(self) -> str:
+        return (
+            f'<{self.__class__.__name__}'
+            f' id: {id(self)},'
+            f' items: {len(self.work_items)}/{self._seq_length},'
+            f' clusters: {len(self.clusters)}/{self._clusters_limit}>'
+        )
+
+    @overload
+    def __getitem__(self, item: int, /) -> _T: ...
+
+    @overload
+    def __getitem__(self, item: slice, /) -> list[_T]: ...
+
+    def __getitem__(self, item: int|slice, /) -> _T | list[_T]:
+        if isinstance(item, int):
+            return self._get_item(item)
+        return [self._get_item(x) for x in range(item.start or 0, item.stop, item.step or 1)]
+
+    def _get_item(self, idx: int) -> _T:
+        """
+        Return an item by index.
+        """
+        if not (-self._seq_length <= idx < self._seq_length):
+            raise IndexError('Index is out of range.')
+        if idx < 0:
+            idx += self._seq_length
+
+        if not (self.work_items and self.work_cluster.is_index_contained(idx)):
+            self.work_cluster = self._get_work_cluster(idx)
+            self.work_items = self._get_sorted_items(self.work_cluster)
+        if self.work_cluster.is_homogeneous():
+            return self.work_items[0]
+        return self.work_items[idx - self.work_cluster.idx]
+
+    def _get_work_cluster(self, idx: int) -> Cluster[_V]:
+        """
+        Find a cluster which contains an item with the defined index and split it if necessary.
+        """
+        cluster: Cluster[_V] = self.clusters.find_by_index(idx)
+        while cluster.size > self._work_items_limit:
+            if cluster.is_homogeneous():
+                break
+            clusters_should_be_saved: int = max(
+                self._clusters_limit // 2,
+                self._clusters_limit - cluster.size // (self._work_items_limit // 2) - 1
+            )
+            self.clusters = self._shorten_the_chain_of_clusters(
+                clusters=self.clusters,
+                protected=cluster,
+                max_cluster_size=self._work_items_limit,
+                max_clusters=clusters_should_be_saved
+            )
+            if self._key_fn is None:
+                cluster_values: Iterable[_V] = filter(cluster.is_value_contained, self._seq)
+            else:
+                cluster_values: Iterable[_V] = filter(
+                    cluster.is_value_contained,
+                    map(self._key_fn, self._seq[cluster.src_idx_from:cluster.src_idx_to])
+                )
+            new_clusters: Cluster[_V] = self._determine_clusters(
+                values=cluster_values,
+                limit=self._clusters_limit - len(self.clusters) + 1,
+                reverse=self._reverse
+            )
+            for new_cluster in new_clusters:
+                new_cluster.src_idx_from += cluster.src_idx_from
+                new_cluster.src_idx_to = cluster.src_idx_to
+                new_cluster.idx += cluster.idx
+            self.clusters = self.clusters.replace(cluster, new_clusters)
+            cluster = new_clusters.find_by_index(idx)
+        return cluster
+
+    def _get_sorted_items(self, cluster: Cluster[_V]) -> tuple[_T, ...]:
+        """
+        Sort items that contained by cluster.
+        """
+        if cluster.is_homogeneous():
+            return (self._seq[cluster.src_idx_from],)
+
+        if self._key_fn is None:
+            cluster_items: Iterable[_T] = filter(
+                cluster.is_value_contained,
+                self._seq[cluster.src_idx_from:cluster.src_idx_to]
+            )
+        else:
+            cluster_items: Iterable[_T] = filter(
+                lambda x: cluster.is_value_contained(self._key_fn(x)),
+                self._seq[cluster.src_idx_from:cluster.src_idx_to]
+            )
+        return tuple(self._sort_fn(cluster_items, key=self._key_fn, reverse=self._reverse))
+
+    @staticmethod
+    def _shorten_the_chain_of_clusters(
+            clusters: Cluster[_V], protected: Cluster[_V], max_cluster_size: int, max_clusters: int
+    ) -> Cluster[_V]:
+        """
+        Reduce the number of clusters.
+        """
+        # Join all pairs of clusters whose total size is less than max_cluster_size.
+        # The reclustering process is resource-intensive. Therefore, it's more efficient to merge clusters while their
+        # size is less than max_cluster_size, even if the number of clusters becomes smaller than required.
+        cluster: Cluster[_V] = clusters
+        while cluster.next is not None:
+            if cluster is not protected:
+                while (
+                        cluster.next is not protected
+                        and cluster.next is not None
+                        and cluster.size + cluster.next.size <= max_cluster_size
+                ):
+                    cluster.join_next()
+            if cluster.next is not None:
+                cluster = cluster.next
+
+        # Join pairs of clusters whose total size the smallest.
+        for _ in range(len(clusters) - max_clusters):
+            pairs: Iterable[Cluster[_V]] = filter(
+                lambda x:not (x.next is None or x is protected or x.next is protected),
+                clusters
+            )
+            cluster: Cluster[_V] = min(pairs, key=lambda x: x.size + x.next.size)
+            cluster.join_next()
+        return clusters
+
+    @staticmethod
+    def _determine_clusters(values: Iterable[_V], limit: int, reverse: bool) -> Cluster[_V]:
+        """
+        Analyze the dataset and identify clusters of values.
+        """
+        data_iter: Iterator[_V] = iter(values)
+        clusters: list[Cluster[_V]] = [Cluster.new(next(data_iter))]
+        for src_idx, value in enumerate(data_iter, 1):
+            left: int = 0
+            right: int = len(clusters) - 1
+            cursor: int = right // 2
+            while left != right:
+                cluster = clusters[cursor]
+                if cluster.val_max < value:
+                    left = cursor + 1
+                elif value < cluster.val_min:
+                    right = max(left, cursor - 1)
+                else:
+                    break
+                cursor = left + (right - left) // 2
+            cluster = clusters[cursor]
+
+            if value < cluster.val_min:
+                if len(clusters) < limit:
+                    clusters.insert(cursor, Cluster.new(value, src_idx=src_idx))
+                elif cursor == 0:
+                    cluster.add_value(value, src_idx=src_idx)
+                elif clusters[cursor - 1].size <= cluster.size:
+                    clusters[cursor - 1].add_value(value, src_idx=src_idx)
+                else:
+                    cluster.add_value(value, src_idx=src_idx)
+            elif value > cluster.val_max:
+                if len(clusters) < limit:
+                    clusters.insert(cursor + 1, Cluster.new(value, src_idx=src_idx))
+                elif cursor == len(clusters) - 1:
+                    cluster.add_value(value, src_idx=src_idx)
+                elif cluster.size <= clusters[cursor + 1].size:
+                    cluster.add_value(value, src_idx=src_idx)
+                else:
+                    clusters[cursor + 1].add_value(value, src_idx=src_idx)
+            else:
+                cluster.add_value(value, src_idx=src_idx)
+
+        if reverse:
+            clusters.reverse()
+        idx: int = 0
+        for cluster in clusters:
+            cluster.idx = idx
+            idx += cluster.size
+        bond_all(clusters)
+        return clusters[0]

lazysort/protocols.py

@@ -0,0 +1,38 @@
+from collections.abc import Iterator, Iterable, Callable
+from typing import Protocol, overload, TypeVar
+
+_T_cov = TypeVar('_T_cov', covariant=True)
+_T = TypeVar('_T')
+_V_con = TypeVar('_V_con', contravariant=True)
+
+
+class RandomAccessSource(Protocol[_T_cov]):
+    def __len__(self) -> int: ...
+
+    def __iter__(self) -> Iterator[_T_cov]: ...
+
+    @overload
+    def __getitem__(self, item: int, /) -> _T_cov: ...
+
+    @overload
+    def __getitem__(self, item: slice, /) -> Iterable[_T_cov]: ...
+
+    @overload
+    def __getitem__(self, item: int | slice, /) -> _T_cov | Iterable[_T_cov]: ...
+
+
+class SortFunction(Protocol[_T, _V_con]):
+    @overload
+    def __call__(
+            self, iterable: Iterable[_T], /, *, key: Callable[[_T], _V_con], reverse: bool = False
+    ) -> Iterable[_T]: ...
+
+    @overload
+    def __call__(
+            self, iterable: Iterable[_T], /, *, key: None = None, reverse: bool = False
+    ) -> Iterable[_T]: ...
+
+    @overload
+    def __call__(
+            self, iterable: Iterable[_T], /, *, key: Callable[[_T], _V_con] | None = None, reverse: bool = False
+    ) -> Iterable[_T]: ...

lazysort-1.0.0.dev0.dist-info/METADATA

@@ -0,0 +1,8 @@
+Metadata-Version: 2.4
+Name: lazysort
+Version: 1.0.0.dev0
+Summary: Python module for lazy sorting by parts.
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+
+# LazySort

lazysort-1.0.0.dev0.dist-info/RECORD

@@ -0,0 +1,8 @@
+lazysort/__init__.py,sha256=M2gUdHtRpkAmldQm8Be9A4uImQkbu5fKKYMQHqd2ZxU,213
+lazysort/clusters.py,sha256=vA8kOq-gocSjal5W6xBacRGpFh_db_0s3f0m7VZutA8,5493
+lazysort/main.py,sha256=s6xViQOmvIbpYKhPUr96IL-JZaYL1S0uerttrrdXy98,9847
+lazysort/protocols.py,sha256=OcJxekk-lzriweKTQsZrjEJaLkX__yPoMiN5xloZ0QY,1115
+lazysort-1.0.0.dev0.dist-info/METADATA,sha256=T64dzvpYr3EX5Q2b2l0cXzjs5KegEp43y_D88E3XP6w,183
+lazysort-1.0.0.dev0.dist-info/WHEEL,sha256=YVMoNqKzERt-wjUZwJ33xBGAwnFl-4cqbYkTtWa4itE,91
+lazysort-1.0.0.dev0.dist-info/top_level.txt,sha256=_CeyhGl4DF2OzyZIdvxV1CegBnHYad4hxi1VpBmA3y0,9
+lazysort-1.0.0.dev0.dist-info/RECORD,,

lazysort-1.0.0.dev0.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (84.0.0)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

lazysort-1.0.0.dev0.dist-info/top_level.txt

@@ -0,0 +1 @@
+lazysort