uncapped 0.1.0__py3-none-any.whl

uncapped/__init__.py

@@ -0,0 +1,40 @@
+"""uncapped: get every result from searches that stop at N results.
+
+GitHub search stops at 1,000 results, Elasticsearch at 10,000, most listing
+sites at a few hundred pages. uncapped cuts one big search into windows that
+each fit under the cap, by splitting the filters you give it (price, date,
+category...), so paging through every window returns everything.
+
+    >>> from uncapped import split, DateRange
+    >>> for window in split(count, cap=1000, dims=[DateRange("created", "2008-01-01", "today")]):
+    ...     fetch_all(window.filters)
+"""
+
+from .core import (
+    AsyncSplit,
+    CapExceeded,
+    Split,
+    Stats,
+    UncappedWarning,
+    Window,
+    asplit,
+    split,
+)
+from .dims import DateRange, Dimension, Range, Values
+
+__all__ = [
+    "split",
+    "asplit",
+    "Range",
+    "DateRange",
+    "Values",
+    "Dimension",
+    "Window",
+    "Stats",
+    "Split",
+    "AsyncSplit",
+    "CapExceeded",
+    "UncappedWarning",
+]
+
+__version__ = "0.1.0"

uncapped/core.py

@@ -0,0 +1,446 @@
+"""The splitting engine."""
+
+from __future__ import annotations
+
+import asyncio
+import inspect
+import json
+import math
+import warnings
+from dataclasses import dataclass, field
+from typing import (
+    Any,
+    AsyncIterator,
+    Awaitable,
+    Callable,
+    Dict,
+    Iterator,
+    List,
+    MutableMapping,
+    Optional,
+    Sequence,
+    Tuple,
+    Union,
+)
+
+from .dims import Dimension, Slice
+
+Filters = Dict[str, Any]
+CountFn = Callable[[Filters], int]
+AsyncCountFn = Callable[[Filters], Union[int, Awaitable[int]]]
+
+_States = Dict[str, Slice]
+
+
+class UncappedWarning(UserWarning):
+    """Some results cannot be fetched. See the message and :class:`Stats`."""
+
+
+class CapExceeded(Exception):
+    """Raised with ``on_capped="raise"`` when a window cannot be narrowed under the cap."""
+
+    def __init__(self, window: "Window", cap: int) -> None:
+        super().__init__(
+            f"{window.count:,} results for {window.filters} and no dimension can narrow it "
+            f"further (cap is {cap:,}). Add another dimension or a finer step."
+        )
+        self.window = window
+
+
+@dataclass(frozen=True)
+class Window:
+    """One slice of the search that returns at most ``cap`` results.
+
+    Attributes:
+        filters: Filter values for this window, keyed by dimension name, in the
+            order the dimensions were given. Dimensions that were never needed
+            are left out, so the dict is empty when the whole search fits.
+        count: What your ``count`` function returned for these filters.
+        complete: ``False`` when ``count`` is still over the cap because every
+            dimension is down to a single point. Only ``cap`` results of such a
+            window can be fetched.
+        depth: How many splits it took to get here (for a merged window, the
+            deepest of its parts).
+    """
+
+    filters: Filters
+    count: int
+    complete: bool = True
+    depth: int = 0
+
+
+@dataclass
+class Stats:
+    """What happened during a run. Read it from ``run.stats`` after iterating.
+
+    Attributes:
+        total: Count for the unfiltered search.
+        windows: Windows yielded.
+        fetchable: Results you can fetch by paging through every window
+            (``min(count, cap)`` summed over windows).
+        capped_windows: Windows still over the cap (``complete=False``).
+        unaccounted: Results the dimensions did not reach: the children of a
+            split added up to less than their parent. Usually items with no
+            value for that filter (a listing without a price) or values outside
+            the range you gave.
+        queries: Calls made to your ``count`` function.
+        cache_hits: Counts answered from the cache instead.
+        max_depth: Deepest split.
+    """
+
+    total: int = 0
+    windows: int = 0
+    fetchable: int = 0
+    capped_windows: int = 0
+    unaccounted: int = 0
+    queries: int = 0
+    cache_hits: int = 0
+    max_depth: int = 0
+
+    @property
+    def coverage(self) -> float:
+        """``fetchable / total``, between 0 and 1 (1.0 when there is nothing to fetch)."""
+        return 1.0 if self.total == 0 else min(1.0, self.fetchable / self.total)
+
+
+@dataclass
+class _Plan:
+    """Settings and helpers shared by the sync and async runners."""
+
+    cap: int
+    dims: Sequence[Dimension]
+    fanout: Union[int, str]
+    max_fanout: int
+    cache: Optional[MutableMapping[str, int]]
+    on_capped: str
+    merge: bool
+    stats: Stats = field(default_factory=Stats)
+    # The newest window, held back so the next one can be merged into it:
+    # (states, window, stats.unaccounted when it was held).
+    _held: Optional[Tuple[_States, Window, int]] = field(default=None, init=False, repr=False)
+
+    def __post_init__(self) -> None:
+        if not isinstance(self.cap, int) or self.cap < 1:
+            raise ValueError(f"cap must be a positive integer, got {self.cap!r}")
+        if isinstance(self.dims, Dimension):
+            self.dims = [self.dims]
+        self.dims = list(self.dims)
+        names = [d.name for d in self.dims]
+        if len(set(names)) != len(names):
+            raise ValueError(f"dimension names must be unique, got {names}")
+        if self.fanout != "auto" and not (isinstance(self.fanout, int) and self.fanout >= 2):
+            raise ValueError(f'fanout must be "auto" or an integer >= 2, got {self.fanout!r}')
+        if self.max_fanout < 2:
+            raise ValueError(f"max_fanout must be >= 2, got {self.max_fanout!r}")
+        if self.on_capped not in ("warn", "raise", "ignore"):
+            raise ValueError(f'on_capped must be "warn", "raise" or "ignore", got {self.on_capped!r}')
+        if not isinstance(self.merge, bool):
+            raise ValueError(f"merge must be True or False, got {self.merge!r}")
+
+    # -- building filters -------------------------------------------------
+
+    def render(self, states: _States) -> Filters:
+        return {d.name: d.render(states[d.name]) for d in self.dims if d.name in states}
+
+    @staticmethod
+    def key(filters: Filters) -> str:
+        return json.dumps(filters, sort_keys=True, default=str)
+
+    def children(self, states: _States, count: int) -> Optional[List[_States]]:
+        """Split along the first dimension that still has room, or None if none has."""
+        for dim in self.dims:
+            first, last = states.get(dim.name, dim.full)
+            if first == last:
+                continue
+            k = self._fanout(count, last - first + 1)
+            return [{**states, dim.name: part} for part in dim.parts((first, last), k)]
+        return None
+
+    def _fanout(self, count: int, width: int) -> int:
+        if self.fanout == "auto":
+            # Aim for children at ~80% of the cap if results were spread evenly;
+            # uneven ones are simply split again.
+            k = min(self.max_fanout, math.ceil(count * 1.25 / self.cap))
+        else:
+            k = int(self.fanout)
+        return max(2, min(k, width))
+
+    # -- counting ---------------------------------------------------------
+
+    def cached(self, filters: Filters) -> Optional[int]:
+        if self.cache is None:
+            return None
+        hit = self.cache.get(self.key(filters))
+        if hit is not None:
+            self.stats.cache_hits += 1
+        return hit
+
+    def record(self, filters: Filters, n: Any) -> int:
+        if isinstance(n, bool) or not isinstance(n, int) or n < 0:
+            raise TypeError(f"count() must return a non-negative int, got {n!r} for {filters}")
+        self.stats.queries += 1
+        if self.cache is not None:
+            self.cache[self.key(filters)] = n
+        return n
+
+    # -- results ----------------------------------------------------------
+
+    def account(self, parent: int, children: Sequence[int]) -> None:
+        self.stats.unaccounted += max(0, parent - sum(children))
+
+    def emit(self, states: _States, n: int, depth: int) -> Window:
+        complete = n <= self.cap
+        window = Window(self.render(states), n, complete, depth)
+        if not complete:
+            if self.on_capped == "raise":
+                raise CapExceeded(window, self.cap)
+            self.stats.capped_windows += 1
+        s = self.stats
+        s.windows += 1
+        s.fetchable += min(n, self.cap)
+        s.max_depth = max(s.max_depth, depth)
+        return window
+
+    # -- merging ----------------------------------------------------------
+    #
+    # Splitting a window that sits next to a dense spot leaves many tiny
+    # neighbours (1, 3, 2... results), and each one costs at least one page
+    # request to fetch. Neighbours that together still fit under the cap are
+    # sent as one wider window instead.
+    #
+    # Two windows are merged only when they differ in nothing but their last
+    # split dimension, and the merged count is exact: windows come out in
+    # order, so everything between the two (in that dimension) was counted as
+    # empty. The one exception is results no dimension could reach (items with
+    # no value for a filter): they sit inside such a gap and a wider window
+    # would include them. If any were found since the held window, it is not
+    # merged.
+
+    def hold(self, states: _States, window: Window) -> Optional[Window]:
+        """Take a new window; return the previous one if it can no longer grow."""
+        held = self._held
+        if held is not None:
+            merged = self._merged(held, states, window)
+            if merged is not None:
+                self._held = merged
+                self.stats.windows -= 1
+                return None
+        self._held = (states, window, self.stats.unaccounted)
+        return held[1] if held is not None else None
+
+    def release(self) -> Optional[Window]:
+        """The last window, at the end of a run."""
+        held, self._held = self._held, None
+        return held[1] if held is not None else None
+
+    def _merged(
+        self, held: Tuple[_States, Window, int], states: _States, window: Window
+    ) -> Optional[Tuple[_States, Window, int]]:
+        prev_states, prev, unaccounted = held
+        if not (self.merge and prev.complete and window.complete):
+            return None
+        if prev.count + window.count > self.cap or self.stats.unaccounted != unaccounted:
+            return None
+        if not states or prev_states.keys() != states.keys():
+            return None
+        last = next(d for d in reversed(self.dims) if d.name in states)
+        if not last.mergeable:
+            return None
+        if any(prev_states[name] != states[name] for name in states if name != last.name):
+            return None
+        span = {**states, last.name: (prev_states[last.name][0], states[last.name][1])}
+        count = prev.count + window.count
+        merged = Window(self.render(span), count, True, max(prev.depth, window.depth))
+        return span, merged, unaccounted
+
+    def finish(self) -> None:
+        s = self.stats
+        if self.on_capped == "warn" and s.capped_windows:
+            warnings.warn(
+                f"{s.capped_windows} window(s) still have more than {self.cap:,} results; only "
+                f"the first {self.cap:,} of each can be fetched. Add another dimension or use a "
+                f"finer step. Coverage: {s.coverage:.1%}.",
+                UncappedWarning,
+                stacklevel=3,
+            )
+        if s.unaccounted:
+            warnings.warn(
+                f"{s.unaccounted:,} result(s) were not reachable through the dimensions (for "
+                f"example items with no value for a filter, or values outside its range). "
+                f"Widen the ranges or fetch those items separately.",
+                UncappedWarning,
+                stacklevel=3,
+            )
+
+
+class Split(_Plan):
+    """A planned run. Iterate it to get :class:`Window` objects; read ``.stats`` after."""
+
+    def __init__(self, count: CountFn, **settings: Any) -> None:
+        super().__init__(**settings)
+        self.count = count
+
+    def _count(self, states: _States) -> int:
+        filters = self.render(states)
+        hit = self.cached(filters)
+        if hit is not None:
+            return hit
+        return self.record(filters, self.count(dict(filters)))
+
+    def __iter__(self) -> Iterator[Window]:
+        self.stats = Stats()
+        self._held = None
+        root: _States = {}
+        total = self._count(root)
+        self.stats.total = total
+        stack: List[Tuple[_States, int, int]] = [(root, total, 0)]
+        while stack:
+            states, n, depth = stack.pop()
+            if n == 0:
+                continue
+            kids = None if n <= self.cap else self.children(states, n)
+            if kids is None:
+                done = self.hold(states, self.emit(states, n, depth))
+                if done is not None:
+                    yield done
+                continue
+            counts = [self._count(kid) for kid in kids]
+            self.account(n, counts)
+            stack.extend((kid, c, depth + 1) for kid, c in reversed(list(zip(kids, counts))))
+        done = self.release()
+        if done is not None:
+            yield done
+        self.finish()
+
+
+class AsyncSplit(_Plan):
+    """Async version of :class:`Split`. Use ``async for window in run``."""
+
+    def __init__(self, count: AsyncCountFn, *, concurrency: int = 4, **settings: Any) -> None:
+        super().__init__(**settings)
+        if concurrency < 1:
+            raise ValueError(f"concurrency must be >= 1, got {concurrency!r}")
+        self.count = count
+        self.concurrency = concurrency
+
+    async def _count(self, states: _States, gate: asyncio.Semaphore) -> int:
+        filters = self.render(states)
+        hit = self.cached(filters)
+        if hit is not None:
+            return hit
+        async with gate:
+            n = self.count(dict(filters))
+            if inspect.isawaitable(n):
+                n = await n
+        return self.record(filters, n)
+
+    def __aiter__(self) -> AsyncIterator[Window]:
+        return self._run()
+
+    async def _run(self) -> AsyncIterator[Window]:
+        self.stats = Stats()
+        self._held = None
+        gate = asyncio.Semaphore(self.concurrency)
+        root: _States = {}
+        total = await self._count(root, gate)
+        self.stats.total = total
+        stack: List[Tuple[_States, int, int]] = [(root, total, 0)]
+        while stack:
+            states, n, depth = stack.pop()
+            if n == 0:
+                continue
+            kids = None if n <= self.cap else self.children(states, n)
+            if kids is None:
+                done = self.hold(states, self.emit(states, n, depth))
+                if done is not None:
+                    yield done
+                continue
+            counts = await asyncio.gather(*(self._count(kid, gate) for kid in kids))
+            self.account(n, counts)
+            stack.extend((kid, c, depth + 1) for kid, c in reversed(list(zip(kids, counts))))
+        done = self.release()
+        if done is not None:
+            yield done
+        self.finish()
+
+
+def split(
+    count: CountFn,
+    *,
+    cap: int,
+    dims: Union[Dimension, Sequence[Dimension]],
+    fanout: Union[int, str] = "auto",
+    max_fanout: int = 16,
+    cache: Optional[MutableMapping[str, int]] = None,
+    on_capped: str = "warn",
+    merge: bool = True,
+) -> Split:
+    """Cut a search into windows that each return at most ``cap`` results.
+
+    Args:
+        count: ``count(filters) -> int``. Return how many results the search has
+            with these filters. ``filters`` maps dimension names to values;
+            dimensions not in it should not be filtered on.
+        cap: The most results the source will return for one search.
+        dims: What to split on, in order of preference. The first dimension is
+            split until it is down to a single point, then the next one is used.
+        fanout: How many parts to cut a window into. ``"auto"`` (default) picks
+            it from how far over the cap the window is, which saves calls;
+            ``2`` always halves.
+        max_fanout: Upper bound for ``"auto"``.
+        cache: Any dict-like object (``dict``, ``shelve.open(...)``) to keep
+            counts in. Pass a persistent one to resume a run without spending
+            API calls on counts you already have.
+        on_capped: What to do when a window is still over the cap and cannot be
+            narrowed: ``"warn"`` (default; it is yielded with
+            ``complete=False``), ``"raise"`` :class:`CapExceeded`, or
+            ``"ignore"``.
+        merge: Merge neighbouring windows that together still fit under the
+            cap (default ``True``), so fetching them takes fewer requests. Each
+            window then comes out one step later. ``False`` keeps every window
+            as it was split.
+
+    Returns:
+        A :class:`Split`. Iterate it for :class:`Window` objects in dimension
+        order; read ``.stats`` afterwards.
+    """
+    return Split(
+        count,
+        cap=cap,
+        dims=dims,
+        fanout=fanout,
+        max_fanout=max_fanout,
+        cache=cache,
+        on_capped=on_capped,
+        merge=merge,
+    )
+
+
+def asplit(
+    count: AsyncCountFn,
+    *,
+    cap: int,
+    dims: Union[Dimension, Sequence[Dimension]],
+    fanout: Union[int, str] = "auto",
+    max_fanout: int = 16,
+    cache: Optional[MutableMapping[str, int]] = None,
+    on_capped: str = "warn",
+    merge: bool = True,
+    concurrency: int = 4,
+) -> AsyncSplit:
+    """Async :func:`split`. ``count`` may be async; sibling counts run concurrently.
+
+    ``concurrency`` limits how many ``count`` calls run at the same time.
+    Use ``async for window in asplit(...)``.
+    """
+    return AsyncSplit(
+        count,
+        cap=cap,
+        dims=dims,
+        fanout=fanout,
+        max_fanout=max_fanout,
+        cache=cache,
+        on_capped=on_capped,
+        merge=merge,
+        concurrency=concurrency,
+    )

uncapped/dims.py

@@ -0,0 +1,223 @@
+"""Dimensions: the filters a search can be narrowed by.
+
+Every dimension is an ordered list of points (prices, dates, categories...).
+While splitting, uncapped tracks a dimension's current slice as an inclusive
+pair of point indices ``(first, last)`` and only turns it into real values
+when it builds the filters it passes to your ``count`` function.
+"""
+
+from __future__ import annotations
+
+from datetime import date, datetime, timedelta
+from decimal import Decimal
+from typing import Any, Hashable, Iterable, List, Sequence, Tuple, Union
+
+#: An inclusive slice of a dimension, as indices into its points.
+Slice = Tuple[int, int]
+
+Number = Union[int, float]
+DateLike = Union[date, datetime, str]
+
+
+class Dimension:
+    """Base class for something a search can be filtered by.
+
+    Subclasses define how many points they have (``__len__``), how a slice of
+    points is shown to your ``count`` function (:meth:`render`) and, if needed,
+    how a slice is cut into smaller slices (:meth:`parts`).
+    """
+
+    name: str
+
+    #: Whether two neighbouring slices can be sent as one filter (a wider range).
+    #: uncapped uses this to merge small windows. ``False`` for single-value
+    #: :class:`Values`, where the source only accepts one value at a time.
+    mergeable: bool = True
+
+    def __len__(self) -> int:  # pragma: no cover - abstract
+        raise NotImplementedError
+
+    @property
+    def full(self) -> Slice:
+        """The slice covering every point of the dimension."""
+        return (0, len(self) - 1)
+
+    def render(self, part: Slice) -> Any:  # pragma: no cover - abstract
+        """The filter value your ``count`` function receives for ``part``."""
+        raise NotImplementedError
+
+    def parts(self, part: Slice, k: int) -> List[Slice]:
+        """Cut ``part`` into ``k`` contiguous, non-overlapping slices."""
+        return _chunks(part, k)
+
+    def __repr__(self) -> str:
+        return f"{type(self).__name__}({self.name!r}, {len(self)} points)"
+
+
+class Range(Dimension):
+    """A numeric range filter such as ``price_min``/``price_max``.
+
+    Your ``count`` function receives ``(lo, hi)``, both **inclusive**.
+
+    Args:
+        name: Key used in the filters dict.
+        lo: Smallest value to cover.
+        hi: Largest value to cover.
+        step: The smallest difference the source can tell apart. Use ``1`` for
+            whole numbers (the default) or e.g. ``0.01`` for prices in cents.
+
+    Example::
+
+        Range("price", 0, 5_000_000)          # -> filters["price"] == (0, 2_499_999)
+        Range("rating", 0, 5, step=0.1)       # -> filters["rating"] == (0.0, 2.4)
+    """
+
+    def __init__(self, name: str, lo: Number, hi: Number, step: Number = 1) -> None:
+        if step <= 0:
+            raise ValueError(f"{name}: step must be positive, got {step!r}")
+        if hi < lo:
+            raise ValueError(f"{name}: hi ({hi!r}) is smaller than lo ({lo!r})")
+        self.name = name
+        self.lo, self.hi, self.step = lo, hi, step
+        self._exact = all(isinstance(v, int) for v in (lo, hi, step))
+        if self._exact:
+            self._n = (hi - lo) // step + 1
+        else:
+            # Decimal keeps points like 0.1, 0.2, 0.3 exact instead of 0.30000000000000004.
+            self._dlo, dhi, self._dstep = (Decimal(str(v)) for v in (lo, hi, step))
+            self._n = int((dhi - self._dlo) // self._dstep) + 1
+
+    def __len__(self) -> int:
+        return self._n
+
+    def point(self, i: int) -> Number:
+        if self._exact:
+            return self.lo + i * self.step
+        return float(self._dlo + i * self._dstep)
+
+    def render(self, part: Slice) -> Tuple[Number, Number]:
+        return (self.point(part[0]), self.point(part[1]))
+
+
+class DateRange(Dimension):
+    """A date or datetime range filter such as ``created:2020-01-01..2020-06-30``.
+
+    Your ``count`` function receives ``(start, end)``, both **inclusive**, as
+    :class:`~datetime.date` or :class:`~datetime.datetime` objects (whichever
+    you passed in).
+
+    Args:
+        name: Key used in the filters dict.
+        start: First date to cover. ``date``, ``datetime`` or ISO string.
+        end: Last date to cover. ``"today"`` and ``"now"`` are accepted.
+        step: Smallest gap the source can tell apart. Defaults to one day for
+            dates and one second for datetimes.
+
+    Example::
+
+        DateRange("created", "2008-01-01", "today")
+        DateRange("posted", datetime(2024, 1, 1), "now", step=timedelta(minutes=1))
+    """
+
+    def __init__(
+        self,
+        name: str,
+        start: DateLike,
+        end: DateLike,
+        step: "timedelta | None" = None,
+    ) -> None:
+        start, end = _as_datelike(start), _as_datelike(end)
+        if isinstance(start, datetime) != isinstance(end, datetime):
+            raise TypeError(f"{name}: start and end must both be dates or both be datetimes")
+        is_datetime = isinstance(start, datetime)
+        if step is None:
+            step = timedelta(seconds=1) if is_datetime else timedelta(days=1)
+        if step <= timedelta(0):
+            raise ValueError(f"{name}: step must be positive, got {step!r}")
+        if not is_datetime and step % timedelta(days=1):
+            raise ValueError(f"{name}: with plain dates the step must be whole days")
+        if end < start:
+            raise ValueError(f"{name}: end ({end}) is before start ({start})")
+        self.name = name
+        self.start, self.end, self.step = start, end, step
+        self._n = (end - start) // step + 1
+
+    def __len__(self) -> int:
+        return self._n
+
+    def point(self, i: int) -> "date | datetime":
+        return self.start + i * self.step
+
+    def render(self, part: Slice) -> "Tuple[date | datetime, date | datetime]":
+        return (self.point(part[0]), self.point(part[1]))
+
+
+class Values(Dimension):
+    """A categorical filter such as property type, language or country.
+
+    Args:
+        name: Key used in the filters dict.
+        values: Every value to cover. Anything not listed here is not fetched
+            (uncapped reports it as ``unaccounted``).
+        multi: Set to ``True`` if the source accepts several values at once
+            (``type in (a, b, c)``). Then values are split in groups and your
+            ``count`` function receives a tuple. Otherwise (the default) it is
+            tried one value at a time and receives a single value.
+
+    Example::
+
+        Values("type", ["apartment", "villa", "townhouse"])
+        Values("language", ["python", "go", "rust"], multi=True)
+    """
+
+    def __init__(self, name: str, values: Iterable[Hashable], *, multi: bool = False) -> None:
+        self.name = name
+        self.values: Sequence[Hashable] = tuple(values)
+        self.multi = multi
+        self.mergeable = multi
+        if not self.values:
+            raise ValueError(f"{name}: values is empty")
+        if len(set(self.values)) != len(self.values):
+            raise ValueError(f"{name}: values contains duplicates")
+
+    def __len__(self) -> int:
+        return len(self.values)
+
+    def render(self, part: Slice) -> Any:
+        first, last = part
+        if self.multi:
+            return tuple(self.values[first : last + 1])
+        if first != last:
+            raise ValueError(f"{self.name}: can only filter on one value at a time")
+        return self.values[first]
+
+    def parts(self, part: Slice, k: int) -> List[Slice]:
+        if self.multi:
+            return _chunks(part, k)
+        return [(i, i) for i in range(part[0], part[1] + 1)]
+
+
+def _chunks(part: Slice, k: int) -> List[Slice]:
+    first, last = part
+    width = last - first + 1
+    k = max(1, min(k, width))
+    size, extra = divmod(width, k)
+    out = []
+    start = first
+    for i in range(k):
+        end = start + size + (1 if i < extra else 0) - 1
+        out.append((start, end))
+        start = end + 1
+    return out
+
+
+def _as_datelike(value: DateLike) -> "date | datetime":
+    if isinstance(value, (date, datetime)):
+        return value
+    if value == "today":
+        return date.today()
+    if value == "now":
+        return datetime.now().replace(microsecond=0)
+    if len(value) == 10:
+        return date.fromisoformat(value)
+    return datetime.fromisoformat(value)

uncapped-0.1.0.dist-info/METADATA

@@ -0,0 +1,254 @@
+Metadata-Version: 2.5
+Name: uncapped
+Version: 0.1.0
+Summary: Get every result from APIs and sites that stop at 1,000 results, by splitting the search into windows that fit under the cap.
+Project-URL: Homepage, https://github.com/bhatti013/uncapped
+Project-URL: Issues, https://github.com/bhatti013/uncapped/issues
+Author: Abdul Wahab
+License-Expression: MIT
+License-File: LICENSE
+Keywords: api,crawler,data extraction,github search,pagination,result limit,scraping,search
+Classifier: Development Status :: 4 - Beta
+Classifier: Intended Audience :: Developers
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3 :: Only
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Programming Language :: Python :: 3.14
+Classifier: Topic :: Internet :: WWW/HTTP
+Classifier: Topic :: Software Development :: Libraries :: Python Modules
+Classifier: Typing :: Typed
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+
+# uncapped
+
+[![PyPI](https://img.shields.io/pypi/v/uncapped)](https://pypi.org/project/uncapped/)
+[![Python](https://img.shields.io/pypi/pyversions/uncapped)](https://pypi.org/project/uncapped/)
+[![CI](https://github.com/bhatti013/uncapped/actions/workflows/ci.yml/badge.svg)](https://github.com/bhatti013/uncapped/actions/workflows/ci.yml)
+[![License: MIT](https://img.shields.io/badge/license-MIT-blue.svg)](https://github.com/bhatti013/uncapped/blob/main/LICENSE)
+
+**Get every result from searches that stop at 1,000.**
+
+Lots of APIs and sites cut you off no matter how many results match:
+
+| Source | Most results per search |
+|---|---|
+| GitHub search API | 1,000 |
+| Algolia (default `paginationLimitedTo`) | 1,000 |
+| Elasticsearch / OpenSearch `from` + `size` (default `max_result_window`) | 10,000 |
+| Google Places text search | 60 |
+| Most listing, travel and job sites | a few hundred to ~1,000 |
+
+The fix is always the same: narrow the search with filters (price, date,
+category...) until each piece is under the limit, then page through every
+piece. uncapped does the narrowing for you, picks how finely to cut so you
+spend as few calls as possible, and tells you if anything still can't be
+reached.
+
+- Zero dependencies, Python 3.10+, fully typed
+- Works with any client: `requests`, `httpx`, an SDK, Scrapy, SQL
+- Sync and async
+- Splits on numbers, dates and categories, and falls back to the next filter when one runs out
+- Resumable: counts can be cached in any dict-like store
+
+## Install
+
+```bash
+pip install uncapped
+```
+
+## Example: every Rust repo with more than 1,000 stars
+
+GitHub's search finds 2,478 of them but will only ever return 1,000. You
+tell uncapped how to **count** results for a set of filters; it gives you back
+windows that each fit under the cap.
+
+```python
+import requests
+from uncapped import split, DateRange
+
+QUERY = "language:rust stars:>1000"
+
+def with_filters(filters):
+    if "created" not in filters:
+        return QUERY
+    lo, hi = filters["created"]
+    return f"{QUERY} created:{lo}..{hi}"
+
+def count(filters):
+    r = requests.get("https://api.github.com/search/repositories",
+                     params={"q": with_filters(filters), "per_page": 1})
+    return r.json()["total_count"]
+
+run = split(count, cap=1000, dims=[DateRange("created", "2008-01-01", "today")])
+
+for window in run:
+    print(window.filters["created"], window.count)
+    # page through with_filters(window.filters) as usual: it's under 1,000
+
+print(run.stats)
+```
+
+Real output (November 2026):
+
+```
+created                     repos
+2008-01-01 .. 2017-06-04      419
+2017-06-05 .. 2022-02-19      984
+2022-02-20 .. 2024-06-29      513
+2024-06-30 .. 2026-11-06      562
+
+2,478 repos match. Without uncapped you'd get 1,000.
+4 windows, 2,478 fetchable (100%), 7 count requests.
+```
+
+Paging through those 4 windows downloaded all 2,478 repos, with no duplicates.
+The runnable version, with rate-limit handling and download, is
+[`examples/github_repos.py`](https://github.com/bhatti013/uncapped/blob/main/examples/github_repos.py).
+
+## How it works
+
+1. Count the whole search. Under the cap? Done: one window, no filters.
+2. Over the cap? Cut the first dimension's range into parts. With the default
+   `fanout="auto"` the number of parts depends on how far over the cap it is
+   (5× over the cap → 7 parts), so you don't pay for many rounds of halving.
+3. Count each part. Parts under the cap become windows; parts over it are cut
+   again. Empty parts are dropped.
+4. When a dimension is down to a single value (every listing in this window
+   costs exactly 42) uncapped moves on to the next dimension.
+5. Neighbouring windows that together still fit under the cap are merged into
+   one wider window, so you page through fewer of them. Around a dense spot
+   (2,500 listings at one price) this turns 27 windows into 6. Merged counts
+   stay exact: uncapped never merges across results it couldn't reach. Pass
+   `merge=False` to keep every window as it was split.
+6. Windows come out in order while the plan is still being worked out, so you
+   can start fetching before it's finished.
+
+## Dimensions
+
+Your `count` function receives a dict keyed by dimension name. A dimension
+that hasn't been needed yet is **left out**, so filter on it only when it's there.
+
+```python
+from datetime import datetime, timedelta
+from uncapped import Range, DateRange, Values
+
+Range("price", 0, 5_000_000)                # filters["price"] == (0, 2_499_999), inclusive
+Range("rating", 0, 5, step=0.1)             # floats work; step is the smallest gap the source sees
+DateRange("listed", "2023-01-01", "today")  # filters["listed"] == (date(...), date(...)), inclusive
+DateRange("posted", datetime(2024, 1, 1), "now", step=timedelta(minutes=1))
+Values("type", ["apartment", "villa", "office"])            # one value at a time: filters["type"] == "villa"
+Values("lang", ["python", "go", "rust"], multi=True)         # groups: filters["lang"] == ("python", "go")
+```
+
+Give several dimensions in order of preference. uncapped splits on the first
+until it can't, then the second:
+
+```python
+split(count, cap=1000, dims=[
+    Range("price", 0, 10_000_000),
+    DateRange("listed", "2020-01-01", "today"),
+])
+```
+
+## Knowing what you didn't get
+
+uncapped never fails silently. After a run, `run.stats` has the numbers and an
+`UncappedWarning` is raised for anything you'd miss:
+
+```python
+Stats(total=48213, windows=61, fetchable=47650, capped_windows=1,
+      unaccounted=313, queries=74, cache_hits=0, max_depth=3)
+```
+
+- **`capped_windows`**: windows still over the cap after every dimension is
+  down to one value. They're yielded with `window.complete == False`; you can
+  fetch the first `cap` results of each. Add another dimension or a finer
+  `step`. Use `on_capped="raise"` to stop instead, or `"ignore"` to stay quiet.
+- **`unaccounted`**: the parts of a split added up to less than the whole.
+  That's almost always items with **no value** for the filter (a listing
+  without a price never matches any price range) or values outside the range
+  you gave. Widen the range, or fetch those items another way.
+
+## Async
+
+`count` can be async. Sibling counts run concurrently, up to `concurrency`:
+
+```python
+import httpx
+from uncapped import asplit, Range
+
+async def count(filters):
+    lo, hi = filters.get("price", (None, None))
+    r = await client.get("/search", params={"min_price": lo, "max_price": hi, "limit": 1})
+    return r.json()["total"]
+
+async for window in asplit(count, cap=1000, dims=[Range("price", 0, 10**7)], concurrency=8):
+    ...
+```
+
+## Resuming and saving calls
+
+Pass any dict-like object as `cache` and counts are stored there. With a
+persistent one, a crashed or repeated run re-plans without calling the API
+again:
+
+```python
+import shelve
+
+with shelve.open("counts") as cache:
+    for window in split(count, cap=1000, dims=dims, cache=cache):
+        ...
+```
+
+Counts are cached by their filters and don't expire. Delete the file when you
+want fresh numbers.
+
+## Recipes
+
+**Elasticsearch / OpenSearch** (when you can't use `search_after` or a point in time):
+
+```python
+def count(filters):
+    query = {"bool": {"filter": []}}
+    if "ts" in filters:
+        lo, hi = filters["ts"]
+        query["bool"]["filter"].append({"range": {"ts": {"gte": lo.isoformat(), "lte": hi.isoformat()}}})
+    return es.count(index="events", query=query)["count"]
+
+split(count, cap=10_000, dims=[DateRange("ts", datetime(2024, 1, 1), "now", step=timedelta(seconds=1))])
+```
+
+**A listing site's JSON API** with min/max price parameters:
+
+```python
+def count(filters):
+    params = {"category": "cars", "page_size": 1}
+    if "price" in filters:
+        params["price_min"], params["price_max"] = filters["price"]
+    return session.get(SEARCH_URL, params=params).json()["meta"]["total"]
+```
+
+## FAQ
+
+**Why not just ask for page 11?** The cap is enforced by the server. Past it
+you get an error or the same results again.
+
+**My source's counts are estimates.** Fine: windows are still correct, but
+`unaccounted` may show small numbers from rounding. Leave some headroom by
+passing a slightly smaller `cap` (e.g. `950`).
+
+**Data changes while I fetch.** Windows are planned from the counts at that
+moment. New items in a window may push it a little over the cap; a smaller
+`cap` gives you headroom here too.
+
+**Is this allowed?** uncapped only decides which searches to run. Respect the
+terms and rate limits of whatever you query.
+
+## License
+
+MIT

uncapped-0.1.0.dist-info/RECORD

@@ -0,0 +1,8 @@
+uncapped/__init__.py,sha256=CSbZdWhJt1LDPSpy_6HnA711tBwWRykEfWO4YWU-DAw,963
+uncapped/core.py,sha256=SrZ__Tl9cbK08sZSTeOBbsRRURBC-6CJnXTbPII3Pfo,16854
+uncapped/dims.py,sha256=W6EOKsWf0fCPUgIp7PN_jFNMfpHVKeDin8o8Am_rveU,8204
+uncapped/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+uncapped-0.1.0.dist-info/METADATA,sha256=axCpZQdj_ZZh6d2Yv6-Yo3QNUMe-hPkkUBR6VeYNAmQ,9645
+uncapped-0.1.0.dist-info/WHEEL,sha256=2iZrJkDIkeDh0k9SikKMqFuRP-ua6mb4Ir1AsER4BEw,87
+uncapped-0.1.0.dist-info/licenses/LICENSE,sha256=J3OJcP_Nw_xrBQQoelguuzmLz5A52W4shQ1YC7uSwn0,1068
+uncapped-0.1.0.dist-info/RECORD,,

uncapped-0.1.0.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: hatchling 1.33.0
+Root-Is-Purelib: true
+Tag: py3-none-any

uncapped-0.1.0.dist-info/licenses/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Abdul Wahab
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.