chunk-memo 0.1.0__tar.gz

chunk_memo-0.1.0/.gitignore

@@ -0,0 +1,9 @@
+output/
+__pycache__/
+*.pyc
+dist/
+build/
+.pytest_cache/
+.ruff_cache/
+.venv/
+.session.vim

chunk_memo-0.1.0/CHANGELOG.md

@@ -0,0 +1,16 @@
+# Changelog
+
+## 0.2.0 (unreleased)
+- **New**: Support callable `axis_values` for memory-efficient lazy loading
+  - Axis values can now be provided as callables instead of lists
+  - Enables lazy loading from databases, files, or APIs
+  - Supports index-based pattern: `lambda idx: data[idx]`
+  - Supports list-returning pattern: `lambda: full_list`
+  - Maintains full backward compatibility with existing list/tuple axis_values
+  - Added `_get_all_axis_values()` helper for internal value resolution
+  - Added `_make_axis_values_serializable()` for JSON serialization
+  - Stores both runtime accessors (`self._axis_values`) and serializable representation (`self._axis_values_serializable`)
+  - See `examples/callable_axis_values.py` for usage examples
+
+## 0.1.0
+- Initial release.

chunk_memo-0.1.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Matthew Farrell
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

chunk_memo-0.1.0/PKG-INFO

@@ -0,0 +1,57 @@
+Metadata-Version: 2.4
+Name: chunk-memo
+Version: 0.1.0
+Summary: Memoization with chunked caching and parallelization.
+Project-URL: Repository, https://github.com/msf235/chunk-memo
+Author-email: Matthew Farrell <matthew.farrell.235@gmail.com>
+License: MIT License
+        
+        Copyright (c) 2026 Matthew Farrell
+        
+        Permission is hereby granted, free of charge, to any person obtaining a copy
+        of this software and associated documentation files (the "Software"), to deal
+        in the Software without restriction, including without limitation the rights
+        to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+        copies of the Software, and to permit persons to whom the Software is
+        furnished to do so, subject to the following conditions:
+        
+        The above copyright notice and this permission notice shall be included in all
+        copies or substantial portions of the Software.
+        
+        THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+        IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+        FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+        AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+        LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+        OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+        SOFTWARE.
+License-File: LICENSE
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3 :: Only
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Requires-Python: >=3.10
+Provides-Extra: bench
+Requires-Dist: joblib>=1.5.3; extra == 'bench'
+Provides-Extra: test
+Requires-Dist: flaky>=3.8.1; extra == 'test'
+Requires-Dist: pytest; extra == 'test'
+Description-Content-Type: text/markdown
+
+# chunk-memo
+
+Chunked memoization for grid-style parameter sweeps. Define a parameter grid, cache
+chunked outputs to disk, and reuse cached work across runs.
+
+## Install
+
+```bash
+pip install chunk-memo
+```
+
+## Links
+
+- Repository: https://github.com/msf235/chunk-memo

chunk_memo-0.1.0/README_PYPI.md

@@ -0,0 +1,14 @@
+# chunk-memo
+
+Chunked memoization for grid-style parameter sweeps. Define a parameter grid, cache
+chunked outputs to disk, and reuse cached work across runs.
+
+## Install
+
+```bash
+pip install chunk-memo
+```
+
+## Links
+
+- Repository: https://github.com/msf235/chunk-memo

chunk_memo-0.1.0/chunk_memo/__init__.py

@@ -0,0 +1,34 @@
+from importlib.metadata import PackageNotFoundError, version
+
+from .cache import ChunkCache
+from .memo import ChunkMemo
+from .identity import params_to_cache_id
+from .runner_protocol import CacheStatus, RunnerContext
+from .runners import (
+    Diagnostics,
+    run,
+    run_parallel,
+    run_streaming,
+)
+
+auto_load = ChunkMemo.auto_load
+
+
+try:
+    __version__ = version("chunk-memo")
+except PackageNotFoundError:
+    __version__ = "unknown"
+
+__all__ = [
+    "ChunkCache",
+    "ChunkMemo",
+    "Diagnostics",
+    "__version__",
+    "auto_load",
+    "run",
+    "run_parallel",
+    "run_streaming",
+    "RunnerContext",
+    "CacheStatus",
+    "params_to_cache_id",
+]

chunk_memo-0.1.0/chunk_memo/_format.py

@@ -0,0 +1,188 @@
+import math
+import threading
+import time
+from dataclasses import dataclass
+from typing import Any, Callable, Mapping, Sequence, Tuple
+
+ChunkKey = Tuple[Tuple[str, Tuple[Any, ...]], ...]
+
+_thread_local = threading.local()
+PROGRESS_REPORT_DIVISOR = 50
+
+
+@dataclass
+class _ProgressTracker:
+    last_len: int = 0
+    last_msg: str = ""
+
+    def print_progress(self, message: str, *, final: bool) -> None:
+        if final:
+            print(message, end="\n", flush=True)
+            self.last_len = 0
+            self.last_msg = ""
+            return
+        pad = max(self.last_len - len(message), 0)
+        print(message + (" " * pad), end="\r", flush=True)
+        self.last_len = len(message)
+        self.last_msg = message
+
+    def print_detail(self, message: str) -> None:
+        if self.last_len:
+            print()
+        print(message)
+        if self.last_msg:
+            print(self.last_msg, end="\r", flush=True)
+            self.last_len = len(self.last_msg)
+
+
+def format_axis_values(values: Any) -> str:
+    if isinstance(values, (list, tuple)):
+        if len(values) <= 4:
+            inner = ", ".join(repr(value) for value in values)
+            return f"[{inner}]"
+        head = ", ".join(repr(value) for value in values[:2])
+        tail = ", ".join(repr(value) for value in values[-2:])
+        return f"[{head}, ..., {tail}]"
+    return repr(values)
+
+
+def format_cache_id(cache_id: str | None) -> list[str]:
+    lines = ["[ChunkCache] cache_id:"]
+    if not cache_id:
+        lines.append("  (none)")
+        return lines
+    lines.append(f"  {cache_id}")
+    return lines
+
+
+def format_params(params: Mapping[str, Any]) -> list[str]:
+    lines = ["[ChunkCache] params:"]
+    if not params:
+        lines.append("  (none)")
+        return lines
+    for key in sorted(params):
+        value = params[key]
+        lines.append(f"  {key}={value!r}")
+    return lines
+
+
+def format_spec(axis_values: Mapping[str, Any], axis_order: Sequence[str]) -> list[str]:
+    lines = ["[ChunkCache] spec:"]
+    for axis in axis_order:
+        values = axis_values.get(axis)
+        lines.append(f"  {axis}={format_axis_values(values)}")
+    return lines
+
+
+def format_eta(seconds: float) -> str:
+    if seconds < 0 or not math.isfinite(seconds):
+        return "--:--:--"
+    total = int(seconds)
+    hours = total // 3600
+    minutes = (total % 3600) // 60
+    secs = total % 60
+    return f"{hours:02d}:{minutes:02d}:{secs:02d}"
+
+
+def format_rate_eta(
+    label: str,
+    processed: int,
+    total: int,
+    start_time: float,
+    *,
+    rate_processed: int | None = None,
+    rate_total: int | None = None,
+) -> str:
+    elapsed = max(time.monotonic() - start_time, 1e-6)
+    if rate_processed is None:
+        rate_processed = processed
+    if rate_total is None:
+        rate_total = total
+    rate = rate_processed / elapsed
+    percent = 100.0 if total <= 0 else (processed / total * 100.0)
+    remaining = 0 if rate_total <= 0 else max(rate_total - rate_processed, 0)
+    eta = remaining / rate if rate > 0 else float("inf")
+    return (
+        f"[ChunkCache] {label} {processed}/{total} "
+        f"({percent:0.1f}%) rate={rate:0.1f}/s ETA={format_eta(eta)}"
+    )
+
+
+def print_progress(message: str, *, final: bool) -> None:
+    if not hasattr(_thread_local, "tracker"):
+        _thread_local.tracker = _ProgressTracker()
+    _thread_local.tracker.print_progress(message, final=final)
+
+
+def print_detail(message: str) -> None:
+    if not hasattr(_thread_local, "tracker"):
+        _thread_local.tracker = _ProgressTracker()
+    _thread_local.tracker.print_detail(message)
+
+
+def chunk_key_size(chunk_key: ChunkKey) -> int:
+    if not chunk_key:
+        return 0
+    return math.prod(len(values) for _, values in chunk_key)
+
+
+def build_plan_lines(
+    cache_id: str | None,
+    params: Mapping[str, Any],
+    axis_values: Mapping[str, Any],
+    axis_order: Sequence[str],
+    cached_count: int,
+    execute_count: int,
+) -> list[str]:
+    lines: list[str] = []
+    lines.extend(format_cache_id(cache_id))
+    lines.extend(format_params(params))
+    lines.extend(format_spec(axis_values, axis_order))
+    lines.append(f"[ChunkCache] plan: cached={cached_count} execute={execute_count}")
+    return lines
+
+
+def prepare_progress(
+    *,
+    total_chunks: int,
+    total_items: int,
+    verbose: int,
+    label: str = "planning",
+) -> tuple[Callable[[int, bool], None], Callable[[int], None]]:
+    progress_step = max(1, total_chunks // PROGRESS_REPORT_DIVISOR)
+    start_time = time.monotonic()
+    processed_items = 0
+
+    def update_processed(count: int) -> None:
+        nonlocal processed_items
+        processed_items += count
+
+    def report_progress(processed: int, final: bool = False) -> None:
+        if verbose != 1:
+            return
+        if not final and processed % progress_step != 0 and processed != total_chunks:
+            return
+        message = format_rate_eta(
+            label,
+            processed_items,
+            total_items,
+            start_time,
+        )
+        print_progress(message, final=final)
+
+    return report_progress, update_processed
+
+
+def print_chunk_summary(
+    diagnostics: Any,
+    verbose: int,
+) -> None:
+    if verbose >= 2:
+        partial_chunks = getattr(diagnostics, "partial_chunks", 0)
+        partial_suffix = f" partial={partial_chunks}" if partial_chunks else ""
+        print_detail(
+            "[ChunkCache] summary "
+            f"cached={diagnostics.cached_chunks} "
+            f"executed={diagnostics.executed_chunks} "
+            f"total={diagnostics.total_chunks}" + partial_suffix
+        )