task-checkpoint 0.1.0__py3-none-any.whl

task_checkpoint-0.1.0.dist-info/METADATA

@@ -0,0 +1,242 @@
+Metadata-Version: 2.4
+Name: task-checkpoint
+Version: 0.1.0
+Summary: A small Python library for resumable task pools in long-running scripts.
+Author: 1nvisibleCat
+License-Expression: MIT
+Keywords: checkpoint,task-checkpoint,task-pool,resume,batch-processing,job-queue
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Developers
+Classifier: Operating System :: POSIX
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3 :: Only
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Topic :: Software Development :: Libraries :: Python Modules
+Classifier: Topic :: Utilities
+Requires-Python: >=3.9
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Dynamic: license-file
+
+# Task Pool
+
+Task Pool is a small Python library for long-running scripts that need to process many small jobs without losing progress.
+
+Turn a fragile `for` loop into a resumable task pool.
+
+When a run stops halfway, Task Pool remembers what is done, what is running, and what still needs work. You can restart the script and continue from the last known point.
+
+Use it for crawling, experiments, data analysis, batch API calls, file processing, and other work where unfinished items should be easy to resume.
+
+It feels like a tiny job queue, but stays inside your Python script. For local use, no database server, message queue, or service setup is needed.
+
+## Quickstart
+
+Install it with:
+
+```bash
+pip install task-checkpoint
+```
+
+Put one URL per line:
+
+```text
+https://example.com/a
+https://example.com/b
+https://example.com/c
+```
+
+Then process the file as a task pool:
+
+```python
+from task_pool import TaskPool
+
+
+pool = TaskPool("urls.txt")
+
+def fetch_one(payload):
+    fetch_url(payload["line"])
+
+
+pool.for_each(fetch_one)
+```
+
+You can also use a regular loop:
+
+```python
+for payload in pool:
+    fetch_url(payload["line"])
+```
+
+If you need task metadata, use `iter_tasks()`:
+
+```python
+for task in pool.iter_tasks():
+    url = task.payload["line"]
+    result = fetch_url(url)
+    save_result(task.key, result)
+```
+
+Each completed task is committed. If an exception is raised, the current task is rolled back to `not_start` and the exception is raised again.
+
+For very small scripts, a lambda also works:
+
+```python
+pool.for_each(lambda payload: fetch_url(payload["line"]))
+```
+
+You can print a small progress summary:
+
+```python
+print(pool.stats())
+# {"total": 10, "not_start": 3, "pending": 0, "committed": 7}
+```
+
+For structured jobs, use a JSON task pool:
+
+```python
+pool = TaskPool("tasks.json")
+
+pool.append({"input_path": "data/a.json", "method": "baseline"})
+pool.append({"input_path": "data/b.json", "method": "baseline"})
+```
+
+## Manual Control
+
+Use `lease()` when you want one task at a time and need explicit control.
+
+```python
+pool = TaskPool("tasks.json")
+
+with pool.lease() as task:
+    if task is None:
+        return
+
+    do_work(task.payload)
+```
+
+If there is no task to process, `lease()` returns `None`.
+
+Leaving the `with` block normally commits the task. If an exception is raised, the task is rolled back automatically.
+
+## Store Selection
+
+You can choose a store by file suffix:
+
+```python
+TaskPool("tasks.json")      # JSONFileStore
+TaskPool("tasks.sqlite")    # SQLiteStore
+TaskPool("tasks.sqlite3")   # SQLiteStore
+TaskPool("tasks.db")        # SQLiteStore
+TaskPool("rows.csv")        # CSVRowStore with sidecar state
+TaskPool("rows.tsv")        # CSVRowStore with sidecar state
+TaskPool("urls.txt")        # TextLineStore with sidecar state
+```
+
+You can also pass a store explicitly:
+
+```python
+from task_pool import JSONFileStore, SQLiteStore, TaskPool
+
+
+pool = TaskPool(JSONFileStore("tasks.json"))
+pool = TaskPool(SQLiteStore("tasks.db"))
+```
+
+## Source Files
+
+Task Pool can also read tasks from source files.
+
+### Text files
+
+```python
+pool = TaskPool("urls.txt")
+```
+
+Example:
+
+```text
+https://example.com/a
+https://example.com/b
+https://example.com/c
+```
+
+Each non-empty line becomes a task:
+
+```python
+{
+    "line": "https://example.com/a",
+    "line_number": 1,
+}
+```
+
+### CSV and TSV files
+
+```python
+pool = TaskPool("rows.csv")
+pool = TaskPool("rows.tsv")
+```
+
+By default, CSV and TSV files are read with a header row:
+
+```csv
+code,name
+A0001,Alpha
+A0002,Beta
+```
+
+The first row becomes:
+
+```python
+{"code": "A0001", "name": "Alpha"}
+```
+
+For files without headers:
+
+```python
+pool = TaskPool.csv("rows.csv", has_header=False)
+pool = TaskPool.tsv("rows.tsv", has_header=False)
+```
+
+Rows are named `col1`, `col2`, and so on:
+
+```python
+{"col1": "A0001", "col2": "Alpha"}
+```
+
+Source files are not modified. Progress is stored in a sidecar JSON file, for example:
+
+```text
+rows.csv.task_pool.json
+urls.txt.task_pool.json
+```
+
+## Task Status
+
+Each task has one of these statuses:
+
+- `not_start`
+- `pending`
+- `committed`
+
+The common flow is:
+
+```text
+not_start -> pending -> committed
+                  |
+                  -> not_start
+```
+
+`pending` means a worker has leased the task and is currently processing it.
+
+## Notes
+
+- The default store is `JSONFileStore("task_pool.json")`.
+- Payloads are stored as JSON.
+- JSON and source-backed stores use file locks and atomic writes.
+- SQLite is better for larger pools or heavier concurrent use.
+- Source-backed stores keep the source file unchanged and write progress to a sidecar JSON file.

task_checkpoint-0.1.0.dist-info/RECORD

@@ -0,0 +1,16 @@
+task_checkpoint-0.1.0.dist-info/licenses/LICENSE,sha256=dU9CUn2-fRJIT0nzCRHsH7NqYT_dVawztFs6eqUCvQY,1069
+task_pool/__init__.py,sha256=F2UIz6jdPhxhyagCidX1z7k7u5lgPa_pvbJ_EsEM1SE,525
+task_pool/codecs.py,sha256=M8eUaYTNXx6Axr_o0L6EAgZpT3ZwZhtt86-R-EFjRAQ,504
+task_pool/core.py,sha256=EbyeglC2hdvEg-NIpS6iSJwBvir9wGCoIT1-2tVggs0,9224
+task_pool/models.py,sha256=sZu2qRnM6lh6EalA0FcQ_fuF3a8FCidru8_J4nsNw4o,946
+task_pool/stores/__init__.py,sha256=vDk1XQv2FDvMO8-Gs0qnMAySw0ovtP6pjkDiSnvLYmE,361
+task_pool/stores/base.py,sha256=5u3o0wAAVZk63EAy4guFg8dN0iGMizUmdx8Rp8ifCJc,761
+task_pool/stores/csv_file.py,sha256=SSMWmPWyDWGh8UXW33KpNJMZMD5EHxpdXwERLixaejI,1421
+task_pool/stores/json_file.py,sha256=S7Fy9SULT_xMxL_qbfPc39bD21S-5YAj2j0uZtWM89w,6242
+task_pool/stores/source_backed.py,sha256=nOTxksL_UZ5y-c8WLka1Z6BxV2m3Nf5T6dx-xbqjFUk,2194
+task_pool/stores/sqlite.py,sha256=jLoelduviPpn9QJbmVlYFqqJXrn-t8LW2GcPwA1MVCw,8121
+task_pool/stores/text_file.py,sha256=RUrjqU0J8SHKkhG0lu__oMqQW5b3lzxOIHqmchEkR5k,986
+task_checkpoint-0.1.0.dist-info/METADATA,sha256=yh6IFydH7JWGz0_Hw4peLd2n0yFm5OxUqx32koLieP4,5549
+task_checkpoint-0.1.0.dist-info/WHEEL,sha256=aeYiig01lYGDzBgS8HxWXOg3uV61G9ijOsup-k9o1sk,91
+task_checkpoint-0.1.0.dist-info/top_level.txt,sha256=DSMZD6PdiuVVyLiLFVGARWMEXOgBjhre5A9YdxwQO3s,10
+task_checkpoint-0.1.0.dist-info/RECORD,,

task_checkpoint-0.1.0.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (82.0.1)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

task_checkpoint-0.1.0.dist-info/licenses/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 1nvisibleCat
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

task_checkpoint-0.1.0.dist-info/top_level.txt

@@ -0,0 +1 @@
+task_pool

task_pool/__init__.py

@@ -0,0 +1,28 @@
+from task_pool.codecs import JsonPayloadCodec
+from task_pool.core import LeasedTask, TaskPool
+from task_pool.models import Status, Task, TaskRecord
+from task_pool.stores import (
+    CSVRowStore,
+    JSONFileStore,
+    SQLiteStore,
+    TaskStore,
+    TextLineStore,
+)
+
+
+__version__ = "0.1.0"
+
+__all__ = [
+    "CSVRowStore",
+    "JSONFileStore",
+    "JsonPayloadCodec",
+    "LeasedTask",
+    "SQLiteStore",
+    "Status",
+    "Task",
+    "TaskPool",
+    "TaskRecord",
+    "TaskStore",
+    "TextLineStore",
+    "__version__",
+]

task_pool/codecs.py

@@ -0,0 +1,18 @@
+import json
+from typing import Any
+
+
+class JsonPayloadCodec:
+    def encode(self, payload: Any) -> str:
+        try:
+            return json.dumps(
+                payload,
+                sort_keys=True,
+                separators=(",", ":"),
+                ensure_ascii=False,
+            )
+        except (TypeError, ValueError) as exc:
+            raise TypeError("payload must be JSON-serializable") from exc
+
+    def decode(self, payload_data: str) -> Any:
+        return json.loads(payload_data)

task_pool/core.py

@@ -0,0 +1,339 @@
+import asyncio
+import hashlib
+import os
+import time
+import uuid
+from typing import Any, Callable, Optional
+
+from task_pool.codecs import JsonPayloadCodec
+from task_pool.models import Status, Task, TaskRecord
+from task_pool.stores.base import TaskStore
+from task_pool.stores.json_file import JSONFileStore
+
+
+class LeasedTask:
+    def __init__(self, pool, task: Task):
+        self._pool = pool
+        self._task = task
+        self._closed = False
+
+    @property
+    def key(self):
+        return self._task.key
+
+    @property
+    def payload(self):
+        return self._task.payload
+
+    @property
+    def status(self):
+        return self._task.status
+
+    @property
+    def closed(self):
+        return self._closed
+
+    def commit(self):
+        self._pool.commit(self.key)
+        self._closed = True
+
+    def rollback(self):
+        self._pool.rollback(self.key)
+        self._closed = True
+
+    def to_task(self):
+        return self._task
+
+
+class _LeaseContext:
+    def __init__(self, pool):
+        self.pool = pool
+        self.task = None
+
+    def __enter__(self):
+        task = self.pool.acquire()
+        if task is None:
+            return None
+        self.task = LeasedTask(self.pool, task)
+        return self.task
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        if self.task is None or self.task.closed:
+            return False
+        if exc_type is None:
+            self.task.commit()
+        else:
+            self.task.rollback()
+        return False
+
+
+class _AsyncLeaseContext:
+    def __init__(self, pool):
+        self.pool = pool
+        self.task = None
+
+    async def __aenter__(self):
+        task = await asyncio.to_thread(self.pool.acquire)
+        if task is None:
+            return None
+        self.task = LeasedTask(self.pool, task)
+        return self.task
+
+    async def __aexit__(self, exc_type, exc_val, exc_tb):
+        if self.task is None or self.task.closed:
+            return False
+        if exc_type is None:
+            await asyncio.to_thread(self.task.commit)
+        else:
+            await asyncio.to_thread(self.task.rollback)
+        return False
+
+
+class TaskPool:
+    def __init__(
+        self,
+        store: Optional[TaskStore] = None,
+        *,
+        unique_payload=True,
+        lease_timeout_seconds=None,
+        payload_codec=None,
+    ):
+        if lease_timeout_seconds is not None and lease_timeout_seconds < 0:
+            raise ValueError("lease_timeout_seconds must be None or >= 0")
+
+        if store is None:
+            self.store = JSONFileStore()
+        elif isinstance(store, (str, os.PathLike)):
+            self.store = self._store_from_path(store)
+        else:
+            self.store = store
+        self.unique_payload = unique_payload
+        self.lease_timeout_seconds = lease_timeout_seconds
+        self.payload_codec = payload_codec or JsonPayloadCodec()
+
+    @classmethod
+    def json(cls, path="task_pool.json", **kwargs):
+        return cls(JSONFileStore(path), **kwargs)
+
+    @classmethod
+    def sqlite(cls, path="task_pool.db", **kwargs):
+        from task_pool.stores.sqlite import SQLiteStore
+
+        return cls(SQLiteStore(path), **kwargs)
+
+    @classmethod
+    def csv(
+        cls,
+        path,
+        *,
+        has_header=True,
+        state_path=None,
+        encoding="utf-8",
+        **kwargs,
+    ):
+        from task_pool.stores.csv_file import CSVRowStore
+
+        return cls(
+            CSVRowStore(
+                path,
+                has_header=has_header,
+                state_path=state_path,
+                encoding=encoding,
+            ),
+            **kwargs,
+        )
+
+    @classmethod
+    def tsv(
+        cls,
+        path,
+        *,
+        has_header=True,
+        state_path=None,
+        encoding="utf-8",
+        **kwargs,
+    ):
+        from task_pool.stores.csv_file import CSVRowStore
+
+        return cls(
+            CSVRowStore(
+                path,
+                delimiter="\t",
+                has_header=has_header,
+                state_path=state_path,
+                encoding=encoding,
+            ),
+            **kwargs,
+        )
+
+    @classmethod
+    def text(
+        cls,
+        path,
+        *,
+        state_path=None,
+        encoding="utf-8",
+        skip_blank=True,
+        **kwargs,
+    ):
+        from task_pool.stores.text_file import TextLineStore
+
+        return cls(
+            TextLineStore(
+                path,
+                state_path=state_path,
+                encoding=encoding,
+                skip_blank=skip_blank,
+            ),
+            **kwargs,
+        )
+
+    def append(self, payload, key=None, override=False, unique_payload=None):
+        if unique_payload is None:
+            unique_payload = self.unique_payload
+
+        payload_data = self.payload_codec.encode(payload)
+        if key is None:
+            key = self._make_key(payload_data, unique_payload)
+
+        now = self._now()
+        record = TaskRecord(
+            key=key,
+            payload_data=payload_data,
+            status=Status.NOT_START,
+            created_at=now,
+            updated_at=now,
+            leased_at=None,
+        )
+        return self.store.add(record, override=override)
+
+    def acquire(self):
+        if self.lease_timeout_seconds is not None:
+            self.reset_stale(self.lease_timeout_seconds)
+        record = self.store.claim_next(now=self._now())
+        if record is None:
+            return None
+        return self._task_from_record(record)
+
+    def lease(self):
+        return _LeaseContext(self)
+
+    def alease(self):
+        return _AsyncLeaseContext(self)
+
+    def __iter__(self):
+        for task in self.iter_tasks():
+            yield task.payload
+
+    def iter_tasks(self):
+        while True:
+            with self.lease() as task:
+                if task is None:
+                    break
+                yield task
+
+    def commit(self, key):
+        self.store.mark_done(key, now=self._now())
+
+    def rollback(self, key):
+        self.store.rollback(key, now=self._now())
+
+    def reset(self):
+        return self.store.reset(now=self._now())
+
+    def reset_stale(self, age_seconds):
+        if age_seconds < 0:
+            raise ValueError("age_seconds must be >= 0")
+        now = self._now()
+        return self.store.reset_stale(cutoff=now - age_seconds, now=now)
+
+    def count(self, status=None):
+        return self.store.count(self._status_value(status))
+
+    def stats(self):
+        counts = {status.value: self.count(status) for status in Status}
+        return {"total": self.count(), **counts}
+
+    def get_all(self, status=None):
+        return [
+            self._task_from_record(record)
+            for record in self.store.list(self._status_value(status))
+        ]
+
+    def for_each(self, action: Callable[[Any], Any]):
+        while True:
+            with self.lease() as task:
+                if task is None:
+                    break
+                action(task.payload)
+
+    def map(self, action: Callable[[Any], Any]):
+        results = []
+        while True:
+            with self.lease() as task:
+                if task is None:
+                    break
+                results.append(action(task.payload))
+        return results
+
+    def _task_from_record(self, record):
+        return Task(
+            key=record.key,
+            payload=self.payload_codec.decode(record.payload_data),
+            status=record.status,
+            created_at=record.created_at,
+            updated_at=record.updated_at,
+            leased_at=record.leased_at,
+        )
+
+    @staticmethod
+    def _make_key(payload_data, unique_payload):
+        seed = payload_data
+        if not unique_payload:
+            seed = f"{payload_data}:{uuid.uuid4().hex}"
+        return hashlib.sha256(seed.encode("utf-8")).hexdigest()[:12]
+
+    @staticmethod
+    def _store_from_path(path):
+        suffix = os.fspath(path).lower().rsplit(".", 1)
+        if len(suffix) == 1:
+            raise ValueError("store path must have a supported suffix")
+        suffix = "." + suffix[1]
+
+        if suffix == ".json":
+            return JSONFileStore(path)
+        if suffix in {".sqlite", ".sqlite3", ".db"}:
+            from task_pool.stores.sqlite import SQLiteStore
+
+            return SQLiteStore(path)
+        if suffix == ".csv":
+            from task_pool.stores.csv_file import CSVRowStore
+
+            return CSVRowStore(path)
+        if suffix == ".tsv":
+            from task_pool.stores.csv_file import CSVRowStore
+
+            return CSVRowStore(path, delimiter="\t")
+        if suffix == ".txt":
+            from task_pool.stores.text_file import TextLineStore
+
+            return TextLineStore(path)
+        raise ValueError(f"unsupported store suffix {suffix}")
+
+    @staticmethod
+    def _status_value(status):
+        if status is None:
+            return None
+        if isinstance(status, Status):
+            return status
+        if isinstance(status, str):
+            if status in Status._value2member_map_:
+                return Status(status)
+            name = status.upper()
+            if name in Status.__members__:
+                return Status[name]
+        raise ValueError(f"invalid status {status}")
+
+    @staticmethod
+    def _now():
+        return time.time()

task_pool/models.py

@@ -0,0 +1,45 @@
+from dataclasses import dataclass
+from enum import Enum
+from typing import Any, Optional
+
+
+class Status(str, Enum):
+    NOT_START = "not_start"
+    PENDING = "pending"
+    COMMITTED = "committed"
+
+    def to_dict(self):
+        return self.value
+
+    def __str__(self):
+        return self.value
+
+
+@dataclass(frozen=True)
+class TaskRecord:
+    key: str
+    payload_data: str
+    status: Status
+    created_at: float = 0
+    updated_at: float = 0
+    leased_at: Optional[float] = None
+
+
+@dataclass(frozen=True)
+class Task:
+    key: str
+    payload: Any
+    status: Status
+    created_at: float = 0
+    updated_at: float = 0
+    leased_at: Optional[float] = None
+
+    def to_dict(self):
+        return {
+            "key": self.key,
+            "payload": self.payload,
+            "status": self.status.value,
+            "created_at": self.created_at,
+            "updated_at": self.updated_at,
+            "leased_at": self.leased_at,
+        }

task_pool/stores/__init__.py

@@ -0,0 +1,14 @@
+from task_pool.stores.base import TaskStore
+from task_pool.stores.csv_file import CSVRowStore
+from task_pool.stores.json_file import JSONFileStore
+from task_pool.stores.sqlite import SQLiteStore
+from task_pool.stores.text_file import TextLineStore
+
+
+__all__ = [
+    "TaskStore",
+    "CSVRowStore",
+    "JSONFileStore",
+    "SQLiteStore",
+    "TextLineStore",
+]

task_pool/stores/base.py

@@ -0,0 +1,32 @@
+from typing import Optional, Protocol
+
+from task_pool.models import Status, TaskRecord
+
+
+class TaskStore(Protocol):
+    def add(self, record: TaskRecord, *, override: bool) -> str:
+        ...
+
+    def claim_next(self, *, now: float) -> Optional[TaskRecord]:
+        ...
+
+    def mark_done(self, key: str, *, now: float) -> None:
+        ...
+
+    def rollback(self, key: str, *, now: float) -> None:
+        ...
+
+    def reset(self, *, now: float) -> int:
+        ...
+
+    def reset_stale(self, *, cutoff: float, now: float) -> int:
+        ...
+
+    def get(self, key: str) -> TaskRecord:
+        ...
+
+    def count(self, status: Optional[Status] = None) -> int:
+        ...
+
+    def list(self, status: Optional[Status] = None) -> list[TaskRecord]:
+        ...

task_pool/stores/csv_file.py

@@ -0,0 +1,46 @@
+import csv
+import os
+
+from task_pool.stores.source_backed import SourceBackedStore
+
+
+class CSVRowStore(SourceBackedStore):
+    def __init__(
+        self,
+        path,
+        *,
+        delimiter=",",
+        has_header=True,
+        state_path=None,
+        encoding="utf-8",
+    ):
+        super().__init__(path, state_path=state_path)
+        self.delimiter = delimiter
+        self.has_header = has_header
+        self.encoding = encoding
+        self._import_rows()
+
+    def _import_rows(self):
+        self._import_payloads(self._read_rows())
+
+    def _read_rows(self):
+        if not os.path.exists(self.path):
+            raise FileNotFoundError(self.path)
+        with open(self.path, "r", newline="", encoding=self.encoding) as f:
+            if self.has_header:
+                reader = csv.DictReader(f, delimiter=self.delimiter)
+                if reader.fieldnames is None:
+                    return
+                for row_number, row in enumerate(reader, start=1):
+                    yield row_number, dict(row)
+            else:
+                reader = csv.reader(f, delimiter=self.delimiter)
+                for row_number, row in enumerate(reader, start=1):
+                    yield row_number, self._unnamed_row_to_dict(row)
+
+    @staticmethod
+    def _unnamed_row_to_dict(row):
+        return {
+            f"col{index}": value
+            for index, value in enumerate(row, start=1)
+        }

task_pool/stores/json_file.py

@@ -0,0 +1,179 @@
+import json
+import os
+import tempfile
+from contextlib import contextmanager
+from typing import Optional
+
+import fcntl
+
+from task_pool.models import Status, TaskRecord
+
+
+class JSONFileStore:
+    def __init__(self, path="task_pool.json"):
+        self.path = str(path)
+        self.lock_path = f"{self.path}.lock"
+        if not os.path.exists(self.path):
+            self._write_data({})
+
+    def add(self, record: TaskRecord, *, override: bool) -> str:
+        with self._locked_data() as data:
+            if override or record.key not in data:
+                data[record.key] = self._record_to_dict(record)
+        return record.key
+
+    def claim_next(self, *, now: float) -> Optional[TaskRecord]:
+        with self._locked_data() as data:
+            for key in sorted(
+                data,
+                key=lambda item: (data[item]["created_at"], item),
+            ):
+                item = data[key]
+                if item["status"] == Status.NOT_START.value:
+                    item["status"] = Status.PENDING.value
+                    item["leased_at"] = now
+                    item["updated_at"] = now
+                    return self._dict_to_record(key, item)
+        return None
+
+    def mark_done(self, key: str, *, now: float) -> None:
+        with self._locked_data() as data:
+            item = self._get_item(data, key)
+            self._require_status(item, key, Status.PENDING)
+            item["status"] = Status.COMMITTED.value
+            item["updated_at"] = now
+            item["leased_at"] = None
+
+    def rollback(self, key: str, *, now: float) -> None:
+        with self._locked_data() as data:
+            item = self._get_item(data, key)
+            item["status"] = Status.NOT_START.value
+            item["updated_at"] = now
+            item["leased_at"] = None
+
+    def reset(self, *, now: float) -> int:
+        with self._locked_data() as data:
+            for item in data.values():
+                item["status"] = Status.NOT_START.value
+                item["updated_at"] = now
+                item["leased_at"] = None
+            return len(data)
+
+    def reset_stale(self, *, cutoff: float, now: float) -> int:
+        count = 0
+        with self._locked_data() as data:
+            for item in data.values():
+                if (
+                    item["status"] == Status.PENDING.value
+                    and item["leased_at"] is not None
+                    and item["leased_at"] <= cutoff
+                ):
+                    item["status"] = Status.NOT_START.value
+                    item["updated_at"] = now
+                    item["leased_at"] = None
+                    count += 1
+        return count
+
+    def get(self, key: str) -> TaskRecord:
+        with self._locked_data(write=False) as data:
+            return self._dict_to_record(key, self._get_item(data, key))
+
+    def count(self, status: Optional[Status] = None) -> int:
+        with self._locked_data(write=False) as data:
+            if status is None:
+                return len(data)
+            return sum(1 for item in data.values() if item["status"] == status.value)
+
+    def list(self, status: Optional[Status] = None) -> list[TaskRecord]:
+        with self._locked_data(write=False) as data:
+            records = []
+            for key in sorted(data, key=lambda item: (data[item]["created_at"], item)):
+                item = data[key]
+                if status is None or item["status"] == status.value:
+                    records.append(self._dict_to_record(key, item))
+            return records
+
+    @contextmanager
+    def _locked_data(self, write=True):
+        os.makedirs(os.path.dirname(os.path.abspath(self.path)), exist_ok=True)
+        with open(self.lock_path, "a+") as lock:
+            fcntl.lockf(lock, fcntl.LOCK_EX)
+            data = self._read_data()
+            try:
+                yield data
+            finally:
+                if write:
+                    self._write_data(data)
+                fcntl.lockf(lock, fcntl.LOCK_UN)
+
+    def _read_data(self):
+        if not os.path.exists(self.path):
+            return {}
+        with open(self.path, "r", encoding="utf-8") as f:
+            return json.load(f)
+
+    def _write_data(self, data):
+        directory = os.path.dirname(os.path.abspath(self.path))
+        os.makedirs(directory, exist_ok=True)
+        fd, tmp_path = tempfile.mkstemp(
+            prefix=".task_pool.",
+            suffix=".tmp",
+            dir=directory,
+            text=True,
+        )
+        try:
+            with os.fdopen(fd, "w", encoding="utf-8") as f:
+                json.dump(data, f, ensure_ascii=False, sort_keys=True)
+                f.flush()
+                os.fsync(f.fileno())
+            os.replace(tmp_path, self.path)
+        except Exception:
+            try:
+                os.unlink(tmp_path)
+            except FileNotFoundError:
+                pass
+            raise
+
+    @staticmethod
+    def _record_to_dict(record):
+        return {
+            "payload_data": record.payload_data,
+            "payload": json.loads(record.payload_data),
+            "status": record.status.value,
+            "created_at": record.created_at,
+            "updated_at": record.updated_at,
+            "leased_at": record.leased_at,
+        }
+
+    @staticmethod
+    def _dict_to_record(key, item):
+        payload_data = item.get("payload_data")
+        if payload_data is None:
+            payload_data = json.dumps(
+                item["payload"],
+                sort_keys=True,
+                separators=(",", ":"),
+                ensure_ascii=False,
+            )
+        return TaskRecord(
+            key=key,
+            payload_data=payload_data,
+            status=Status(item["status"]),
+            created_at=item.get("created_at", 0),
+            updated_at=item.get("updated_at", 0),
+            leased_at=item.get("leased_at"),
+        )
+
+    @staticmethod
+    def _get_item(data, key):
+        try:
+            return data[key]
+        except KeyError:
+            raise KeyError(key) from None
+
+    @staticmethod
+    def _require_status(item, key, status):
+        if item["status"] != status.value:
+            raise ValueError(
+                f"task {key} is {item['status']}, expected {status.value}"
+            )

task_pool/stores/source_backed.py

@@ -0,0 +1,65 @@
+import hashlib
+import json
+import time
+
+from task_pool.models import Status, TaskRecord
+from task_pool.stores.json_file import JSONFileStore
+
+
+class SourceBackedStore:
+    def __init__(self, path, *, state_path=None):
+        self.path = str(path)
+        self.state_path = state_path or f"{self.path}.task_pool.json"
+        self.state_store = JSONFileStore(self.state_path)
+
+    def add(self, record: TaskRecord, *, override: bool) -> str:
+        return self.state_store.add(record, override=override)
+
+    def claim_next(self, *, now: float):
+        return self.state_store.claim_next(now=now)
+
+    def mark_done(self, key: str, *, now: float) -> None:
+        self.state_store.mark_done(key, now=now)
+
+    def rollback(self, key: str, *, now: float) -> None:
+        self.state_store.rollback(key, now=now)
+
+    def reset(self, *, now: float) -> int:
+        return self.state_store.reset(now=now)
+
+    def reset_stale(self, *, cutoff: float, now: float) -> int:
+        return self.state_store.reset_stale(cutoff=cutoff, now=now)
+
+    def get(self, key: str) -> TaskRecord:
+        return self.state_store.get(key)
+
+    def count(self, status=None) -> int:
+        return self.state_store.count(status)
+
+    def list(self, status=None) -> list[TaskRecord]:
+        return self.state_store.list(status)
+
+    def _import_payloads(self, payloads):
+        now = time.time()
+        for row_number, payload in payloads:
+            payload_data = json.dumps(
+                payload,
+                sort_keys=True,
+                separators=(",", ":"),
+                ensure_ascii=False,
+            )
+            key = self._make_source_key(row_number, payload_data)
+            record = TaskRecord(
+                key=key,
+                payload_data=payload_data,
+                status=Status.NOT_START,
+                created_at=now + row_number / 1_000_000,
+                updated_at=now,
+                leased_at=None,
+            )
+            self.state_store.add(record, override=False)
+
+    @staticmethod
+    def _make_source_key(row_number, payload_data):
+        seed = f"{row_number}:{payload_data}"
+        return hashlib.sha256(seed.encode("utf-8")).hexdigest()[:12]

task_pool/stores/sqlite.py

@@ -0,0 +1,246 @@
+import sqlite3
+from contextlib import contextmanager
+from typing import Optional
+
+from task_pool.models import Status, TaskRecord
+
+
+class SQLiteStore:
+    def __init__(self, path="task_pool.db"):
+        self.path = str(path)
+        self._init_db()
+
+    def add(self, record: TaskRecord, *, override: bool) -> str:
+        with self._connection() as conn:
+            conn.execute("BEGIN IMMEDIATE")
+            if override:
+                conn.execute(
+                    """
+                    INSERT INTO tasks (
+                        key, payload_data, status, created_at,
+                        updated_at, leased_at
+                    )
+                    VALUES (?, ?, ?, ?, ?, NULL)
+                    ON CONFLICT(key) DO UPDATE SET
+                        payload_data = excluded.payload_data,
+                        status = excluded.status,
+                        updated_at = excluded.updated_at,
+                        leased_at = NULL
+                    """,
+                    (
+                        record.key,
+                        record.payload_data,
+                        record.status.value,
+                        record.created_at,
+                        record.updated_at,
+                    ),
+                )
+            else:
+                conn.execute(
+                    """
+                    INSERT OR IGNORE INTO tasks (
+                        key, payload_data, status, created_at,
+                        updated_at, leased_at
+                    )
+                    VALUES (?, ?, ?, ?, ?, NULL)
+                    """,
+                    (
+                        record.key,
+                        record.payload_data,
+                        record.status.value,
+                        record.created_at,
+                        record.updated_at,
+                    ),
+                )
+            conn.commit()
+        return record.key
+
+    def claim_next(self, *, now: float) -> Optional[TaskRecord]:
+        with self._connection() as conn:
+            conn.execute("BEGIN IMMEDIATE")
+            row = conn.execute(
+                """
+                SELECT * FROM tasks
+                WHERE status = ?
+                ORDER BY created_at, key
+                LIMIT 1
+                """,
+                (Status.NOT_START.value,),
+            ).fetchone()
+            if row is None:
+                conn.commit()
+                return None
+
+            conn.execute(
+                """
+                UPDATE tasks
+                SET status = ?, leased_at = ?, updated_at = ?
+                WHERE key = ?
+                """,
+                (Status.PENDING.value, now, now, row["key"]),
+            )
+            updated = conn.execute(
+                "SELECT * FROM tasks WHERE key = ?",
+                (row["key"],),
+            ).fetchone()
+            conn.commit()
+        return self._row_to_record(updated)
+
+    def mark_done(self, key: str, *, now: float) -> None:
+        with self._connection() as conn:
+            conn.execute("BEGIN IMMEDIATE")
+            row = self._ensure_exists(conn, key)
+            self._require_status(row, key, Status.PENDING)
+            conn.execute(
+                """
+                UPDATE tasks
+                SET status = ?, updated_at = ?, leased_at = NULL
+                WHERE key = ?
+                """,
+                (Status.COMMITTED.value, now, key),
+            )
+            conn.commit()
+
+    def rollback(self, key: str, *, now: float) -> None:
+        with self._connection() as conn:
+            conn.execute("BEGIN IMMEDIATE")
+            self._ensure_exists(conn, key)
+            conn.execute(
+                """
+                UPDATE tasks
+                SET status = ?, updated_at = ?, leased_at = NULL
+                WHERE key = ?
+                """,
+                (Status.NOT_START.value, now, key),
+            )
+            conn.commit()
+
+    def reset(self, *, now: float) -> int:
+        with self._connection() as conn:
+            conn.execute("BEGIN IMMEDIATE")
+            cur = conn.execute(
+                """
+                UPDATE tasks
+                SET status = ?, updated_at = ?, leased_at = NULL
+                """,
+                (Status.NOT_START.value, now),
+            )
+            conn.commit()
+        return cur.rowcount
+
+    def reset_stale(self, *, cutoff: float, now: float) -> int:
+        with self._connection() as conn:
+            conn.execute("BEGIN IMMEDIATE")
+            cur = conn.execute(
+                """
+                UPDATE tasks
+                SET status = ?, updated_at = ?, leased_at = NULL
+                WHERE status = ? AND leased_at IS NOT NULL AND leased_at <= ?
+                """,
+                (
+                    Status.NOT_START.value,
+                    now,
+                    Status.PENDING.value,
+                    cutoff,
+                ),
+            )
+            conn.commit()
+        return cur.rowcount
+
+    def get(self, key: str) -> TaskRecord:
+        with self._connection() as conn:
+            return self._row_to_record(self._ensure_exists(conn, key))
+
+    def count(self, status: Optional[Status] = None) -> int:
+        with self._connection() as conn:
+            if status is None:
+                row = conn.execute("SELECT COUNT(*) AS count FROM tasks").fetchone()
+            else:
+                row = conn.execute(
+                    "SELECT COUNT(*) AS count FROM tasks WHERE status = ?",
+                    (status.value,),
+                ).fetchone()
+        return row["count"]
+
+    def list(self, status: Optional[Status] = None) -> list[TaskRecord]:
+        with self._connection() as conn:
+            if status is None:
+                rows = conn.execute(
+                    "SELECT * FROM tasks ORDER BY created_at, key"
+                ).fetchall()
+            else:
+                rows = conn.execute(
+                    """
+                    SELECT * FROM tasks
+                    WHERE status = ?
+                    ORDER BY created_at, key
+                    """,
+                    (status.value,),
+                ).fetchall()
+        return [self._row_to_record(row) for row in rows]
+
+    def _init_db(self):
+        with self._connection() as conn:
+            conn.execute("PRAGMA journal_mode=WAL")
+            conn.execute("PRAGMA busy_timeout=30000")
+            conn.execute(
+                """
+                CREATE TABLE IF NOT EXISTS tasks (
+                    key TEXT PRIMARY KEY,
+                    payload_data TEXT NOT NULL,
+                    status TEXT NOT NULL,
+                    created_at REAL NOT NULL,
+                    updated_at REAL NOT NULL,
+                    leased_at REAL
+                )
+                """
+            )
+            conn.execute(
+                """
+                CREATE INDEX IF NOT EXISTS idx_tasks_claim
+                ON tasks(status, created_at, key)
+                """
+            )
+            conn.commit()
+
+    @contextmanager
+    def _connection(self):
+        conn = sqlite3.connect(
+            self.path,
+            timeout=30,
+            isolation_level=None,
+        )
+        conn.row_factory = sqlite3.Row
+        try:
+            conn.execute("PRAGMA busy_timeout=30000")
+            yield conn
+        finally:
+            conn.close()
+
+    @staticmethod
+    def _row_to_record(row):
+        return TaskRecord(
+            key=row["key"],
+            payload_data=row["payload_data"],
+            status=Status(row["status"]),
+            created_at=row["created_at"],
+            updated_at=row["updated_at"],
+            leased_at=row["leased_at"],
+        )
+
+    @staticmethod
+    def _ensure_exists(conn, key):
+        row = conn.execute(
+            "SELECT * FROM tasks WHERE key = ?",
+            (key,),
+        ).fetchone()
+        if row is None:
+            raise KeyError(key)
+        return row
+
+    @staticmethod
+    def _require_status(row, key, status):
+        if row["status"] != status.value:
+            raise ValueError(
+                f"task {key} is {row['status']}, expected {status.value}"
+            )

task_pool/stores/text_file.py

@@ -0,0 +1,34 @@
+import os
+
+from task_pool.stores.source_backed import SourceBackedStore
+
+
+class TextLineStore(SourceBackedStore):
+    def __init__(
+        self,
+        path,
+        *,
+        state_path=None,
+        encoding="utf-8",
+        skip_blank=True,
+    ):
+        super().__init__(path, state_path=state_path)
+        self.encoding = encoding
+        self.skip_blank = skip_blank
+        self._import_lines()
+
+    def _import_lines(self):
+        self._import_payloads(self._read_lines())
+
+    def _read_lines(self):
+        if not os.path.exists(self.path):
+            raise FileNotFoundError(self.path)
+        with open(self.path, "r", encoding=self.encoding) as f:
+            for line_number, line in enumerate(f, start=1):
+                line = line.rstrip("\r\n")
+                if self.skip_blank and not line:
+                    continue
+                yield line_number, {
+                    "line": line,
+                    "line_number": line_number,
+                }