refresh-engine 0.1.0__tar.gz → 1.0.0__tar.gz

refresh_engine-1.0.0/PKG-INFO

@@ -0,0 +1,207 @@
+Metadata-Version: 2.4
+Name: refresh-engine
+Version: 1.0.0
+Summary: Asyncio-first, domain-neutral change detection and dependency-aware refresh orchestration for Python.
+Keywords: asyncio,change-detection,dependency-graph,fingerprint,incremental,orchestration,refresh,scheduler,synchronization
+Author: S. Muni Harish
+License-Expression: Apache-2.0
+License-File: LICENSE
+Classifier: Development Status :: 5 - Production/Stable
+Classifier: Framework :: AsyncIO
+Classifier: Intended Audience :: Developers
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3 :: Only
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Programming Language :: Python :: 3.14
+Classifier: Programming Language :: Python :: 3.15
+Classifier: Topic :: Software Development :: Libraries :: Python Modules
+Classifier: Typing :: Typed
+Requires-Dist: structlog>=26.1.0
+Requires-Dist: tenacity>=9.2.1
+Requires-Dist: opentelemetry-api>=1.45.2,<2 ; extra == 'otel'
+Requires-Dist: opentelemetry-sdk>=1.45.2,<2 ; extra == 'otel'
+Requires-Python: >=3.12
+Project-URL: Homepage, https://github.com/smuniharish/refresh-engine
+Project-URL: Documentation, https://refresh-engine.readthedocs.io
+Project-URL: Repository, https://github.com/smuniharish/refresh-engine
+Project-URL: Issues, https://github.com/smuniharish/refresh-engine/issues
+Project-URL: Changelog, https://refresh-engine.readthedocs.io/en/latest/changelog/
+Provides-Extra: otel
+Description-Content-Type: text/markdown
+
+# refresh-engine
+
+[![PyPI](https://img.shields.io/pypi/v/refresh-engine)](https://pypi.org/project/refresh-engine/)
+[![Python](https://img.shields.io/pypi/pyversions/refresh-engine)](https://pypi.org/project/refresh-engine/)
+[![CI](https://github.com/smuniharish/refresh-engine/actions/workflows/ci.yml/badge.svg?branch=master)](https://github.com/smuniharish/refresh-engine/actions/workflows/ci.yml)
+[![Documentation](https://readthedocs.org/projects/refresh-engine/badge/?version=latest)](https://refresh-engine.readthedocs.io/en/latest/)
+[![License](https://img.shields.io/pypi/l/refresh-engine)](https://github.com/smuniharish/refresh-engine/blob/HEAD/LICENSE)
+
+**Keep anything in sync with its source by refreshing only what changed.**
+
+refresh-engine is an asyncio-first, domain-neutral Python library for keeping
+derived data current: search indexes, caches, embeddings, configuration,
+registries, or anything else built from a source. You describe how to list and
+read resources and how to apply a change; the engine detects what changed,
+plans the work in dependency order, runs it safely, and remembers the outcome.
+
+<picture>
+  <source media="(prefers-color-scheme: dark)" srcset="https://raw.githubusercontent.com/smuniharish/refresh-engine/HEAD/docs/assets/diagrams/pipeline-dark.png">
+  <img alt="Discover, observe, detect changes, plan, execute, and persist" src="https://raw.githubusercontent.com/smuniharish/refresh-engine/HEAD/docs/assets/diagrams/pipeline-light.png">
+</picture>
+
+## Features
+
+- **Precise change detection.** Deterministic fingerprints detect added,
+  modified, and deleted resources. Reliable cheap indicators, such as ETags or
+  version tokens, skip reading unchanged resources entirely.
+- **Four refresh modes.** Incremental, full, dependency-aware, and targeted
+  refreshes, with targets chosen by ID or by composable selectors.
+- **Dependency ordering.** Dependencies are refreshed before their dependents,
+  changes propagate to everything that depends on them, and cycles are isolated.
+- **Safe by default.** Bounded concurrency, retries with jittered exponential
+  backoff, timeouts, and fail-fast or best-effort policies. A partial discovery
+  never deletes anything, and failed work is retried by the next refresh.
+- **Durable state.** State advances only for successful resources, in one atomic
+  commit per refresh. Use the in-memory store or plug in your own database.
+- **Scheduling and overlap control.** A drift-free interval scheduler and
+  policies that coalesce, queue, skip, or supersede overlapping requests.
+- **Observability.** Typed lifecycle events with correlation IDs, structured
+  logging through [structlog](https://www.structlog.org) with secret redaction,
+  metrics, and optional OpenTelemetry spans and metrics using application-owned
+  providers.
+- **Lightweight and typed.** Fully type-annotated, with two runtime
+  dependencies: structlog and [tenacity](https://tenacity.readthedocs.io).
+
+## Installation
+
+refresh-engine requires Python 3.12 or newer.
+
+```bash
+pip install refresh-engine
+```
+
+For the optional OpenTelemetry integration, install
+`refresh-engine[otel]`.
+
+## Quick start
+
+Implement a source that lists and reads resources, and an operation that applies
+each change. The engine does the rest:
+
+<!-- test -->
+```python
+import asyncio
+
+from refresh_engine import (
+    DiscoveryResult,
+    PlanAction,
+    RefreshEngine,
+    ResourceSource,
+    RefreshRequest,
+    Resource,
+    ResourceSnapshot,
+)
+
+CATALOG = {"apples": 120, "pears": 80}
+SEARCH_INDEX: dict[str, int] = {}
+
+
+class InventorySource(ResourceSource):
+    async def discover(self) -> DiscoveryResult:
+        return DiscoveryResult(Resource(item) for item in sorted(CATALOG))
+
+    async def snapshot(self, resource: Resource, /) -> ResourceSnapshot:
+        return ResourceSnapshot(
+            resource.resource_id, content=CATALOG[resource.resource_id]
+        )
+
+
+async def update_index(
+    resource: Resource,
+    snapshot: ResourceSnapshot | None,
+    action: PlanAction,
+    request: RefreshRequest,
+    /,
+) -> None:
+    if action is PlanAction.DELETE:
+        SEARCH_INDEX.pop(resource.resource_id, None)
+    elif snapshot is not None:
+        SEARCH_INDEX[resource.resource_id] = snapshot.content
+
+
+async def main() -> None:
+    async with RefreshEngine(InventorySource(), update_index) as engine:
+        first = await engine.refresh()
+        print(f"{first.status}: refreshed {first.refreshed_count}", SEARCH_INDEX)
+
+        CATALOG["pears"] = 75
+        del CATALOG["apples"]
+        second = await engine.refresh()
+        print(
+            f"{second.status}: modified {second.modified_count},"
+            f" deleted {second.deleted_count}",
+            SEARCH_INDEX,
+        )
+
+
+asyncio.run(main())
+```
+
+```text
+success: refreshed 2 {'apples': 120, 'pears': 80}
+success: modified 1, deleted 1 {'pears': 75}
+```
+
+The second refresh touches only the modified and the deleted resource. Continue
+with the [getting started guide](https://refresh-engine.readthedocs.io/en/latest/getting-started/)
+to add durable state, dependencies, scheduling, and logging.
+
+## Performance
+
+The engine's overhead is small next to real I/O: with in-memory sources, a full
+refresh of 100,000 resources takes about 3 seconds, and an incremental refresh
+that finds nothing changed takes about half a second. See the
+[benchmarks](https://github.com/smuniharish/refresh-engine/blob/HEAD/benchmarks/README.md)
+for the method and details.
+
+## Documentation
+
+The [documentation](https://refresh-engine.readthedocs.io/en/latest/) covers:
+
+- [Getting started](https://refresh-engine.readthedocs.io/en/latest/getting-started/)
+  and [concepts](https://refresh-engine.readthedocs.io/en/latest/concepts/)
+- [Guides](https://refresh-engine.readthedocs.io/en/latest/guides/) for every
+  feature, including a
+  [production checklist](https://refresh-engine.readthedocs.io/en/latest/guides/production/)
+- [Runnable examples](https://refresh-engine.readthedocs.io/en/latest/examples/),
+  including SQLite and PostgreSQL state stores
+- The [API reference](https://refresh-engine.readthedocs.io/en/latest/api/)
+- The [architecture](https://refresh-engine.readthedocs.io/en/latest/architecture/overview/)
+  and its guarantees
+
+## Agent Skills
+
+Coding agents such as Claude Code, Codex, Cursor, and GitHub Copilot can use the
+refresh-engine [Agent Skill](https://refresh-engine.readthedocs.io/en/latest/agent-skills/)
+to integrate the library correctly:
+
+```bash
+npx skills add https://github.com/smuniharish/refresh-engine/tree/master/refresh-engine-skills/skills/refresh-engine
+```
+
+## Contributing
+
+Contributions are welcome. Read the
+[contributing guide](https://github.com/smuniharish/refresh-engine/blob/HEAD/CONTRIBUTING.md),
+and report vulnerabilities as described in the
+[security policy](https://github.com/smuniharish/refresh-engine/blob/HEAD/SECURITY.md).
+Notable changes are listed in the
+[changelog](https://github.com/smuniharish/refresh-engine/blob/HEAD/CHANGELOG.md).
+
+## License
+
+refresh-engine is licensed under the
+[Apache License 2.0](https://github.com/smuniharish/refresh-engine/blob/HEAD/LICENSE).

refresh_engine-1.0.0/README.md

@@ -0,0 +1,174 @@
+# refresh-engine
+
+[![PyPI](https://img.shields.io/pypi/v/refresh-engine)](https://pypi.org/project/refresh-engine/)
+[![Python](https://img.shields.io/pypi/pyversions/refresh-engine)](https://pypi.org/project/refresh-engine/)
+[![CI](https://github.com/smuniharish/refresh-engine/actions/workflows/ci.yml/badge.svg?branch=master)](https://github.com/smuniharish/refresh-engine/actions/workflows/ci.yml)
+[![Documentation](https://readthedocs.org/projects/refresh-engine/badge/?version=latest)](https://refresh-engine.readthedocs.io/en/latest/)
+[![License](https://img.shields.io/pypi/l/refresh-engine)](https://github.com/smuniharish/refresh-engine/blob/HEAD/LICENSE)
+
+**Keep anything in sync with its source by refreshing only what changed.**
+
+refresh-engine is an asyncio-first, domain-neutral Python library for keeping
+derived data current: search indexes, caches, embeddings, configuration,
+registries, or anything else built from a source. You describe how to list and
+read resources and how to apply a change; the engine detects what changed,
+plans the work in dependency order, runs it safely, and remembers the outcome.
+
+<picture>
+  <source media="(prefers-color-scheme: dark)" srcset="https://raw.githubusercontent.com/smuniharish/refresh-engine/HEAD/docs/assets/diagrams/pipeline-dark.png">
+  <img alt="Discover, observe, detect changes, plan, execute, and persist" src="https://raw.githubusercontent.com/smuniharish/refresh-engine/HEAD/docs/assets/diagrams/pipeline-light.png">
+</picture>
+
+## Features
+
+- **Precise change detection.** Deterministic fingerprints detect added,
+  modified, and deleted resources. Reliable cheap indicators, such as ETags or
+  version tokens, skip reading unchanged resources entirely.
+- **Four refresh modes.** Incremental, full, dependency-aware, and targeted
+  refreshes, with targets chosen by ID or by composable selectors.
+- **Dependency ordering.** Dependencies are refreshed before their dependents,
+  changes propagate to everything that depends on them, and cycles are isolated.
+- **Safe by default.** Bounded concurrency, retries with jittered exponential
+  backoff, timeouts, and fail-fast or best-effort policies. A partial discovery
+  never deletes anything, and failed work is retried by the next refresh.
+- **Durable state.** State advances only for successful resources, in one atomic
+  commit per refresh. Use the in-memory store or plug in your own database.
+- **Scheduling and overlap control.** A drift-free interval scheduler and
+  policies that coalesce, queue, skip, or supersede overlapping requests.
+- **Observability.** Typed lifecycle events with correlation IDs, structured
+  logging through [structlog](https://www.structlog.org) with secret redaction,
+  metrics, and optional OpenTelemetry spans and metrics using application-owned
+  providers.
+- **Lightweight and typed.** Fully type-annotated, with two runtime
+  dependencies: structlog and [tenacity](https://tenacity.readthedocs.io).
+
+## Installation
+
+refresh-engine requires Python 3.12 or newer.
+
+```bash
+pip install refresh-engine
+```
+
+For the optional OpenTelemetry integration, install
+`refresh-engine[otel]`.
+
+## Quick start
+
+Implement a source that lists and reads resources, and an operation that applies
+each change. The engine does the rest:
+
+<!-- test -->
+```python
+import asyncio
+
+from refresh_engine import (
+    DiscoveryResult,
+    PlanAction,
+    RefreshEngine,
+    ResourceSource,
+    RefreshRequest,
+    Resource,
+    ResourceSnapshot,
+)
+
+CATALOG = {"apples": 120, "pears": 80}
+SEARCH_INDEX: dict[str, int] = {}
+
+
+class InventorySource(ResourceSource):
+    async def discover(self) -> DiscoveryResult:
+        return DiscoveryResult(Resource(item) for item in sorted(CATALOG))
+
+    async def snapshot(self, resource: Resource, /) -> ResourceSnapshot:
+        return ResourceSnapshot(
+            resource.resource_id, content=CATALOG[resource.resource_id]
+        )
+
+
+async def update_index(
+    resource: Resource,
+    snapshot: ResourceSnapshot | None,
+    action: PlanAction,
+    request: RefreshRequest,
+    /,
+) -> None:
+    if action is PlanAction.DELETE:
+        SEARCH_INDEX.pop(resource.resource_id, None)
+    elif snapshot is not None:
+        SEARCH_INDEX[resource.resource_id] = snapshot.content
+
+
+async def main() -> None:
+    async with RefreshEngine(InventorySource(), update_index) as engine:
+        first = await engine.refresh()
+        print(f"{first.status}: refreshed {first.refreshed_count}", SEARCH_INDEX)
+
+        CATALOG["pears"] = 75
+        del CATALOG["apples"]
+        second = await engine.refresh()
+        print(
+            f"{second.status}: modified {second.modified_count},"
+            f" deleted {second.deleted_count}",
+            SEARCH_INDEX,
+        )
+
+
+asyncio.run(main())
+```
+
+```text
+success: refreshed 2 {'apples': 120, 'pears': 80}
+success: modified 1, deleted 1 {'pears': 75}
+```
+
+The second refresh touches only the modified and the deleted resource. Continue
+with the [getting started guide](https://refresh-engine.readthedocs.io/en/latest/getting-started/)
+to add durable state, dependencies, scheduling, and logging.
+
+## Performance
+
+The engine's overhead is small next to real I/O: with in-memory sources, a full
+refresh of 100,000 resources takes about 3 seconds, and an incremental refresh
+that finds nothing changed takes about half a second. See the
+[benchmarks](https://github.com/smuniharish/refresh-engine/blob/HEAD/benchmarks/README.md)
+for the method and details.
+
+## Documentation
+
+The [documentation](https://refresh-engine.readthedocs.io/en/latest/) covers:
+
+- [Getting started](https://refresh-engine.readthedocs.io/en/latest/getting-started/)
+  and [concepts](https://refresh-engine.readthedocs.io/en/latest/concepts/)
+- [Guides](https://refresh-engine.readthedocs.io/en/latest/guides/) for every
+  feature, including a
+  [production checklist](https://refresh-engine.readthedocs.io/en/latest/guides/production/)
+- [Runnable examples](https://refresh-engine.readthedocs.io/en/latest/examples/),
+  including SQLite and PostgreSQL state stores
+- The [API reference](https://refresh-engine.readthedocs.io/en/latest/api/)
+- The [architecture](https://refresh-engine.readthedocs.io/en/latest/architecture/overview/)
+  and its guarantees
+
+## Agent Skills
+
+Coding agents such as Claude Code, Codex, Cursor, and GitHub Copilot can use the
+refresh-engine [Agent Skill](https://refresh-engine.readthedocs.io/en/latest/agent-skills/)
+to integrate the library correctly:
+
+```bash
+npx skills add https://github.com/smuniharish/refresh-engine/tree/master/refresh-engine-skills/skills/refresh-engine
+```
+
+## Contributing
+
+Contributions are welcome. Read the
+[contributing guide](https://github.com/smuniharish/refresh-engine/blob/HEAD/CONTRIBUTING.md),
+and report vulnerabilities as described in the
+[security policy](https://github.com/smuniharish/refresh-engine/blob/HEAD/SECURITY.md).
+Notable changes are listed in the
+[changelog](https://github.com/smuniharish/refresh-engine/blob/HEAD/CHANGELOG.md).
+
+## License
+
+refresh-engine is licensed under the
+[Apache License 2.0](https://github.com/smuniharish/refresh-engine/blob/HEAD/LICENSE).

refresh_engine-1.0.0/pyproject.toml

@@ -0,0 +1,201 @@
+[build-system]
+requires = ["uv_build>=0.13.7,<0.14"]
+build-backend = "uv_build"
+
+[project]
+name = "refresh-engine"
+version = "1.0.0"
+description = "Asyncio-first, domain-neutral change detection and dependency-aware refresh orchestration for Python."
+readme = "README.md"
+requires-python = ">=3.12"
+license = "Apache-2.0"
+license-files = ["LICENSE"]
+keywords = [
+    "asyncio",
+    "change-detection",
+    "dependency-graph",
+    "fingerprint",
+    "incremental",
+    "orchestration",
+    "refresh",
+    "scheduler",
+    "synchronization",
+]
+classifiers = [
+    "Development Status :: 5 - Production/Stable",
+    "Framework :: AsyncIO",
+    "Intended Audience :: Developers",
+    "Operating System :: OS Independent",
+    "Programming Language :: Python :: 3",
+    "Programming Language :: Python :: 3 :: Only",
+    "Programming Language :: Python :: 3.12",
+    "Programming Language :: Python :: 3.13",
+    "Programming Language :: Python :: 3.14",
+    "Programming Language :: Python :: 3.15",
+    "Topic :: Software Development :: Libraries :: Python Modules",
+    "Typing :: Typed",
+]
+dependencies = [
+    "structlog>=26.1.0",
+    "tenacity>=9.2.1",
+]
+
+[[project.authors]]
+name = "S. Muni Harish"
+
+[project.optional-dependencies]
+otel = [
+    "opentelemetry-api>=1.45.2,<2",
+    "opentelemetry-sdk>=1.45.2,<2",
+]
+
+[project.urls]
+Homepage = "https://github.com/smuniharish/refresh-engine"
+Documentation = "https://refresh-engine.readthedocs.io"
+Repository = "https://github.com/smuniharish/refresh-engine"
+Issues = "https://github.com/smuniharish/refresh-engine/issues"
+Changelog = "https://refresh-engine.readthedocs.io/en/latest/changelog/"
+
+[dependency-groups]
+test = [
+    "coverage[toml]>=7.17.2",
+    "hypothesis>=6.170.15",
+    "pytest>=9.2.0",
+    "pytest-asyncio>=1.4.1.post1",
+    "pytest-cov>=7.1.0",
+    "pytest-timeout>=2.4.0",
+    "skills-ref>=0.1.1,<1",
+]
+lint = ["ruff>=0.17.4"]
+types = ["pyrefly>=1.4.1"]
+docs = [
+    "griffe-pydantic>=1.3.2,<2",
+    "mkdocstrings-python>=2.0.10,<3",
+    "zensical>=0.1.0,<0.2",
+]
+examples = ["psycopg[binary,pool]>=3.3.8,<4"]
+package = ["twine>=7.0.0"]
+audit = ["pip-audit>=2.10.1"]
+
+[[dependency-groups.dev]]
+include-group = "test"
+
+[[dependency-groups.dev]]
+include-group = "lint"
+
+[[dependency-groups.dev]]
+include-group = "types"
+
+[tool.uv]
+default-groups = ["dev"]
+
+[tool.pytest]
+minversion = "9.0"
+addopts = [
+    "-ra",
+    "--strict-markers",
+    "--strict-config",
+]
+testpaths = ["tests"]
+xfail_strict = true
+filterwarnings = ["error"]
+timeout = "120"
+asyncio_mode = "auto"
+asyncio_default_fixture_loop_scope = "function"
+asyncio_default_test_loop_scope = "function"
+
+[tool.coverage.run]
+branch = true
+source_pkgs = ["refresh_engine"]
+
+[tool.coverage.report]
+fail_under = 100
+show_missing = true
+skip_covered = true
+exclude_also = [
+    "if TYPE_CHECKING:",
+    '^\s*\.\.\.\s*$',
+]
+
+[tool.ruff]
+target-version = "py312"
+line-length = 88
+
+[tool.ruff.format]
+docstring-code-format = true
+
+[tool.ruff.lint]
+select = [
+    "ASYNC",
+    "B",
+    "C4",
+    "D",
+    "E",
+    "ERA",
+    "F",
+    "FURB",
+    "I",
+    "N",
+    "PERF",
+    "PGH",
+    "PIE",
+    "PT",
+    "RET",
+    "RUF",
+    "S",
+    "SIM",
+    "T20",
+    "UP",
+    "W",
+]
+
+[tool.ruff.lint.pydocstyle]
+convention = "google"
+
+[tool.ruff.lint.per-file-ignores]
+"tests/**" = [
+    "D",
+    "S101",
+    "S311",
+]
+"src/refresh_engine/testing.py" = ["S101"]
+"examples/**" = [
+    "D",
+    "S101",
+    "T201",
+]
+"benchmarks/**" = [
+    "D",
+    "S311",
+    "T201",
+]
+"scripts/**" = [
+    "D",
+    "S603",
+    "S607",
+    "T201",
+]
+"refresh-engine-skills/**" = ["T201"]
+"tests/skill/**" = [
+    "D",
+    "S101",
+    "T201",
+]
+
+[tool.pyrefly]
+project-includes = [
+    "src",
+    "tests",
+    "examples",
+    "scripts",
+    "benchmarks",
+    "refresh-engine-skills",
+    "tests/skill",
+]
+search-path = [
+    "src",
+    ".",
+    "examples",
+    "scripts",
+]
+python-version = "3.12"