carbonation 0.0.1__tar.gz

carbonation-0.0.1/.github/workflows/release.yml

@@ -0,0 +1,26 @@
+name: "Publish"
+
+on:
+  release:
+    types: [published]
+
+jobs:
+  publish:
+    runs-on: ubuntu-latest
+    environment:
+      name: pypi
+    permissions:
+      id-token: write
+      contents: read
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v6
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@v7
+
+      - name: Build
+        run: uv build
+
+      - name: Publish
+        run: uv publish

carbonation-0.0.1/.github/workflows/test.yml

@@ -0,0 +1,26 @@
+name: Test
+
+on: [push, pull_request]
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: ["3.11", "3.12", "3.13"]
+
+    steps:
+    - uses: actions/checkout@v6
+
+    - name: Install uv and set the Python version
+      uses: astral-sh/setup-uv@v7
+      with:
+        enable-cache: true
+        python-version: ${{ matrix.python-version }}
+
+    - name: Install the project
+      run: uv sync --locked --all-extras --dev
+
+    - name: Run pytest
+      run: uv run pytest tests/

carbonation-0.0.1/.gitignore

@@ -0,0 +1,139 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+pip-wheel-metadata/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+__about__.py
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+.python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+#Pipfile.lock
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+# hatch-vcs
+_version.py
+
+# ruff
+.ruff_cache
+
+# testing
+test_data

carbonation-0.0.1/CLAUDE.md

@@ -0,0 +1,110 @@
+# Carbonation
+
+File delivery monitoring and archival service. Watches directories for incoming files, groups them by stem, validates integrity, extracts metadata via plugins, applies selection/routing rules, and archives to configurable destinations.
+
+## Quick Reference
+
+```bash
+uv run pytest tests/                          # all tests (~24s)
+uv run pytest tests/ -m "not integration"     # unit tests only (~2s)
+uv run pytest tests/test_integration.py       # integration tests only (~22s)
+uv run ruff check src/ tests/                 # lint
+uv run ruff check --fix src/ tests/           # lint + autofix
+```
+
+## Project Layout
+
+```
+src/carbonation/          # main package (src layout, built with hatchling)
+  cli.py                  # Click CLI: check-config, db init/status, run, reconcile
+  config.py               # Pydantic models + TOML loading for config.toml and rules.toml
+  service.py              # Main orchestrator: watcher -> rules -> archive pipeline
+  watcher.py              # Watchdog integration with settle-timer debouncing
+  rules.py                # Integrity, completeness, selection, routing evaluation
+  archive.py              # File copy/move/hardlink/symlink with permissions
+  reconcile.py            # Sync delivery/archive directories with DB
+  plugins.py              # Plugin registry: loads .py files, indexes callables
+  log.py                  # Loguru configuration
+  exceptions.py           # CarbonationError hierarchy
+  db/
+    models.py             # SQLAlchemy ORM: FileComponent, FileBase (abstract)
+    queries.py            # All DB operations (no auto-commit; callers commit)
+    engine.py             # Engine/session factory, schema init/verify
+examples/                 # Example config.toml, rules.toml, plugin
+scripts/                  # generate_fake_data.py for test data
+tests/                    # pytest suite
+```
+
+## Architecture
+
+**Processing pipeline** (in `service.py:_process_event_inner`):
+1. Record component (insert-first, handle IntegrityError for races)
+2. Integrity checks (min_size, readability)
+3. Completeness grouping (stem-based or plugin-based, with group locking)
+4. Metadata extraction (plugin `extract()` function)
+5. Selection rules (OR logic, has_keys / field matches / plugin)
+6. Routing (glob or plugin, first-match-wins)
+7. Archive (copy/move/hardlink/symlink with configurable permissions)
+
+**Threading model**: Main thread runs event loop, ThreadPoolExecutor processes events, per-group locks (WeakValueDictionary) protect completeness checks, periodic Timer for reconciliation + completeness timeouts.
+
+**Database**: Commits happen at caller boundaries, not in individual query functions. `_process_event` commits once after the full pipeline. `reconcile_delivery`/`reconcile_archive` commit at the end.
+
+**Plugin system**: Plugins are `.py` files loaded from a configured directory. Must provide a `FileBase` subclass with `__tablename__ = "files"` for the File model, and an `extract(file_paths) -> dict` function for metadata. Domain columns on the File model must be nullable (populated after group creation).
+
+## Testing
+
+- **Unit tests** (`test_*.py` except `test_integration.py`): Fast, in-memory SQLite, no filesystem watchers
+- **Integration tests** (`test_integration.py`): Start real service in background thread, write files to disk, verify archive + DB state. Marked `@pytest.mark.integration`.
+- **conftest.py** registers a `_TestFile(FileBase)` model at module level with both unit test fields (`label`) and integration test fields (`start`, `stop`, `category`, `source`). Integration test plugins must NOT define their own FileBase subclass — only provide `extract()`.
+- Permission tests (`test_archive.py`) are skipped on Windows (`@pytest.mark.skipif(sys.platform == "win32")`)
+
+## Configuration
+
+Two TOML files:
+- **config.toml**: service settings, logging, database, plugins, watch directories, archive destinations
+- **rules.toml**: integrity checks, completeness grouping, metadata module, selection rules, routing rules
+
+Paths in config.toml are resolved relative to the config file's directory. `archive.destination` is resolved relative to `archive.base_path` when set.
+
+**Hot-reload**: The service watches `rules.toml` for changes (debounced 0.5s). On modification it validates the new file, checks that all watch configs still reference valid rule sets, and atomically swaps `self.rules_config`. Invalid files or missing rule sets are rejected with a log warning — the service keeps running with the previous rules.
+
+**Database credentials**: `DatabaseConfig` uses structured fields (`drivername`, `host`, `port`, `username`, `password`, `name`) instead of a raw URL string. The `url` property builds the SQLAlchemy URL via `URL.create()`. Credentials are layered: config.toml values take priority over `~/.config/carbonation/secrets.toml`. Secrets file uses the same `[database]` section format. SQLite `name` paths are resolved relative to the config file.
+
+## Reliability
+
+- **Retry logic**: Archive failures mark components `RETRY_PENDING` instead of `FAILED`. Periodic checker re-enqueues them after `retry_delay`. After `max_retries` attempts, permanently marked `FAILED`. Manual retry via `carbonation retry`.
+- **Post-archive verification**: File size compared after copy/move. Mismatch deletes the corrupt copy and raises an error (triggers retry).
+- **Watch liveness**: Each reconciliation cycle verifies watch directories are present and readable.
+
+## Observability
+
+- **Heartbeat**: `service_state` table updated on its own timer (`heartbeat_interval`, default `60s`) with `last_heartbeat`, `queue_depth`. Also pings the systemd watchdog (`WATCHDOG=1`) each cycle.
+- **`carbonation status`**: Shows heartbeat age, component counts, watch directory health.
+- **Stats logging**: Periodic summary logged each heartbeat: `Stats: archived=42, failed=2, queue_depth=0`.
+- **Structured logging**: Set `format = "json"` in `[logging]` config for JSON lines output (loguru `serialize=True`).
+- **`carbonation check-rules`**: Dry-run a candidate rules file against existing DB groups to preview selection/routing changes.
+
+## CLI Commands
+
+```
+carbonation check-config          # validate config + rules
+carbonation db init                # create database schema
+carbonation db status              # component counts, metadata check
+carbonation run [--dry-run] [--once] # start the service
+carbonation status                 # heartbeat, counts, watch health
+carbonation retry [--watch-name X] # re-enqueue failed components
+carbonation check-rules FILE       # dry-run rules changes
+carbonation reconcile delivery     # sync delivery dirs with DB
+carbonation reconcile archive      # sync archive dirs with DB
+carbonation query files [filters]  # dynamic query with plugin columns
+```
+
+## Common Patterns
+
+- All config models are Pydantic with field validators
+- Duration strings: `"5s"`, `"1m"`, `"24h"`, `"7d"` (parsed by `config.parse_duration`)
+- File permissions: `file_mode`/`dir_mode` as octal ints (default `0o440`/`0o550`), only applied on copy/move (not hardlink/symlink)
+- `signal.signal()` is guarded for non-main-thread usage (needed for test harness)
+- DB commits happen at caller boundaries, not in query functions
+- Retry count tracked per-component; periodic checker increments on re-enqueue

carbonation-0.0.1/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Jonathan Olsten
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

carbonation-0.0.1/PKG-INFO

@@ -0,0 +1,354 @@
+Metadata-Version: 2.4
+Name: carbonation
+Version: 0.0.1
+Summary: File delivery monitoring and archival service
+Author-email: Jonathan Olsten <jonathan.olsten@gmail.com>
+License-File: LICENSE
+Requires-Python: >=3.11
+Requires-Dist: alembic<2,>=1.13
+Requires-Dist: click>=8.1
+Requires-Dist: loguru>=0.7.3
+Requires-Dist: pydantic<3,>=2.10
+Requires-Dist: rich>=14.3.3
+Requires-Dist: sdnotify<1,>=0.3
+Requires-Dist: sqlalchemy<3,>=2.0.48
+Requires-Dist: watchdog>=6.0.0
+Provides-Extra: mariadb
+Requires-Dist: mysqlclient<3,>=2.2; extra == 'mariadb'
+Provides-Extra: postgresql
+Requires-Dist: psycopg2<3,>=2.9; extra == 'postgresql'
+Description-Content-Type: text/markdown
+
+# Carbonation
+
+File delivery monitoring and archival service. Watches directories for incoming files, groups them by stem, validates integrity, extracts metadata via plugins, applies selection and routing rules, and archives to configurable destinations.
+
+## Installation
+
+Requires Python 3.11+.
+
+```bash
+pip install carbonation
+```
+
+For MariaDB or PostgreSQL support:
+
+```bash
+pip install "carbonation[mariadb]"     # mysqlclient
+pip install "carbonation[postgresql]"  # psycopg2
+```
+
+## Quick Start
+
+```bash
+# 1. Scaffold a new project
+mkdir /opt/carbonation && cd /opt/carbonation
+carbonation init
+
+# 2. Edit the generated files
+$EDITOR config.toml       # set watch paths, archive destination, database
+$EDITOR rules.toml        # configure integrity, completeness, selection rules
+$EDITOR plugins/file_model.py  # define your File model columns and extract()
+
+# 3. Validate, initialize, and run
+carbonation check-config   # validate configuration
+carbonation db init        # create database tables
+carbonation run            # start the service
+```
+
+`carbonation init` generates a complete starter project:
+
+```
+config.toml             # service, logging, database, watch, and archive config
+rules.toml              # integrity, completeness, metadata, selection, routing
+plugins/file_model.py   # File model + extract() function template
+```
+
+Edit `plugins/file_model.py` to define the domain-specific columns for your use case and the `extract()` function that reads metadata from your files. See [Plugin System](#plugin-system) for details.
+
+## CLI Reference
+
+```
+carbonation check-config              Validate config.toml and rules.toml
+carbonation db init                    Create schema (migrations + plugin columns)
+carbonation db upgrade                 Apply pending schema migrations only
+carbonation db status                  Component counts and metadata check
+carbonation run [--dry-run] [--once]   Start the service
+carbonation status                     Heartbeat, status table, watch health
+carbonation retry [--watch-name X]     Re-enqueue failed components
+carbonation check-rules FILE           Dry-run rules changes against DB
+carbonation reconcile delivery         Sync delivery directories with DB
+carbonation reconcile archive          Sync archive directories with DB
+carbonation query files [filters]      Query with dynamic plugin filters
+```
+
+### Query Examples
+
+```bash
+# Filter by plugin columns (dynamically generated from your model)
+carbonation query files --category alpha --limit 50
+
+# Date range overlap
+carbonation query files --daterange 2026-01-01/2026-02-01
+
+# Recent files by age
+carbonation query files --age 7d
+
+# Output formats
+carbonation query files --category alpha --format json
+carbonation query files --format csv > export.csv
+
+# Count only
+carbonation query files --category alpha --count
+
+# Specific columns, sorted
+carbonation query files --columns group_key,category,start --order-by -start
+```
+
+## Programmatic API
+
+Query the database from Python scripts without going through the CLI:
+
+```python
+from carbonation import connect
+
+with connect("config.toml") as db:
+    # All files from a watch
+    rows = db.query_files({"watch_name": ["incoming"]}, limit=50)
+    for row in rows:
+        print(row["group_key"], row["created_at"])
+
+    # Count
+    n = db.query_files({"complete": True}, count_only=True)
+
+    # Date filtering + plugin columns
+    from datetime import datetime
+    rows = db.query_files({
+        "category": ["alpha"],
+        "created_at_after": datetime(2026, 1, 1),
+    }, order_by="-created_at", columns=["group_key", "category", "start"])
+```
+
+For one-off scripts, use the convenience function that handles setup and teardown in one call:
+
+```python
+from carbonation import query_files
+
+rows = query_files("config.toml", {"category": ["alpha"]}, limit=50)
+```
+
+### Watermark-based polling
+
+For external consumers (e.g. an orchestrator) that need to poll for newly completed files without gaps:
+
+```python
+from carbonation import connect
+
+cursor = None  # persist this between invocations
+with connect("config.toml") as db:
+    while True:
+        rows, cursor = db.get_new_files(cursor=cursor)
+        if not rows:
+            break
+        for row in rows:
+            print(row["group_key"], row["completed_at"])
+
+    # Check if the service is alive
+    status = db.get_service_status()
+    if status:
+        print(f"Last heartbeat: {status['last_heartbeat']}")
+```
+
+### ORM access
+
+For direct ORM access to query, modify, and commit changes:
+
+```python
+from carbonation import session
+from carbonation.db.models import FileComponent, FileStatus
+
+with session("config.toml") as s:
+    comps = s.query(FileComponent).filter_by(status=FileStatus.ARCHIVED).all()
+    for c in comps:
+        c.status = FileStatus.CLEARED
+    s.commit()
+```
+
+### Filter operators
+
+| Suffix | Operator | Example |
+|---|---|---|
+| `_after` / `_before` | `>=` / `<=` (datetime) | `{"created_at_after": datetime(2026, 1, 1)}` |
+| `_min` / `_max` | `>=` / `<=` (numeric) | `{"id_min": 100}` |
+| `_daterange` | start/stop overlap | `{"_daterange": (start, end)}` |
+| `_age` | recency | `{"_age": datetime(2026, 3, 20)}` |
+| (none) | exact / IN / LIKE | `{"category": ["alpha", "beta"]}` |
+
+## Architecture
+
+### Processing Pipeline
+
+```mermaid
+flowchart TD
+    A["File arrives in<br>watch directory"] --> B["Watchdog detects<br>creation / modification"]
+    B --> C["Settle timer<br>debounces"]
+    C --> D["Event enqueued"]
+    D --> E["Worker thread<br>picks up event"]
+
+    E --> S1["1 — Record component"]
+    S1 --> IC{{"2 — Integrity checks<br>(min_size, readability)"}}
+
+    IC -- fail --> IF["INTEGRITY_FAILED"]
+    IC -- pass --> S3{{"3 — Completeness<br>grouping"}}
+
+    S3 -- "incomplete<br>(waiting for extensions)" --> W["WAITING"]
+    S3 -- "timeout<br>(on_timeout=skip)" --> TO["TIMED_OUT"]
+    S3 -- "complete / standalone" --> S4["4 — Metadata extraction<br>(plugin extract)"]
+
+    S4 --> S5{{"5 — Selection rules"}}
+    S5 -- rejected --> NS["NOT_SELECTED"]
+    S5 -- accepted --> S6["6 — Routing<br>(choose archive)"]
+
+    S6 --> S7{{"7 — Archive<br>(copy / move / hardlink / symlink)"}}
+    S7 -- success --> AR["ARCHIVED"]
+    S7 -- "failure<br>(retries left)" --> RP["RETRY_PENDING"]
+    S7 -- "failure<br>(retries exhausted)" --> FA["FAILED"]
+
+    RP -. "retry_delay<br>elapsed" .-> E
+    IF -. "reconciliation<br>(re-assessed)" .-> E
+    NS -. "rules hot-reload<br>(re-evaluated)" .-> E
+
+    style IF fill:#d32f2f,color:#fff
+    style FA fill:#d32f2f,color:#fff
+    style TO fill:#757575,color:#fff
+    style NS fill:#757575,color:#fff
+    style W fill:#f9a825
+    style RP fill:#f57c00,color:#fff
+    style AR fill:#388e3c,color:#fff
+```
+
+### Reliability
+
+- **Retry with backoff**: Archive failures are retried up to `max_retries` times with `retry_delay` between attempts. Only permanently failed after all retries exhausted.
+- **Post-archive verification**: File size verified after copy/move. Mismatched files are deleted and retried.
+- **Reconciliation**: Periodic sync between filesystem and database catches missed files, clears removed files, re-enqueues pending work.
+- **Rules hot-reload**: The service watches `rules.toml` for changes and validates before applying. Invalid changes are rejected without disruption.
+
+### Plugin System
+
+Plugins are `.py` files in the configured plugins directory. A plugin must provide:
+
+- A `FileBase` subclass with `__tablename__ = "files"` defining domain-specific columns
+- An `extract(file_paths: list[Path]) -> dict` function that returns metadata for the domain columns
+
+Domain columns must be nullable since they are populated after the group row is created.
+
+### Configuration
+
+- **config.toml**: Service settings, logging, database, plugins, watch directories, archive destinations
+- **rules.toml**: Integrity checks, completeness grouping, metadata module, selection rules, routing rules
+
+Paths are resolved relative to the config file's directory. `archive.destination` is resolved relative to `archive.base_path`.
+
+Rules can be hot-reloaded by editing `rules.toml` while the service is running.
+
+## Deployment
+
+### Setting up a production install
+
+Create a dedicated virtualenv so the service has an isolated, reproducible Python environment:
+
+```bash
+# Create the project directory and venv
+sudo mkdir -p /opt/carbonation
+cd /opt/carbonation
+python3 -m venv .venv
+
+# Install carbonation into the venv
+.venv/bin/pip install carbonation
+# Add database drivers if needed:
+# .venv/bin/pip install "carbonation[mariadb]"
+# .venv/bin/pip install "carbonation[postgresql]"
+
+# Scaffold config, rules, and plugin template
+.venv/bin/carbonation init
+
+# Edit to match your environment
+$EDITOR config.toml rules.toml plugins/file_model.py
+
+# Validate and initialize
+.venv/bin/carbonation check-config
+.venv/bin/carbonation db init
+```
+
+### systemd
+
+Generate a systemd unit file during scaffolding:
+
+```bash
+.venv/bin/carbonation init --systemd --user carbonation
+```
+
+Then install and enable it:
+
+```bash
+sudo cp carbonation.service /etc/systemd/system/
+sudo systemctl daemon-reload
+sudo systemctl enable --now carbonation
+sudo systemctl status carbonation
+```
+
+The generated unit file uses `Type=notify` with `WatchdogSec=300` — systemd waits for carbonation to signal readiness before marking it as started, and automatically restarts it if the watchdog heartbeat stops (e.g. hung process). Edit `ReadWritePaths=` in the unit file to include your delivery and archive directories.
+
+`WatchdogSec` should be at least 2x your `heartbeat_interval` in `config.toml` (default: `60s`).
+
+### Cron
+
+For environments where a persistent service is not needed, run carbonation as a cron job:
+
+```bash
+*/5 * * * * /opt/carbonation/.venv/bin/carbonation -c /opt/carbonation/config.toml run --once
+```
+
+`--once` reconciles delivery directories, processes all pending events, and exits. It can also supplement a running service as defense-in-depth.
+
+### Upgrading
+
+```bash
+cd /opt/carbonation
+.venv/bin/pip install --upgrade carbonation
+.venv/bin/carbonation -c config.toml db upgrade   # apply schema migrations
+.venv/bin/carbonation -c config.toml db init       # sync plugin columns
+sudo systemctl restart carbonation
+```
+
+`db upgrade` applies pending Alembic migrations (carbonation's base tables). `db init` then adds any new plugin-defined columns. Both are idempotent.
+
+### Monitoring
+
+```bash
+.venv/bin/carbonation -c /opt/carbonation/config.toml status
+```
+
+The `status` command shows:
+- Service heartbeat age (warns if stale)
+- Component status breakdown by time window (1h/4h/1d/7d/30d)
+- Delivery and archive totals
+- Retry queue depth
+- Oldest incomplete group
+- Watch directory health
+
+For log aggregation, set `format = "json"` in the logging config to produce structured JSON lines.
+
+## Development
+
+```bash
+uv sync                                    # install dependencies
+uv run pytest tests/                       # all tests (~50s)
+uv run pytest tests/ -m "not integration"  # unit tests only (~2s)
+uv run ruff check src/ tests/              # lint
+```
+
+## License
+
+MIT