celerp 1.0.3__py3-none-any.whl

celerp/__init__.py

@@ -0,0 +1,6 @@
+# Copyright (c) 2026 Noah Severs
+# SPDX-License-Identifier: BSL-1.1
+
+from __future__ import annotations
+
+__version__ = "1.0.0"

celerp/ai/__init__.py

@@ -0,0 +1,2 @@
+# Copyright (c) 2026 Noah Severs
+# SPDX-License-Identifier: BSL-1.1

celerp/ai/batch.py

@@ -0,0 +1,228 @@
+# Copyright (c) 2026 Noah Severs
+# SPDX-License-Identifier: BSL-1.1
+
+"""AI batch processing - parallel multi-file processing via OpenRouter.
+
+Processes 2-100 files in parallel (up to BATCH_CONCURRENCY concurrent LLM calls).
+Each file gets its own API call with the bulk extraction model.
+
+Usage:
+    job = await start_batch(session, company_id, user_id, query, file_ids, credits)
+    # Background task runs the batch
+    # SSE events emitted per-file completion
+    # Notification created on finish
+"""
+
+from __future__ import annotations
+
+import asyncio
+import base64
+import logging
+import uuid
+from datetime import datetime, timezone
+
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from celerp.ai.files import load_file, load_file_for_llm
+from celerp.ai.llm import call_llm
+from celerp.ai.models import BULK_EXTRACTION
+from celerp.models.ai import AIBatchJob
+
+log = logging.getLogger(__name__)
+
+BATCH_CONCURRENCY = 10
+MAX_BATCH_FILES = 100
+
+_BATCH_SYSTEM_PROMPT = """\
+You are analyzing a business document (receipt, invoice, or contract).
+Extract structured data: vendor name, date, total amount, line items with description/quantity/unit price.
+
+Output a JSON block:
+```json
+{
+  "vendor_name": "string",
+  "date": "YYYY-MM-DD",
+  "total": 0.00,
+  "line_items": [
+    {"description": "string", "quantity": 1, "unit_price": 0.00}
+  ]
+}
+```
+
+If you cannot extract structured data, return a brief text summary instead (no JSON block).
+"""
+
+
+async def _process_single_file(
+    file_id: str,
+    query: str,
+    company_id: uuid.UUID,
+    semaphore: asyncio.Semaphore,
+) -> dict:
+    """Process a single file through the LLM. Returns result dict."""
+    try:
+        data_bytes, meta = load_file(file_id, company_id)
+    except (FileNotFoundError, PermissionError):
+        return {"file_id": file_id, "status": "error", "error": f"File {file_id} not found"}
+
+    b64 = base64.b64encode(data_bytes).decode("utf-8")
+    files = [{"media_type": meta.get("content_type", "image/jpeg"), "data": b64}]
+
+    prompt = f"{query}\n\nAnalyze the attached document." if query else "Analyze the attached document."
+
+    async with semaphore:
+        try:
+            answer = await call_llm(BULK_EXTRACTION, _BATCH_SYSTEM_PROMPT, prompt, files=files)
+            return {
+                "file_id": file_id,
+                "filename": meta.get("filename", file_id),
+                "status": "success",
+                "answer": answer,
+            }
+        except Exception as exc:
+            log.warning("Batch file %s failed: %s", file_id, exc)
+            return {
+                "file_id": file_id,
+                "filename": meta.get("filename", file_id),
+                "status": "error",
+                "error": "Processing failed for this file",
+            }
+
+
+async def create_batch_job(
+    session: AsyncSession,
+    company_id: uuid.UUID,
+    user_id: uuid.UUID,
+    query: str,
+    file_ids: list[str],
+    credits: int,
+    conversation_id: uuid.UUID | None = None,
+) -> AIBatchJob:
+    """Create a pending batch job record."""
+    if len(file_ids) > MAX_BATCH_FILES:
+        raise ValueError(f"Maximum {MAX_BATCH_FILES} files per batch")
+    if len(file_ids) < 2:
+        raise ValueError("Batch requires at least 2 files")
+
+    job = AIBatchJob(
+        company_id=company_id,
+        user_id=user_id,
+        conversation_id=conversation_id,
+        query=query,
+        file_ids=file_ids,
+        total_files=len(file_ids),
+        credits_consumed=credits,
+        status="pending",
+    )
+    session.add(job)
+    await session.flush()
+    return job
+
+
+async def run_batch(
+    job_id: uuid.UUID,
+    company_id: uuid.UUID,
+    user_id: uuid.UUID,
+    query: str,
+    file_ids: list[str],
+    db_factory,
+    on_progress=None,
+) -> None:
+    """Execute a batch job: parallel file processing.
+
+    Args:
+        db_factory: Callable that returns an AsyncSession context manager.
+        on_progress: Optional async callback(job_id, completed, failed, total, result).
+    """
+    semaphore = asyncio.Semaphore(BATCH_CONCURRENCY)
+    results: list[dict] = []
+    completed = 0
+    failed = 0
+
+    # Update status to running
+    async with db_factory() as session:
+        job = await session.get(AIBatchJob, job_id)
+        if job:
+            job.status = "running"
+            session.add(job)
+            await session.commit()
+
+    # Fan out: process all files concurrently (bounded by semaphore)
+    tasks = [
+        _process_single_file(fid, query, company_id, semaphore)
+        for fid in file_ids
+    ]
+
+    for coro in asyncio.as_completed(tasks):
+        result = await coro
+        results.append(result)
+
+        if result["status"] == "success":
+            completed += 1
+        else:
+            failed += 1
+
+        # Update DB progress
+        async with db_factory() as session:
+            job = await session.get(AIBatchJob, job_id)
+            if job:
+                job.completed_files = completed
+                job.failed_files = failed
+                session.add(job)
+                await session.commit()
+
+        # Notify progress
+        if on_progress:
+            try:
+                await on_progress(job_id, completed, failed, len(file_ids), result)
+            except Exception:
+                log.debug("on_progress callback failed", exc_info=True)
+
+    # Finalize
+    final_status = "failed" if failed == len(file_ids) else "completed"
+    async with db_factory() as session:
+        job = await session.get(AIBatchJob, job_id)
+        if job:
+            job.status = final_status
+            job.results = {"files": results}
+            job.completed_at = datetime.now(timezone.utc)
+            session.add(job)
+            await session.commit()
+
+    # Create notification
+    try:
+        from celerp.notifications.service import create as create_notification
+        async with db_factory() as session:
+            if final_status == "completed":
+                await create_notification(
+                    session, company_id, "ai",
+                    f"Batch complete: {completed}/{len(file_ids)} files processed",
+                    f"{completed} files processed successfully, {failed} failed.",
+                    user_id=user_id,
+                    action_url="/ai",
+                    priority="high",
+                )
+            else:
+                await create_notification(
+                    session, company_id, "ai",
+                    f"Batch failed: {failed}/{len(file_ids)} files",
+                    "All files failed to process. Please try again.",
+                    user_id=user_id,
+                    action_url="/ai",
+                    priority="high",
+                )
+            await session.commit()
+    except Exception:
+        log.warning("Failed to create batch notification", exc_info=True)
+
+
+async def get_batch_job(
+    session: AsyncSession,
+    job_id: uuid.UUID,
+    company_id: uuid.UUID,
+) -> AIBatchJob | None:
+    """Get a batch job by ID, scoped to company."""
+    job = await session.get(AIBatchJob, job_id)
+    if job is None or job.company_id != company_id:
+        return None
+    return job

celerp/ai/cleanup.py

@@ -0,0 +1,84 @@
+# Copyright (c) 2026 Noah Severs
+# SPDX-License-Identifier: BSL-1.1
+
+"""AI file cleanup - periodic removal of old upload files.
+
+All files older than 30 days are deleted unconditionally.
+Runs every 6 hours via background task.
+
+Does NOT require a database session - operates purely on filesystem.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import logging
+import time
+from pathlib import Path
+
+from celerp.ai.files import upload_dir
+
+log = logging.getLogger(__name__)
+
+CLEANUP_INTERVAL_SECONDS = 6 * 3600  # 6 hours
+MAX_AGE_DAYS = 30
+
+
+def _delete_file_pair(meta_path: Path) -> None:
+    """Delete both .meta and .bin files."""
+    bin_path = meta_path.with_suffix(".bin")
+    try:
+        if bin_path.exists():
+            bin_path.unlink()
+        if meta_path.exists():
+            meta_path.unlink()
+    except OSError:
+        log.warning("Failed to delete %s", meta_path.stem, exc_info=True)
+
+
+def cleanup_uploads() -> int:
+    """Remove AI upload files older than 30 days. Returns count deleted."""
+    try:
+        ud = upload_dir()
+    except (PermissionError, OSError) as exc:
+        log.debug("AI upload dir unavailable, skipping cleanup: %s", exc)
+        return 0
+    if not ud.exists():
+        return 0
+
+    now = time.time()
+    deleted = 0
+
+    for meta_path in list(ud.glob("*.meta")):
+        try:
+            age_days = (now - meta_path.stat().st_mtime) / 86400
+        except OSError:
+            continue
+
+        if age_days > MAX_AGE_DAYS:
+            _delete_file_pair(meta_path)
+            deleted += 1
+            log.debug("Deleted old file %s (%.0f days)", meta_path.stem, age_days)
+
+    # Clean up orphaned .bin files (no matching .meta)
+    for bin_path in list(ud.glob("*.bin")):
+        if not bin_path.with_suffix(".meta").exists():
+            try:
+                bin_path.unlink()
+                deleted += 1
+            except OSError:
+                pass
+
+    if deleted:
+        log.info("AI file cleanup: removed %d files", deleted)
+    return deleted
+
+
+async def run_cleanup_loop() -> None:
+    """Background task: run cleanup every CLEANUP_INTERVAL_SECONDS."""
+    while True:
+        await asyncio.sleep(CLEANUP_INTERVAL_SECONDS)
+        try:
+            cleanup_uploads()
+        except Exception:
+            log.warning("AI file cleanup failed", exc_info=True)

celerp/ai/commands.py

@@ -0,0 +1,184 @@
+# Copyright (c) 2026 Noah Severs
+# SPDX-License-Identifier: BSL-1.1
+
+"""AI command parsing and execution - structured bill creation from LLM output.
+
+Two-phase flow:
+  1. parse_bill_commands() - validate LLM JSON, return structured data
+  2. create_bills() - execute against event store (only after user confirms)
+"""
+
+from __future__ import annotations
+
+import hashlib
+import logging
+import re
+import uuid
+
+from pydantic import BaseModel, field_validator
+from sqlalchemy import cast, select, String
+from sqlalchemy.ext.asyncio import AsyncSession
+
+log = logging.getLogger(__name__)
+
+
+# -- Pydantic models -------------------------------------------------------
+
+class LineItem(BaseModel):
+    description: str
+    quantity: float
+    unit_price: float
+
+    @field_validator("quantity")
+    @classmethod
+    def qty_positive(cls, v: float) -> float:
+        if v <= 0:
+            raise ValueError("quantity must be > 0")
+        return v
+
+    @field_validator("unit_price")
+    @classmethod
+    def price_non_negative(cls, v: float) -> float:
+        if v < 0:
+            raise ValueError("unit_price must be >= 0")
+        return v
+
+
+class DraftBill(BaseModel):
+    vendor_name: str
+    date: str
+    total: float
+    source_file_id: str | None = None
+    line_items: list[LineItem]
+
+
+# -- Phase 1: Parse + validate ---------------------------------------------
+
+def parse_bill_commands(raw_json: dict) -> list[DraftBill]:
+    """Validate LLM JSON output into structured bill data.
+
+    Raises ValueError with details on validation failure.
+    """
+    bills_data = raw_json.get("create_draft_bills", [])
+    if not bills_data:
+        return []
+    bills: list[DraftBill] = []
+    for i, entry in enumerate(bills_data):
+        try:
+            bills.append(DraftBill.model_validate(entry))
+        except Exception as exc:
+            raise ValueError(f"Bill #{i + 1} validation failed: {exc}") from exc
+    return bills
+
+
+# -- Phase 2: Execute (after user confirmation) ----------------------------
+
+async def create_bills(
+    session: AsyncSession,
+    company_id: uuid.UUID,
+    user_id: uuid.UUID,
+    bills: list[DraftBill],
+) -> str:
+    """Create draft bills and contacts in the event store. Returns feedback text."""
+    from celerp.events.engine import emit_event
+    from celerp.models.company import Company
+    from celerp.models.projections import Projection
+
+    company = await session.get(Company, company_id)
+    if not company:
+        return ""
+
+    feedback_lines: list[str] = []
+    currency = company.settings.get("currency", "USD")
+
+    for bill in bills:
+        # Find vendor in projection
+        vendor_row = (await session.execute(
+            select(Projection).where(
+                Projection.company_id == company_id,
+                Projection.entity_type == "contact",
+                cast(Projection.state["name"], String).ilike(f"%{bill.vendor_name}%"),
+            )
+        )).scalars().first()
+
+        if vendor_row:
+            contact_id = str(vendor_row.entity_id)
+        else:
+            slug = re.sub(r"[^a-zA-Z0-9]+", "-", bill.vendor_name.lower()) or "vendor"
+            ref_id = f"{slug}-{str(uuid.uuid4())[:4]}"
+            contact_id = f"contact:{ref_id}"
+            idem_key = _idempotency_key(company_id, "contact", bill.vendor_name, bill.date)
+            await emit_event(
+                session,
+                company_id=company_id,
+                entity_id=contact_id,
+                entity_type="contact",
+                event_type="contact.created",
+                data={
+                    "name": bill.vendor_name,
+                    "contact_type": "vendor",
+                    "status": "draft",
+                    "ref_id": ref_id,
+                    "currency": currency,
+                },
+                actor_id=user_id,
+                location_id=None,
+                source="ai",
+                idempotency_key=idem_key,
+                metadata_={},
+            )
+
+        try:
+            from celerp_docs.sequences import next_doc_ref
+            bill_ref = next_doc_ref(company, "bill")
+        except ImportError:
+            bill_ref = f"BIL-{str(uuid.uuid4())[:6].upper()}"
+
+        entity_id = f"doc:{bill_ref}"
+        line_items = [
+            {
+                "description": li.description,
+                "quantity": li.quantity,
+                "unit_price": li.unit_price,
+                "line_total": li.quantity * li.unit_price,
+            }
+            for li in bill.line_items
+        ]
+        idem_key = _idempotency_key(company_id, "bill", bill.vendor_name, bill.date, str(bill.total))
+
+        await emit_event(
+            session,
+            company_id=company_id,
+            entity_id=entity_id,
+            entity_type="doc",
+            event_type="doc.created",
+            data={
+                "doc_type": "bill",
+                "status": "draft",
+                "ref_id": bill_ref,
+                "date": bill.date,
+                "contact_id": contact_id,
+                "location_id": "loc:default",
+                "total": bill.total,
+                "subtotal": bill.total,
+                "currency": currency,
+                "amount_outstanding": bill.total,
+                "line_items": line_items,
+            },
+            actor_id=user_id,
+            location_id=None,
+            source="ai",
+            idempotency_key=idem_key,
+            metadata_={"ai_source_file_id": bill.source_file_id} if bill.source_file_id else {},
+        )
+        feedback_lines.append(
+            f"Created Draft Bill {bill_ref} for {bill.vendor_name} ({currency} {bill.total:.2f})"
+        )
+
+    return "\n".join(feedback_lines)
+
+
+def _idempotency_key(company_id: uuid.UUID, entity: str, *parts: str) -> str:
+    """Deterministic idempotency key from company + entity + parts."""
+    raw = f"{company_id}:{entity}:" + ":".join(parts)
+    return f"ai_{entity}_{hashlib.sha256(raw.encode()).hexdigest()[:12]}"