celerp 2.3.3__tar.gz → 2.3.5__tar.gz

{celerp-2.3.3 → celerp-2.3.5}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: celerp
-Version: 2.3.3
+Version: 2.3.5
 Summary: Free, self-hosted ERP for small businesses
 License: Business Source License 1.1
         

celerp-2.3.5/celerp/capacity.py

@@ -0,0 +1,22 @@
+# Copyright (c) 2026 Noah Severs
+# SPDX-License-Identifier: BUSL-1.1
+"""Request-capacity constants shared by the API pool and the local UI transport.
+
+The app launches one uvicorn worker (both `celerp start` and the Electron shell
+run `uvicorn celerp.main:app` without `--workers`), so one API process owns one
+request connection pool. These constants are the single source of truth for that
+pool's size and for how many connections the web UI is allowed to open into that
+process, so the two can never drift apart. The base size is the interactive
+ceiling; the overflow is held back for background work, relay, auth, and bursts.
+
+This module has no side effects and imports nothing from the app, so both the
+database engine and the UI transport can read it without an import cycle.
+"""
+from __future__ import annotations
+
+# Base connections the single API request pool keeps open, and the extra it may
+# open under burst before refusing. The web UI's interactive transport uses the
+# base size as its own connection ceiling, so a page fan-out can never demand
+# more connections than the pool holds for interactive work.
+REQUEST_DB_POOL_SIZE = 10
+REQUEST_DB_MAX_OVERFLOW = 5

{celerp-2.3.3 → celerp-2.3.5}/celerp/db.py

@@ -9,6 +9,7 @@ from sqlalchemy import text as _sql_text
 from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker, create_async_engine
 from sqlalchemy.pool import NullPool
 
+from celerp.capacity import REQUEST_DB_MAX_OVERFLOW, REQUEST_DB_POOL_SIZE
 from celerp.config import settings
 
 # Shared advisory-lock key that serialises schema migrations across every process
@@ -33,11 +34,13 @@ def mask_db_credentials(text: str) -> str:
 _REQUEST_LOCK_TIMEOUT_MS = "3000"
 _REQUEST_STATEMENT_TIMEOUT_MS = "30000"
 
-# Pool budget: total_possible = (api_workers * (pool_size + max_overflow))
-#                              + (gui_workers * (gui_pool_size + gui_max_overflow))
-# Default workers 2 API + 1 GUI → 2*(10+5) + 1*(5+5) = 40 connections max.
-# Postgres default max_connections=100 leaves 60 for migrations, admin tools, etc.
-# If you increase worker counts, recalculate this budget before deploying.
+# Pool budget: the app runs one API worker (both `celerp start` and the Electron
+# shell launch uvicorn without --workers), so this single process owns one request
+# pool of REQUEST_DB_POOL_SIZE base connections plus REQUEST_DB_MAX_OVERFLOW under
+# burst. The base size is also the web UI's interactive connection ceiling, kept in
+# lockstep through celerp.capacity so the two never drift. Postgres default
+# max_connections=100 leaves ample headroom for migrations, admin tools, and
+# background jobs beyond this pool.
 if os.environ.get("CELERP_TEST_NULLPOOL"):
     # Test mode: a fresh connection per use that closes on return, so a failed
     # test can't leave a poisoned/locked connection lingering in a pooled
@@ -49,8 +52,8 @@ else:
         settings.database_url,
         future=True,
         pool_pre_ping=True,
-        pool_size=10,
-        max_overflow=5,
+        pool_size=REQUEST_DB_POOL_SIZE,
+        max_overflow=REQUEST_DB_MAX_OVERFLOW,
         # Bound how long any query may wait on a lock or run, so a stuck query is
         # cancelled instead of pinning one of the few pooled connections for the
         # full request lifetime (lock_timeout 3s, statement_timeout 30s, in ms).

{celerp-2.3.3 → celerp-2.3.5}/celerp/gateway/client.py

@@ -26,7 +26,6 @@ from websockets.exceptions import ConnectionClosed
 
 log = logging.getLogger(__name__)
 
-_PING_INTERVAL = 30   # seconds
 _BACKOFF_MAX = 60     # seconds
 # Consecutive auth rejections before the client stops retrying: a rejected
 # credential never heals on its own, so beyond a small allowance for a
@@ -37,6 +36,10 @@ _AUTH_FAILED_MAX = 3
 # a revoke-then-reshare so it does not flap connect/disconnect).
 _REAP_INTERVAL = 15 * 60   # seconds
 _REAP_GRACE = 5 * 60       # seconds
+# A single websocket send that cannot be handed to the transport within this long means the
+# socket is wedged (a stalled or backpressured peer). Bound every send so it surfaces as an
+# error instead of pinning the proxy task holding a relay slot forever.
+_SEND_DEADLINE = 30        # seconds
 
 
 def _shop_key(handle: str | None) -> str:
@@ -85,6 +88,22 @@ class GatewayClient:
         # Hold strong refs to fire-and-forget tasks so the loop can't GC them mid-run
         # (a dropped task = a lost proxy response or webhook sync).
         self._bg_tasks: set = set()
+        # In-flight proxy tasks keyed by relay request id so an http.cancel frame can
+        # abort exactly the one request it names, and no other.
+        self._inflight: dict[str, asyncio.Task] = {}
+        # Connection generation: bumped on every teardown so a response produced by a
+        # request that outlived its socket can be recognised as stale and dropped
+        # rather than sent on the freshly rebuilt connection.
+        self._generation: int = 0
+        # One shared httpx client for all proxied requests, built lazily with bounded
+        # connection limits. A fresh client per request opened an unbounded number of
+        # pools under load; one bounded client caps the concurrency into the local app.
+        self._http_client: Any = None
+        # Observability counters for the proxy path, surfaced when the connection ends.
+        self._proxied_count = 0
+        self._cancelled_count = 0
+        self._timeout_count = 0
+        self._stale_dropped_count = 0
         # Resolve local server ports for proxy routing.
         # In Electron builds the ports are dynamic; Electron passes them via
         # CELERP_API_PORT / CELERP_UI_PORT env vars so we don't rely on the
@@ -122,6 +141,22 @@ class GatewayClient:
         self._bg_tasks.add(task)
         task.add_done_callback(self._bg_tasks.discard)
 
+    def _spawn_proxy(self, payload: dict) -> None:
+        """Run a proxied request fire-and-forget, keyed by its relay request id so an
+        http.cancel frame can abort exactly that task. The key is cleared when the task
+        finishes (a cancel arriving after completion is then a harmless no-op)."""
+        request_id = payload.get("id", "")
+        task = asyncio.create_task(self._handle_proxy_request(payload))
+        self._bg_tasks.add(task)
+        self._inflight[request_id] = task
+
+        def _done(t: asyncio.Task) -> None:
+            self._bg_tasks.discard(t)
+            if self._inflight.get(request_id) is t:
+                del self._inflight[request_id]
+
+        task.add_done_callback(_done)
+
     # ── Public API ─────────────────────────────────────────────────────
 
     async def run(self) -> None:
@@ -198,6 +233,7 @@ class GatewayClient:
                 await ws.close()
             except Exception:
                 pass
+        await self._close_http_client()
 
     async def _should_reap(self) -> bool:
         """A free instance with no live share and no recent proxied request should
@@ -293,40 +329,61 @@ class GatewayClient:
             import ssl as _ssl
             _ssl_ctx = _ssl.create_default_context()
             _ssl_ctx.set_alpn_protocols(["http/1.1"])
-        async with websockets.connect(
-            self._url, ssl=_ssl_ctx, ping_interval=20, ping_timeout=20, max_size=160 * 1024 * 1024
-        ) as ws:
-            self._ws = ws
-            # Read current TOS version from config
-            from celerp.config import read_config
-            cfg = read_config() or {}
-            tos_version = cfg.get("cloud", {}).get("tos_version", "")
-            # App version: the Electron wrapper passes the release version via
-            # CELERP_APP_VERSION; fall back to the package version. Sent on every
-            # connect so the handshake reflects the build that is actually running.
-            from celerp import __version__ as _pkg_version
-            app_version = os.environ.get("CELERP_APP_VERSION") or _pkg_version
-            # Send hello handshake
-            await self._send(ws, {
-                "type": "hello",
-                "id": str(uuid.uuid4()),
-                "payload": {
-                    "gateway_token": self._token,
-                    "instance_id": self._instance_id,
-                    "tos_version": tos_version,
-                    "version": app_version,
-                },
-            })
-            # Message dispatch loop
-            async for raw in ws:
-                try:
-                    msg = json.loads(raw)
-                except json.JSONDecodeError:
-                    log.warning("Gateway sent non-JSON frame: %r", raw)
-                    continue
-                await self._dispatch(msg)
-        self._ws = None
-        self._set_status("inactive")
+        try:
+            async with websockets.connect(
+                self._url, ssl=_ssl_ctx, ping_interval=20, ping_timeout=20, max_size=160 * 1024 * 1024
+            ) as ws:
+                self._ws = ws
+                # Read current TOS version from config
+                from celerp.config import read_config
+                cfg = read_config() or {}
+                tos_version = cfg.get("cloud", {}).get("tos_version", "")
+                # App version: the Electron wrapper passes the release version via
+                # CELERP_APP_VERSION; fall back to the package version. Sent on every
+                # connect so the handshake reflects the build that is actually running.
+                from celerp import __version__ as _pkg_version
+                app_version = os.environ.get("CELERP_APP_VERSION") or _pkg_version
+                # Send hello handshake
+                await self._send(ws, {
+                    "type": "hello",
+                    "id": str(uuid.uuid4()),
+                    "payload": {
+                        "gateway_token": self._token,
+                        "instance_id": self._instance_id,
+                        "tos_version": tos_version,
+                        "version": app_version,
+                    },
+                })
+                # Message dispatch loop
+                async for raw in ws:
+                    try:
+                        msg = json.loads(raw)
+                    except json.JSONDecodeError:
+                        log.warning("Gateway sent non-JSON frame: %r", raw)
+                        continue
+                    await self._dispatch(msg)
+        finally:
+            # Teardown runs on every exit - clean end of the message stream OR an
+            # abnormal disconnect raised mid-serve. Kept in a finally so an exception
+            # from the async-for can't leave a dangling socket, an unfenced generation,
+            # and a stuck status behind.
+            self._ws = None
+            # A new generation for the next connection: any proxy response still in
+            # flight from this socket is now stale and must not be sent on the rebuilt
+            # one. Cancel the in-flight proxy tasks that belong to the dead connection -
+            # their responses can never be delivered on a socket that no longer exists.
+            self._generation += 1
+            for task in list(self._inflight.values()):
+                if not task.done():
+                    task.cancel()
+            self._inflight.clear()
+            log.debug(
+                "Gateway proxy activity this connection: proxied=%d cancelled=%d "
+                "timed_out=%d stale_dropped=%d",
+                self._proxied_count, self._cancelled_count,
+                self._timeout_count, self._stale_dropped_count,
+            )
+            self._set_status("inactive")
 
     async def _dispatch(self, msg: dict) -> None:
         msg_type = msg.get("type", "")
@@ -414,7 +471,13 @@ class GatewayClient:
             set_subscription_state(tier, status)
 
         elif msg_type == "http.request":
-            self._spawn(self._handle_proxy_request(payload))
+            self._spawn_proxy(payload)
+
+        elif msg_type == "http.cancel":
+            request_id = payload.get("id", "")
+            task = self._inflight.get(request_id)
+            if task is not None and not task.done():
+                task.cancel()
 
         elif msg_type == "shopify.webhook":
             self._spawn(self._handle_shopify_webhook(payload))
@@ -533,6 +596,31 @@ class GatewayClient:
         except Exception as exc:
             log.warning("invoice.payment handling failed (entity=%s): %s", entity_id, exc)
 
+    def _get_http_client(self):
+        """Return the one shared httpx client used for every proxied request, building
+        it lazily on first use. A bounded connection pool caps how many concurrent
+        requests this client can drive into the local app, so a burst of relayed
+        requests can no longer open an unbounded number of pools. The default timeout
+        is the fixed cap; a per-request timeout_ms tightens it further."""
+        import httpx
+
+        if self._http_client is None:
+            self._http_client = httpx.AsyncClient(
+                timeout=180.0,
+                limits=httpx.Limits(max_connections=32, max_keepalive_connections=8),
+            )
+        return self._http_client
+
+    async def _close_http_client(self) -> None:
+        """Close the shared httpx client if one was built. Safe to call when none was."""
+        client = self._http_client
+        self._http_client = None
+        if client is not None:
+            try:
+                await client.aclose()
+            except Exception:
+                pass
+
     async def _handle_proxy_request(self, payload: dict) -> None:
         """Handle a proxied HTTP request from the relay.
 
@@ -545,10 +633,14 @@ class GatewayClient:
         local access.
         """
         import base64
-        import httpx
 
         # Activity stamp for the reaper's grace window: any proxied request counts.
         self._last_request_monotonic = time.monotonic()
+        self._proxied_count += 1
+        # The generation this request belongs to: if the socket is rebuilt before the
+        # response is ready, self._generation advances past this and the response is
+        # dropped rather than sent on a connection that never issued the request.
+        generation = self._generation
 
         request_id = payload.get("id", "")
         method = payload.get("method", "GET")
@@ -557,6 +649,13 @@ class GatewayClient:
         headers = payload.get("headers", {})
         body_b64 = payload.get("body_b64", "")
         body = base64.b64decode(body_b64) if body_b64 else None
+        # Additive per-request deadline: a numeric timeout_ms bounds this request below
+        # the fixed cap so a single slow local handler can't pin a relay slot for 180s.
+        # Absent or non-numeric leaves the default cap in force.
+        timeout_ms = payload.get("timeout_ms")
+        deadline_s: float | None = None
+        if isinstance(timeout_ms, (int, float)) and not isinstance(timeout_ms, bool) and timeout_ms > 0:
+            deadline_s = timeout_ms / 1000.0
 
         # SSE / long-poll paths cannot be proxied over the WS request/response
         # protocol. Return an empty stream so the browser doesn't 500.
@@ -599,47 +698,98 @@ class GatewayClient:
         if query:
             url = f"{url}?{query}"
 
+        client = self._get_http_client()
         try:
-            async with httpx.AsyncClient(timeout=180.0) as client:
+            if deadline_s is not None:
+                # asyncio.wait_for, not asyncio.timeout: the latter is 3.11+, and this package
+                # supports Python >=3.10 where it does not exist. wait_for cancels the request and
+                # raises TimeoutError on the deadline, which the handler below turns into an error
+                # response.
+                resp = await asyncio.wait_for(
+                    client.request(
+                        method=method,
+                        url=url,
+                        headers=headers,
+                        content=body,
+                    ),
+                    deadline_s,
+                )
+            else:
                 resp = await client.request(
                     method=method,
                     url=url,
                     headers=headers,
                     content=body,
                 )
-            resp_body_b64 = base64.b64encode(resp.content).decode() if resp.content else ""
-            # Serialize as an ORDERED LIST of pairs (not a dict): a response can carry several
-            # Set-Cookie headers (login/refresh emit access_token + refresh_token), and a dict would
-            # collapse them into one comma-joined value the browser can't parse — so the refresh
-            # cookie never reaches the browser and the session dies at the access token's hard cap.
-            # multi_items() keeps each header separate. Drop content-length; the relay recomputes it.
-            _skip = {"transfer-encoding", "connection", "keep-alive", "content-length"}
-            resp_headers = [
-                [k, v] for k, v in resp.headers.multi_items()
-                if k.lower() not in _skip
-            ]
-            await self._send(self._ws, {
-                "type": "http.response",
-                "payload": {
-                    "id": request_id,
-                    "status": resp.status_code,
-                    "headers": resp_headers,
-                    "body_b64": resp_body_b64,
-                },
-            })
+        except asyncio.CancelledError:
+            # http.cancel aborted this request: emit nothing and let the cancellation
+            # propagate so the task is recorded cancelled.
+            self._cancelled_count += 1
+            raise
+        except TimeoutError:
+            # The per-request deadline elapsed before the local handler answered. Surface
+            # it as an error response instead of hanging to the fixed cap.
+            self._timeout_count += 1
+            log.warning("Proxy request timed out after %sms for %s %s", timeout_ms, method, path)
+            if generation == self._generation:
+                await self._send(self._ws, {
+                    "type": "http.response",
+                    "payload": {
+                        "id": request_id,
+                        "status": 504,
+                        "headers": [["content-type", "text/plain"]],
+                        "body_b64": base64.b64encode(b"Local app timed out").decode(),
+                    },
+                })
+            else:
+                self._stale_dropped_count += 1
+            return
         except Exception as exc:
             log.warning("Proxy request failed for %s %s: %s", method, path, exc)
-            await self._send(self._ws, {
-                "type": "http.response",
-                "payload": {
-                    "id": request_id,
-                    "status": 502,
-                    "headers": [["content-type", "text/plain"]],
-                    "body_b64": base64.b64encode(
-                        f"Local app error: {type(exc).__name__}: {exc}".encode()
-                    ).decode(),
-                },
-            })
+            if generation == self._generation:
+                await self._send(self._ws, {
+                    "type": "http.response",
+                    "payload": {
+                        "id": request_id,
+                        "status": 502,
+                        "headers": [["content-type", "text/plain"]],
+                        "body_b64": base64.b64encode(
+                            f"Local app error: {type(exc).__name__}: {exc}".encode()
+                        ).decode(),
+                    },
+                })
+            else:
+                self._stale_dropped_count += 1
+            return
+
+        # The socket that carried this request may have been rebuilt while it was in
+        # flight. A response from a superseded generation belongs to a connection the
+        # relay no longer has; sending it on the new socket would answer the wrong
+        # request, so it is dropped.
+        if generation != self._generation:
+            self._stale_dropped_count += 1
+            return
+
+        resp_body_b64 = base64.b64encode(resp.content).decode() if resp.content else ""
+        # Serialize as an ORDERED LIST of pairs (not a dict): a response can carry several
+        # Set-Cookie headers (login/refresh emit access_token + refresh_token), and a dict would
+        # collapse them into one comma-joined value the browser can't parse, so the refresh
+        # cookie never reaches the browser and the session dies at the access token's hard cap.
+        # multi_items() keeps each header separate. Drop content-length; the relay recomputes it.
+        _skip = {"transfer-encoding", "connection", "keep-alive", "content-length"}
+        resp_headers = [
+            [k, v] for k, v in resp.headers.multi_items()
+            if k.lower() not in _skip
+        ]
+        await self._send(self._ws, {
+            "type": "http.response",
+            "payload": {
+                "id": request_id,
+                "status": resp.status_code,
+                "headers": resp_headers,
+                "body_b64": resp_body_b64,
+            },
+        })
 
     async def send_message(self, msg_type: str, **payload) -> None:
         """Send a JSON message over the active WS connection.
@@ -652,7 +802,12 @@ class GatewayClient:
 
     @staticmethod
     async def _send(ws, message: dict) -> None:
-        await ws.send(json.dumps(message))
+        # Bound the send: a websocket send awaits until the frame is handed to the transport, and a
+        # stalled or backpressured peer would otherwise block it forever, pinning the proxy task and
+        # its relay slot. On the deadline the send is cancelled and TimeoutError surfaces to the
+        # caller, which lets the wedged socket be torn down and rebuilt. asyncio.wait_for, not
+        # asyncio.timeout: the latter is 3.11+ and this package supports Python >=3.10.
+        await asyncio.wait_for(ws.send(json.dumps(message)), _SEND_DEADLINE)
 
 
 # Module-level singleton — set by main.py lifespan

{celerp-2.3.3 → celerp-2.3.5}/celerp/main.py

@@ -26,6 +26,7 @@ from fastapi.staticfiles import StaticFiles
 from celerp.routers import auth, companies, ledger
 from celerp.routers import health, notifications, system, events as events_router_mod
 from celerp.routers import stars as stars_router_mod
+from celerp.routers import search as search_router_mod
 
 import celerp.models  # noqa: F401 - ensures kernel models (UserCompany, ImportBatch, DocShareToken) are registered
 
@@ -466,6 +467,7 @@ app.include_router(system.router, prefix="/system", tags=["system"])
 app.include_router(stars_router_mod.router, prefix="/stars", tags=["stars"])
 app.include_router(notifications.router)
 app.include_router(events_router_mod.router)
+app.include_router(search_router_mod.router, tags=["search"])
 
 # Backup router — always registered; individual endpoints gate on cloud connection.
 # celerp_backup lives in default_modules/celerp-backup/ which is not on sys.path