amfs-http-server 0.3.39__tar.gz → 0.3.41__tar.gz

{amfs_http_server-0.3.39 → amfs_http_server-0.3.41}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: amfs-http-server
-Version: 0.3.39
+Version: 0.3.41
 Summary: AMFS HTTP/REST API server with SSE support
 License-Expression: Apache-2.0
 Requires-Python: >=3.11
@@ -9,6 +9,6 @@ Requires-Dist: amfs>=0.5.27
 Requires-Dist: fastapi>=0.115
 Requires-Dist: httpx>=0.27
 Requires-Dist: sse-starlette>=2.0
-Requires-Dist: uvicorn[standard]>=0.32
+Requires-Dist: uvicorn[standard]>=0.37
 Provides-Extra: pro
 Requires-Dist: amfs-patterns; extra == 'pro'

{amfs_http_server-0.3.39 → amfs_http_server-0.3.41}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "amfs-http-server"
-version = "0.3.39"
+version = "0.3.41"
 description = "AMFS HTTP/REST API server with SSE support"
 requires-python = ">=3.11"
 license = "Apache-2.0"
@@ -29,7 +29,10 @@ dependencies = [
     # agent's private keys. Either way the floor is what makes the rule real.
     "amfs-core>=0.3.37",
     "fastapi>=0.115",
-    "uvicorn[standard]>=0.32",
+    # 0.37 is where uvicorn.run() grew timeout_worker_healthcheck, which
+    # main() passes unconditionally; on anything older that is a TypeError
+    # before the first socket is bound, one worker or twenty.
+    "uvicorn[standard]>=0.37",
     "sse-starlette>=2.0",
     "httpx>=0.27",
 ]

{amfs_http_server-0.3.39 → amfs_http_server-0.3.41}/src/amfs_http/server.py

@@ -134,8 +134,19 @@ _async_adapter = None  # AsyncPostgresAdapter | None, set in lifespan
 
 @asynccontextmanager
 async def _lifespan(application: FastAPI):  # noqa: ARG001
-    """Open/close the async connection pool for hot-path DB access."""
-    global _async_adapter
+    """Open/close the async connection pool for hot-path DB access, and run
+    the embedded Cortex worker for the life of this process.
+
+    The Cortex worker starts here rather than in ``main()`` because this is the
+    hook that runs once in every process that serves requests. With
+    ``--workers`` above one, uvicorn imports ``amfs_http.server:app`` afresh
+    in each child; ``main()`` runs only in the supervisor, so a worker started
+    there is invisible to the ``/api/v1/cortex/*`` routes in the children —
+    they read the module global and saw ``None``. Starting it per process
+    keeps those routes truthful and costs one LISTEN connection and a
+    two-connection pool per worker instead of per instance.
+    """
+    global _async_adapter, _cortex_worker
     dsn = os.environ.get("AMFS_POSTGRES_DSN")
     if dsn and not os.environ.get("AMFS_HTTP_URL"):
         try:
@@ -147,12 +158,24 @@ async def _lifespan(application: FastAPI):  # noqa: ARG001
         except Exception:
             logger.warning("Failed to start async adapter — falling back to sync", exc_info=True)
             _async_adapter = None
+    if _env_flag("AMFS_WITH_CORTEX"):
+        _start_embedded_cortex()
     yield
+    if _cortex_worker is not None:
+        try:
+            _cortex_worker.stop()
+        except Exception:  # noqa: BLE001 - shutting down; nothing to do about it
+            logger.debug("Cortex worker stop raised", exc_info=True)
     if _async_adapter is not None:
         await _async_adapter.close()
         logger.info("Async Postgres adapter closed")
 
 
+def _env_flag(name: str) -> bool:
+    """Whether environment variable *name* is set to a truthy value."""
+    return (os.environ.get(name) or "").strip().lower() in {"1", "true", "yes", "on"}
+
+
 app = FastAPI(
     title="AMFS HTTP API",
     description="Agent Memory File System — REST API with SSE support",
@@ -233,8 +256,72 @@ _bg_executor = ThreadPoolExecutor(max_workers=8, thread_name_prefix="amfs-bg")
 # checkouts from the sync pool, not a throughput target. ``_offload`` copies
 # the calling context into the thread so the tenant ContextVars the RLS pool
 # reads are the request's, not the thread's leftovers.
+#
+# Routes whose whole body is synchronous — the admin, team, API-key and
+# pattern routes that open ``pool.connection()`` and return — are plain
+# ``def``, on purpose. FastAPI runs a ``def`` endpoint on the anyio worker
+# pool, with the caller's contextvars copied in, so the body needs no
+# ``_offload`` wrapping; declared ``async`` the same body holds the loop for
+# the length of the query, and for however long the pool checkout has to wait
+# when the pool is busy. Do not "fix" one back to ``async def`` unless it
+# gains an ``await``.
+
+
+def _effective_cpu_count() -> int:
+    """The CPUs this process may actually use, not the ones the host has.
+
+    ``os.cpu_count()`` reports the machine. In a container it is the node's
+    core count, while the scheduler holds the process to whatever the cgroup
+    quota or affinity mask says — on Cloud Run a 2-vCPU service can read 8 or
+    more here. Sizing a thread pool for the host's cores then puts several
+    ONNX threads on every core the container is allowed, which is the thrash
+    the pool was sized to avoid.
+
+    Takes the smallest of: the affinity mask (Linux), the cgroup v2 quota
+    (``cpu.max``, ``quota/period`` rounded up), the cgroup v1 quota, and
+    ``os.cpu_count()``. ``AMFS_MODEL_THREADS`` overrides all of it for the
+    deployment that knows better. Never below one.
+    """
+    override = os.environ.get("AMFS_MODEL_THREADS")
+    if override:
+        try:
+            return max(1, int(override))
+        except ValueError:
+            logger.warning("AMFS_MODEL_THREADS=%r is not an integer; ignoring", override)
+
+    candidates: list[int] = []
+    host = os.cpu_count()
+    if host:
+        candidates.append(host)
+    try:
+        candidates.append(len(os.sched_getaffinity(0)))  # type: ignore[attr-defined]
+    except (AttributeError, OSError):
+        pass
+    for quota_path, period_path in (
+        ("/sys/fs/cgroup/cpu.max", None),
+        ("/sys/fs/cgroup/cpu/cpu.cfs_quota_us", "/sys/fs/cgroup/cpu/cpu.cfs_period_us"),
+    ):
+        try:
+            with open(quota_path) as fh:
+                first = fh.read().split()
+            if period_path is None:
+                quota_s, period_s = first[0], first[1]
+            else:
+                quota_s = first[0]
+                with open(period_path) as fh:
+                    period_s = fh.read().split()[0]
+            if quota_s in ("max", "-1"):
+                continue
+            quota, period = int(quota_s), int(period_s)
+            if quota > 0 and period > 0:
+                candidates.append(max(1, -(-quota // period)))
+        except (OSError, ValueError, IndexError):
+            continue
+    return max(1, min(candidates) if candidates else 2)
+
+
 _model_executor = ThreadPoolExecutor(
-    max_workers=max(1, os.cpu_count() or 2), thread_name_prefix="amfs-model"
+    max_workers=_effective_cpu_count(), thread_name_prefix="amfs-model"
 )
 _db_executor = ThreadPoolExecutor(max_workers=8, thread_name_prefix="amfs-db")
 
@@ -248,6 +335,19 @@ async def _offload(executor: ThreadPoolExecutor, fn: Any, /, *args: Any, **kwarg
     )
 
 
+def _sync_search(adapter: Any, sq: Any, branch: Any) -> list[Any]:
+    """``adapter.search`` for adapters with and without a ``branch`` parameter.
+
+    Meant to run on ``_db_executor`` via ``_offload``; the sync search is the
+    fallback the async path takes when it returns nothing, and it must not
+    hold the loop any more than the primary path does.
+    """
+    try:
+        return adapter.search(sq, branch=branch)
+    except TypeError:
+        return adapter.search(sq)
+
+
 _known_agents: set[str] = set()
 # Tracks (agent, namespace, user) triples whose owner linkage was already
 # upserted, so the hot write path doesn't repeat the DB call. Kept separate
@@ -604,8 +704,28 @@ except ImportError:
     from amfs_http.pro_proxy import mount_pro_proxy
     mount_pro_proxy(app)
 
+_memory_lock = threading.Lock()
+
+
 def _get_memory() -> AgentMemory:
-    """Lazily initialise the shared AgentMemory singleton."""
+    """Return the shared AgentMemory singleton, building it on first call.
+
+    Double-checked under a lock: the sync-bodied routes run on the threadpool,
+    so the first requests after a worker starts — a dashboard load fires
+    several at once — can arrive here together, and without the lock each
+    would build its own ``AgentMemory`` and adapter pool, with every loser
+    dropped unclosed. The fast path stays lock-free.
+    """
+    if _memory is not None:
+        return _memory
+    with _memory_lock:
+        if _memory is not None:
+            return _memory
+        return _build_memory()
+
+
+def _build_memory() -> AgentMemory:
+    """Construct the shared AgentMemory. Call through ``_get_memory``."""
     global _memory
     if _memory is not None:
         return _memory
@@ -1635,7 +1755,7 @@ async def _read_entry(
 
 
 @app.get("/api/v1/quality/{entity_path:path}/{key}")
-async def entry_quality(
+def entry_quality(
     request: Request,
     entity_path: str,
     key: str,
@@ -1736,11 +1856,10 @@ async def _scope_block(
     vis = _get_visibility_filter(request) if request is not None else None
     if vis is not None and vis.should_filter():
         try:
-            visible = {
-                e.key for e in vis.filter_entries(
-                    _get_memory()._adapter.list(entity_path, branch=branch)
-                )
-            }
+            listed = await _offload(
+                _db_executor, _get_memory()._adapter.list, entity_path, branch=branch
+            )
+            visible = {e.key for e in vis.filter_entries(listed)}
             counts = {**counts, "keys": [k for k in counts.get("keys", []) if k in visible]}
         except Exception:  # noqa: BLE001 - drop the sample rather than leak it
             counts = {**counts, "keys": []}
@@ -2404,10 +2523,9 @@ async def search_entries(
             logger.warning("Async search failed — falling back to sync", exc_info=True)
             results = []
         if not results:
-            try:
-                sync_results = mem._adapter.search(sq, branch=branch)
-            except TypeError:
-                sync_results = mem._adapter.search(sq)
+            # Runs on every search that finds nothing, so it goes off the loop
+            # like the primary path did.
+            sync_results = await _offload(_db_executor, _sync_search, mem._adapter, sq, branch)
             if sync_results:
                 logger.warning(
                     "Async search returned 0 results but sync found %d — RLS context mismatch",
@@ -2415,10 +2533,7 @@ async def search_entries(
                 )
                 results = sync_results
     else:
-        try:
-            results = mem._adapter.search(sq, branch=branch)
-        except TypeError:
-            results = mem._adapter.search(sq)
+        results = await _offload(_db_executor, _sync_search, mem._adapter, sq, branch)
 
     vis = _get_visibility_filter(request)
     if vis is not None and vis.should_filter():
@@ -2593,9 +2708,7 @@ async def retrieve_entries(
     if not candidates:
         mem = _get_memory()
         try:
-            results = mem._adapter.search(sq_lex, branch=branch)
-        except TypeError:
-            results = mem._adapter.search(sq_lex)
+            results = await _offload(_db_executor, _sync_search, mem._adapter, sq_lex, branch)
         except Exception:
             results = []
         for entry in results:
@@ -5076,7 +5189,7 @@ Return ONLY valid JSON, no markdown formatting."""
 
 
 @app.get("/api/v1/admin/usage")
-async def get_usage(
+def get_usage(
     request: Request,
     _auth: str | None = Depends(verify_api_key),
 ) -> dict[str, Any]:
@@ -6618,7 +6731,7 @@ def _audit_log(
 
 
 @app.get("/api/v1/admin/api-keys")
-async def list_api_keys(
+def list_api_keys(
     request: Request,
     _auth: str | None = Depends(verify_api_key),
 ) -> dict[str, Any]:
@@ -6675,7 +6788,7 @@ async def list_api_keys(
 
 
 @app.post("/api/v1/admin/api-keys")
-async def create_api_key(
+def create_api_key(
     req: CreateAPIKeyRequest,
     request: Request,
     _auth: str | None = Depends(verify_api_key),
@@ -6729,7 +6842,7 @@ async def create_api_key(
 
 
 @app.delete("/api/v1/admin/api-keys/{key_id}")
-async def revoke_api_key(
+def revoke_api_key(
     key_id: str,
     request: Request,
     _auth: str | None = Depends(verify_api_key),
@@ -6776,7 +6889,7 @@ async def revoke_api_key(
 
 
 @app.get("/api/v1/admin/audit")
-async def list_audit_log(
+def list_audit_log(
     request: Request,
     action: str | None = Query(None),
     limit: int = Query(200),
@@ -6876,7 +6989,7 @@ async def list_patterns(
 
 
 @app.get("/api/v1/admin/teams")
-async def list_teams(
+def list_teams(
     request: Request,
     _auth: str | None = Depends(verify_api_key),
 ) -> dict[str, Any]:
@@ -6919,7 +7032,7 @@ async def list_teams(
 
 
 @app.post("/api/v1/admin/teams")
-async def create_team(
+def create_team(
     req: CreateTeamRequest,
     request: Request,
     _auth: str | None = Depends(verify_api_key),
@@ -6954,7 +7067,7 @@ async def create_team(
 
 
 @app.patch("/api/v1/admin/teams/{team_id}")
-async def update_team(
+def update_team(
     team_id: str,
     req: UpdateTeamRequest,
     request: Request,
@@ -7009,7 +7122,7 @@ async def update_team(
 
 
 @app.delete("/api/v1/admin/teams/{team_id}")
-async def delete_team(
+def delete_team(
     team_id: str,
     request: Request,
     _auth: str | None = Depends(verify_api_key),
@@ -7044,7 +7157,7 @@ async def delete_team(
 
 
 @app.get("/api/v1/admin/teams/{team_id}/members")
-async def list_team_members(
+def list_team_members(
     request: Request,
     team_id: str,
     include_removed: bool = Query(False),
@@ -7100,7 +7213,7 @@ async def list_team_members(
 
 
 @app.post("/api/v1/admin/teams/{team_id}/members")
-async def add_team_member(
+def add_team_member(
     team_id: str,
     req: AddTeamMemberRequest,
     request: Request,
@@ -7157,7 +7270,7 @@ async def add_team_member(
 
 
 @app.patch("/api/v1/admin/teams/{team_id}/members/{member_id}")
-async def update_team_member(
+def update_team_member(
     team_id: str,
     member_id: str,
     req: UpdateTeamMemberRequest,
@@ -7213,7 +7326,7 @@ async def update_team_member(
 
 
 @app.delete("/api/v1/admin/teams/{team_id}/members/{member_id}")
-async def remove_team_member(
+def remove_team_member(
     team_id: str,
     member_id: str,
     request: Request,
@@ -7247,7 +7360,7 @@ async def remove_team_member(
 
 
 @app.post("/api/v1/admin/teams/{team_id}/members/{member_id}/reinstate")
-async def reinstate_team_member(
+def reinstate_team_member(
     team_id: str,
     member_id: str,
     request: Request,
@@ -7291,7 +7404,7 @@ async def reinstate_team_member(
 
 
 @app.get("/api/v1/admin/members/check-email")
-async def check_member_email(
+def check_member_email(
     request: Request,
     email: str = Query(...),
     _auth: str | None = Depends(verify_api_key),
@@ -7356,7 +7469,7 @@ async def check_member_email(
 
 
 @app.get("/api/v1/admin/patterns")
-async def list_detected_patterns(
+def list_detected_patterns(
     pattern_type: str | None = Query(None),
     category: str | None = Query(None),
     severity: str | None = Query(None),
@@ -7437,7 +7550,7 @@ async def list_detected_patterns(
 
 
 @app.post("/api/v1/admin/patterns/scan")
-async def run_pattern_scan(
+def run_pattern_scan(
     req: RunPatternDetectionRequest,
     request: Request,
     _auth: str | None = Depends(verify_api_key),
@@ -7543,7 +7656,7 @@ async def run_pattern_scan(
 
 
 @app.patch("/api/v1/admin/patterns/{pattern_id}/resolve")
-async def resolve_pattern(
+def resolve_pattern(
     pattern_id: str,
     request: Request,
     _auth: str | None = Depends(verify_api_key),
@@ -8162,7 +8275,9 @@ async def _credit_briefing_reuse(
             if _async_adapter is not None:
                 entry = await _async_adapter.read(entity_path, key, branch=branch)
             else:
-                entry = _get_memory()._adapter.read(entity_path, key, branch=branch)
+                entry = await _offload(
+                    _db_executor, _get_memory()._adapter.read, entity_path, key, branch=branch
+                )
         except Exception:  # noqa: BLE001 - the value block is reporting, not the answer
             logger.debug("briefing reuse lookup failed", exc_info=True)
 
@@ -8711,6 +8826,23 @@ async def ingest_event(
 # CLI Entry Point
 # ──────────────────────────────────────────────────────────────────────
 
+# With more than one worker, uvicorn runs a supervisor that pings every worker
+# twice a second and SIGKILLs any that has not answered within
+# ``timeout_worker_healthcheck`` — 5 seconds by default — then spawns a fresh
+# one and logs the same "Child process [pid] died" it would for a real crash.
+# The pong comes from a thread the worker starts on entry, so it needs the GIL,
+# and a fresh worker does not have one to give: spawn re-imports this module
+# and its C extensions and the lifespan loads the embedding models, all of it
+# CPU-bound, all of it competing with the worker next door that is serving
+# traffic. On a 2-vCPU Cloud Run instance that reliably took longer than 5
+# seconds, so one slot per instance was killed every 8 seconds forever and
+# never once logged "Started server process" (2026-09-21, 36 kills a minute
+# across the fleet, every request that landed on the dying slot truncated).
+# Two minutes is generous room to start and rides out any GIL hold a request
+# could plausibly cause, while a worker that is genuinely wedged still gets
+# replaced. A single worker has no supervisor and is unaffected either way.
+DEFAULT_WORKER_HEALTHCHECK_S = 120
+
 
 def _parse_args() -> argparse.Namespace:
     parser = argparse.ArgumentParser(
@@ -8747,6 +8879,16 @@ def _parse_args() -> argparse.Namespace:
         default=int(os.environ.get("AMFS_HTTP_WORKERS", "1")),
         help="Number of uvicorn worker processes (default: 1, env: AMFS_HTTP_WORKERS)",
     )
+    parser.add_argument(
+        "--worker-healthcheck-timeout",
+        type=int,
+        default=int(os.environ.get("AMFS_HTTP_WORKER_HEALTHCHECK_S", str(DEFAULT_WORKER_HEALTHCHECK_S))),
+        help=(
+            "Seconds a worker may go without answering the supervisor's ping before "
+            "it is killed and replaced; only meaningful with --workers > 1 "
+            f"(default: {DEFAULT_WORKER_HEALTHCHECK_S}, env: AMFS_HTTP_WORKER_HEALTHCHECK_S)"
+        ),
+    )
     return parser.parse_args()
 
 
@@ -8778,86 +8920,104 @@ def _make_tenant_provider(dsn: str):
     return _provider
 
 
-def main() -> None:
-    """Run the AMFS HTTP server via uvicorn."""
+def _start_embedded_cortex() -> None:
+    """Start the embedded Cortex worker in this process, if it can be.
+
+    Called from the app lifespan, so it runs once per serving process — see
+    ``_lifespan`` for why not from ``main()``. Idempotent: a second call in the
+    same process (a test re-entering the lifespan) is a no-op.
+    """
     global _cortex_worker
-    args = _parse_args()
+    if _cortex_worker is not None:
+        return
+    dsn = os.environ.get("AMFS_POSTGRES_DSN")
+    if not dsn:
+        logger.warning("AMFS_WITH_CORTEX requires AMFS_POSTGRES_DSN")
+        return
 
-    if args.with_cortex:
-        dsn = os.environ.get("AMFS_POSTGRES_DSN")
-        if dsn:
-            import threading
+    try:
+        from amfs_postgres.adapter import PostgresAdapter
+        from amfs_cortex.compiler import DigestCompiler
+        from amfs_cortex.worker import CortexWorker
+
+        namespace = os.environ.get("AMFS_NAMESPACE", "default")
+        # One background thread compiling digests on a timer, sharing a
+        # process with the request path. Sized for that rather than left
+        # on the pool default, which would have it hold as many
+        # connections as the endpoints do while using one at a time —
+        # multiplied by every process the service scales out to.
+        adapter = PostgresAdapter(
+            dsn=dsn, namespace=namespace, min_pool_size=1, max_pool_size=2
+        )
 
-            try:
-                from amfs_postgres.adapter import PostgresAdapter
-                from amfs_cortex.compiler import DigestCompiler
-                from amfs_cortex.worker import CortexWorker
-
-                namespace = os.environ.get("AMFS_NAMESPACE", "default")
-                # One background thread compiling digests on a timer, sharing a
-                # process with the request path. Sized for that rather than left
-                # on the pool default, which would have it hold as many
-                # connections as the endpoints do while using one at a time —
-                # multiplied by every instance the service scales out to.
-                adapter = PostgresAdapter(
-                    dsn=dsn, namespace=namespace, min_pool_size=1, max_pool_size=2
-                )
+        strategies = []
+        try:
+            from amfs_cortex_pro import get_pro_strategies
+            strategies = get_pro_strategies()
+            logger.info("Pro compilation strategies loaded")
+        except ImportError:
+            pass
 
-                strategies = []
-                try:
-                    from amfs_cortex_pro import get_pro_strategies
-                    strategies = get_pro_strategies()
-                    logger.info("Pro compilation strategies loaded")
-                except ImportError:
-                    pass
+        compiler = DigestCompiler(
+            adapter=adapter,
+            strategies=strategies or None,
+            namespace=namespace,
+        )
+        tenant_provider = _make_tenant_provider(dsn)
+        # The catch-up scan runs in every serving process (no advisory lock),
+        # so its interval sets the fleet-wide scan rate: 36 processes at the
+        # 300 s default is one tenant-wide scan every ~8 s. A deployment that
+        # runs several workers per instance should raise this in proportion
+        # to keep the rate where it was. The scan is a GROUP BY since the
+        # adapter grew list_scopes(), but the knob stays: 0 disables it on
+        # deployments where the event path is trusted to compile every scope.
+        catchup_s = float(os.environ.get("AMFS_CORTEX_CATCHUP_INTERVAL_S", "300"))
+        worker = CortexWorker(
+            dsn=dsn,
+            compiler=compiler,
+            use_advisory_lock=False,
+            catchup_interval_s=catchup_s,
+            tenant_provider=tenant_provider,
+        )
 
-                compiler = DigestCompiler(
-                    adapter=adapter,
-                    strategies=strategies or None,
-                    namespace=namespace,
-                )
-                tenant_provider = _make_tenant_provider(dsn)
-                # The catch-up scan runs on every instance (no advisory lock), so
-                # its interval sets the fleet-wide scan rate: 36 instances at the
-                # 300 s default is one tenant-wide scan every ~8 s. The scan is
-                # a GROUP BY since the adapter grew list_scopes(), but the knob
-                # stays: 0 disables it on deployments where the event path is
-                # trusted to compile every scope.
-                catchup_s = float(os.environ.get("AMFS_CORTEX_CATCHUP_INTERVAL_S", "300"))
-                _cortex_worker = CortexWorker(
-                    dsn=dsn,
-                    compiler=compiler,
-                    use_advisory_lock=False,
-                    catchup_interval_s=catchup_s,
-                    tenant_provider=tenant_provider,
-                )
+        try:
+            from amfs_cortex_pro import get_outcome_wiring, HotContextTracker
+            wiring = get_outcome_wiring(adapter, namespace)
+            if wiring:
+                worker._outcome_wiring = wiring
+                logger.info("Outcome wiring attached to embedded Cortex worker")
+            tracker = HotContextTracker()
+            worker._hot_tracker = tracker
+            logger.info("Hot context tracker attached to embedded Cortex worker")
+        except ImportError:
+            pass
 
-                try:
-                    from amfs_cortex_pro import get_outcome_wiring, HotContextTracker
-                    wiring = get_outcome_wiring(adapter, namespace)
-                    if wiring:
-                        _cortex_worker._outcome_wiring = wiring
-                        logger.info("Outcome wiring attached to embedded Cortex worker")
-                    tracker = HotContextTracker()
-                    _cortex_worker._hot_tracker = tracker
-                    logger.info("Hot context tracker attached to embedded Cortex worker")
-                except ImportError:
-                    pass
+        from amfs_http.pro_proxy import create_forwarder
+        forwarder = create_forwarder()
+        if forwarder:
+            worker._pro_forwarder = forwarder
 
-                from amfs_http.pro_proxy import create_forwarder
-                forwarder = create_forwarder()
-                if forwarder:
-                    _cortex_worker._pro_forwarder = forwarder
+        t = threading.Thread(target=worker.run, daemon=True, name="cortex-embedded")
+        t.start()
+        _cortex_worker = worker
+        logger.info("Embedded Cortex worker started (pid=%d)", os.getpid())
+    except ImportError:
+        logger.warning("AMFS_WITH_CORTEX requires amfs-cortex package")
+    except Exception:
+        logger.exception("Failed to start embedded Cortex worker — server will run without Cortex")
 
-                t = threading.Thread(target=_cortex_worker.run, daemon=True, name="cortex-embedded")
-                t.start()
-                logger.info("Embedded Cortex worker started")
-            except ImportError:
-                logger.warning("--with-cortex requires amfs-cortex package")
-            except Exception:
-                logger.exception("Failed to start embedded Cortex worker — server will run without Cortex")
-        else:
-            logger.warning("--with-cortex requires AMFS_POSTGRES_DSN")
+
+def main() -> None:
+    """Run the AMFS HTTP server via uvicorn."""
+    args = _parse_args()
+
+    # The flag becomes an environment variable rather than a start here, so
+    # that every uvicorn worker process — which inherits the environment but
+    # not this function's locals — starts its own Cortex from the lifespan.
+    # Setting the variable directly (docker-compose, a systemd unit) works the
+    # same without the flag.
+    if args.with_cortex:
+        os.environ["AMFS_WITH_CORTEX"] = "1"
 
     workers = args.workers
     if args.reload and workers > 1:
@@ -8865,7 +9025,8 @@ def main() -> None:
         workers = 1
 
     logger.info(
-        "Starting AMFS HTTP server on %s:%d (workers=%d)", args.host, args.port, workers
+        "Starting AMFS HTTP server on %s:%d (workers=%d, worker healthcheck=%ds)",
+        args.host, args.port, workers, args.worker_healthcheck_timeout,
     )
     uvicorn.run(
         "amfs_http.server:app",
@@ -8873,6 +9034,7 @@ def main() -> None:
         port=args.port,
         reload=args.reload,
         workers=workers,
+        timeout_worker_healthcheck=args.worker_healthcheck_timeout,
     )
 
 

amfs_http_server-0.3.41/src/amfs_http/sse.py

@@ -0,0 +1,189 @@
+"""Server-Sent Events for real-time AMFS watch notifications.
+
+Subscribers are ``asyncio.Queue`` objects owned by the event loop that opened
+the stream. Broadcasts, though, arrive from wherever the write happened: an
+``async`` route on that loop, or — since the sync-bodied routes run on FastAPI's
+threadpool — a worker thread. ``asyncio.Queue.put_nowait`` is not thread-safe:
+called from another thread it appends the item but wakes the waiting consumer
+through ``call_soon``, which does not rouse a sleeping loop, so the event sits
+until something else happens to wake it (and raises under asyncio debug mode).
+The manager records each queue's loop at subscribe time and, when a broadcast
+comes from outside it, hands the put to ``call_soon_threadsafe``.
+"""
+from __future__ import annotations
+
+import asyncio
+import json
+import logging
+import threading
+from typing import Any
+
+from amfs_core.models import MemoryEntry
+
+logger = logging.getLogger(__name__)
+
+
+class SSEManager:
+    def __init__(self) -> None:
+        self._subscribers: dict[str, list[asyncio.Queue[dict[str, Any]]]] = {}
+        self._room_subscribers: dict[str, list[asyncio.Queue[dict[str, Any]]]] = {}
+        # The loop each queue belongs to, keyed by the queue's id. Subscribe and
+        # unsubscribe run on that loop; broadcasts may run on any thread, so the
+        # registries are read under the lock and iterated as copies.
+        self._loops: dict[int, asyncio.AbstractEventLoop] = {}
+        self._lock = threading.Lock()
+
+    # ── Subscriptions ────────────────────────────────────────────────
+
+    def _new_queue(self) -> asyncio.Queue[dict[str, Any]]:
+        queue: asyncio.Queue[dict[str, Any]] = asyncio.Queue()
+        try:
+            loop: asyncio.AbstractEventLoop | None = asyncio.get_running_loop()
+        except RuntimeError:  # subscribed outside a loop (tests); deliver directly
+            loop = None
+        if loop is not None:
+            with self._lock:
+                self._loops[id(queue)] = loop
+        return queue
+
+    def _forget_queue(self, queue: asyncio.Queue[dict[str, Any]]) -> None:
+        with self._lock:
+            self._loops.pop(id(queue), None)
+
+    def subscribe(self, entity_path: str = "*") -> asyncio.Queue[dict[str, Any]]:
+        queue = self._new_queue()
+        with self._lock:
+            self._subscribers.setdefault(entity_path, []).append(queue)
+        return queue
+
+    def unsubscribe(self, entity_path: str, queue: asyncio.Queue[dict[str, Any]]) -> None:
+        with self._lock:
+            subs = self._subscribers.get(entity_path, [])
+            if queue in subs:
+                subs.remove(queue)
+        self._forget_queue(queue)
+
+    def subscribe_room(self, room_id: str) -> asyncio.Queue[dict[str, Any]]:
+        """Subscribe to all events for a specific room."""
+        queue = self._new_queue()
+        with self._lock:
+            self._room_subscribers.setdefault(room_id, []).append(queue)
+        return queue
+
+    def unsubscribe_room(self, room_id: str, queue: asyncio.Queue[dict[str, Any]]) -> None:
+        with self._lock:
+            subs = self._room_subscribers.get(room_id, [])
+            if queue in subs:
+                subs.remove(queue)
+        self._forget_queue(queue)
+
+    # ── Delivery ─────────────────────────────────────────────────────
+
+    @staticmethod
+    def _put(queue: asyncio.Queue[dict[str, Any]], event: dict[str, Any], what: str) -> None:
+        try:
+            queue.put_nowait(event)
+        except asyncio.QueueFull:
+            logger.debug("SSE queue full, dropping %s", what)
+
+    def _deliver(
+        self, queue: asyncio.Queue[dict[str, Any]], event: dict[str, Any], what: str
+    ) -> None:
+        """Put ``event`` on ``queue`` from whatever thread this is.
+
+        On the queue's own loop the put is direct. From any other thread — a
+        threadpool worker running a sync route — it is scheduled onto that loop
+        with ``call_soon_threadsafe``, which is the one asyncio entry point that
+        may be called from outside the loop and that wakes it.
+        """
+        with self._lock:
+            owner = self._loops.get(id(queue))
+        if owner is None:
+            self._put(queue, event, what)
+            return
+        try:
+            running: asyncio.AbstractEventLoop | None = asyncio.get_running_loop()
+        except RuntimeError:
+            running = None
+        if running is owner:
+            self._put(queue, event, what)
+            return
+        if owner.is_closed():
+            return
+        try:
+            owner.call_soon_threadsafe(self._put, queue, event, what)
+        except RuntimeError:  # the loop closed between the check and the call
+            logger.debug("SSE subscriber loop closed, dropping %s", what)
+
+    def broadcast(self, entry: MemoryEntry) -> None:
+        data = entry.model_dump(mode="json")
+        data.pop("embedding", None)
+        event = {"type": "write", "entry": data}
+        with self._lock:
+            targets = [
+                queue
+                for pattern, queues in self._subscribers.items()
+                if (
+                    pattern == "*"
+                    or entry.entity_path == pattern
+                    or entry.entity_path.startswith(pattern + "/")
+                )
+                for queue in queues
+            ]
+        for queue in targets:
+            self._deliver(queue, event, "event")
+
+    def broadcast_room_event(
+        self,
+        room_id: str,
+        event_type: str,
+        data: dict[str, Any],
+    ) -> None:
+        """Broadcast a room-scoped event (join, leave, write, read, etc.)."""
+        event = {"type": event_type, "room_id": room_id, **data}
+        with self._lock:
+            targets = list(self._room_subscribers.get(room_id, []))
+        for queue in targets:
+            # Each subscriber pops "type" from its own copy in the generator.
+            self._deliver(queue, dict(event), f"event for room {room_id}")
+
+    # ── Generators ───────────────────────────────────────────────────
+
+    async def event_generator(self, entity_path: str = "*", predicate=None):
+        """Yield SSE events, optionally filtered per subscriber.
+
+        ``predicate`` receives the entry dict and returns True when the
+        subscriber may see it (used for per-user visibility). Events that
+        fail the predicate — or whose predicate raises — are dropped.
+        """
+        queue = self.subscribe(entity_path)
+        try:
+            while True:
+                event = await queue.get()
+                if predicate is not None:
+                    try:
+                        if not predicate(event.get("entry", {})):
+                            continue
+                    except Exception:
+                        logger.debug("SSE visibility predicate failed — dropping event", exc_info=True)
+                        continue
+                yield {
+                    "event": event["type"],
+                    "data": json.dumps(event["entry"], default=str),
+                }
+        finally:
+            self.unsubscribe(entity_path, queue)
+
+    async def room_event_generator(self, room_id: str):
+        """SSE generator for room-scoped events."""
+        queue = self.subscribe_room(room_id)
+        try:
+            while True:
+                event = await queue.get()
+                event_type = event.pop("type", "room_event")
+                yield {
+                    "event": event_type,
+                    "data": json.dumps(event, default=str),
+                }
+        finally:
+            self.unsubscribe_room(room_id, queue)

amfs_http_server-0.3.39/src/amfs_http/sse.py

@@ -1,112 +0,0 @@
-"""Server-Sent Events for real-time AMFS watch notifications."""
-from __future__ import annotations
-
-import asyncio
-import json
-import logging
-from typing import Any
-
-from amfs_core.models import MemoryEntry
-
-logger = logging.getLogger(__name__)
-
-
-class SSEManager:
-    def __init__(self) -> None:
-        self._subscribers: dict[str, list[asyncio.Queue[dict[str, Any]]]] = {}
-        self._room_subscribers: dict[str, list[asyncio.Queue[dict[str, Any]]]] = {}
-
-    def subscribe(self, entity_path: str = "*") -> asyncio.Queue[dict[str, Any]]:
-        queue: asyncio.Queue[dict[str, Any]] = asyncio.Queue()
-        if entity_path not in self._subscribers:
-            self._subscribers[entity_path] = []
-        self._subscribers[entity_path].append(queue)
-        return queue
-
-    def unsubscribe(self, entity_path: str, queue: asyncio.Queue[dict[str, Any]]) -> None:
-        subs = self._subscribers.get(entity_path, [])
-        if queue in subs:
-            subs.remove(queue)
-
-    def subscribe_room(self, room_id: str) -> asyncio.Queue[dict[str, Any]]:
-        """Subscribe to all events for a specific room."""
-        queue: asyncio.Queue[dict[str, Any]] = asyncio.Queue()
-        if room_id not in self._room_subscribers:
-            self._room_subscribers[room_id] = []
-        self._room_subscribers[room_id].append(queue)
-        return queue
-
-    def unsubscribe_room(self, room_id: str, queue: asyncio.Queue[dict[str, Any]]) -> None:
-        subs = self._room_subscribers.get(room_id, [])
-        if queue in subs:
-            subs.remove(queue)
-
-    def broadcast(self, entry: MemoryEntry) -> None:
-        data = entry.model_dump(mode="json")
-        data.pop("embedding", None)
-        event = {"type": "write", "entry": data}
-        for pattern, queues in self._subscribers.items():
-            if (
-                pattern == "*"
-                or entry.entity_path == pattern
-                or entry.entity_path.startswith(pattern + "/")
-            ):
-                for queue in queues:
-                    try:
-                        queue.put_nowait(event)
-                    except asyncio.QueueFull:
-                        logger.debug("SSE queue full, dropping event")
-
-    def broadcast_room_event(
-        self,
-        room_id: str,
-        event_type: str,
-        data: dict[str, Any],
-    ) -> None:
-        """Broadcast a room-scoped event (join, leave, write, read, etc.)."""
-        event = {"type": event_type, "room_id": room_id, **data}
-        queues = self._room_subscribers.get(room_id, [])
-        for queue in queues:
-            try:
-                queue.put_nowait(event)
-            except asyncio.QueueFull:
-                logger.debug("Room SSE queue full, dropping event for room %s", room_id)
-
-    async def event_generator(self, entity_path: str = "*", predicate=None):
-        """Yield SSE events, optionally filtered per subscriber.
-
-        ``predicate`` receives the entry dict and returns True when the
-        subscriber may see it (used for per-user visibility). Events that
-        fail the predicate — or whose predicate raises — are dropped.
-        """
-        queue = self.subscribe(entity_path)
-        try:
-            while True:
-                event = await queue.get()
-                if predicate is not None:
-                    try:
-                        if not predicate(event.get("entry", {})):
-                            continue
-                    except Exception:
-                        logger.debug("SSE visibility predicate failed — dropping event", exc_info=True)
-                        continue
-                yield {
-                    "event": event["type"],
-                    "data": json.dumps(event["entry"], default=str),
-                }
-        finally:
-            self.unsubscribe(entity_path, queue)
-
-    async def room_event_generator(self, room_id: str):
-        """SSE generator for room-scoped events."""
-        queue = self.subscribe_room(room_id)
-        try:
-            while True:
-                event = await queue.get()
-                event_type = event.pop("type", "room_event")
-                yield {
-                    "event": event_type,
-                    "data": json.dumps(event, default=str),
-                }
-        finally:
-            self.unsubscribe_room(room_id, queue)