crazy-workers 1.3.0__tar.gz → 1.4.1__tar.gz

{crazy_workers-1.3.0 → crazy_workers-1.4.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: crazy-workers
-Version: 1.3.0
+Version: 1.4.1
 Summary: A Python library for managing background worker processes with persistent state, automatic recovery, and a CLI.
 Author: GioVanni Colasanto
 License: MIT

crazy_workers-1.4.1/crazy_workers/__init__.py

@@ -0,0 +1,6 @@
+from .client import WorkerClient
+from .core.manager import WorkerManager
+from .database.schema import DesiredStatus, WorkerStatus
+
+
+__all__ = ['WorkerManager', 'WorkerClient', 'WorkerStatus', 'DesiredStatus']

{crazy_workers-1.3.0 → crazy_workers-1.4.1}/crazy_workers/cli/commands/params.py

@@ -4,28 +4,21 @@ from rich.prompt import IntPrompt
 from ..ui import console, err_console
 
 
-def show_params(manager, worker_key):
-
-  workers = manager.list_workers()
+def show_params(client, worker_key):
+  workers = client.list()
   if not workers:
     console().print('[yellow]No workers found.[/yellow]')
     return False
 
   if not worker_key:
     # Interactive mode
-    active_workers = [w for w in workers if w['worker_key'] is not None]
-
-    if not active_workers:
-      console().print('[yellow]No registered workers to show parameters for.[/yellow]')
-      return False
-
     console().print('\n[bold cyan]Select a worker to show parameters:[/bold cyan]')
-    for i, w in enumerate(active_workers, 1):
+    for i, w in enumerate(workers, 1):
       status_style = 'green' if w['status'] == 'RUNNING' else 'dim'
       console().print(f'  [bold]{i})[/bold] {w["worker_key"]} [{status_style}]({w["status"]})[/{status_style}]')
 
-    choice = IntPrompt.ask('Enter the number', choices=[str(i) for i in range(1, len(active_workers) + 1)])
-    selected_worker = active_workers[choice - 1]
+    choice = IntPrompt.ask('Enter the number', choices=[str(i) for i in range(1, len(workers) + 1)])
+    selected_worker = workers[choice - 1]
   else:
     selected_worker = next((w for w in workers if w['worker_key'] == worker_key), None)
     if not selected_worker:

crazy_workers-1.4.1/crazy_workers/cli/commands/starter.py

@@ -0,0 +1,36 @@
+import os
+from rich.prompt import IntPrompt
+
+from ..ui import console, err_console
+
+
+def start_worker(client, workers_dir, worker_type, worker_key=None, parameters=None):
+  """Request a worker to run. The daemon performs the actual spawn."""
+  if not worker_type:
+    # Interactive mode: list .py files in workers_dir
+    try:
+      files = [f[:-3] for f in os.listdir(workers_dir) if f.endswith('.py') and f != '__init__.py']
+    except Exception as e:
+      err_console().print(f'[bold red]Error reading workers directory:[/bold red] {e}')
+      return False
+
+    if not files:
+      console().print(f'[yellow]No worker scripts found in {workers_dir}[/yellow]')
+      return False
+
+    console().print('\n[bold cyan]Select a worker type to start:[/bold cyan]')
+    for i, f in enumerate(files, 1):
+      console().print(f'  [bold]{i})[/bold] {f}')
+
+    choice = IntPrompt.ask('Enter the number', choices=[str(i) for i in range(1, len(files) + 1)])
+    worker_type = files[choice - 1]
+
+  # Surface a typo here rather than as a daemon CRASHED/retry loop later.
+  if not os.path.exists(os.path.join(workers_dir, f'{worker_type}.py')):
+    err_console().print(f'[bold red]Error:[/bold red] Worker file {worker_type}.py not found in {workers_dir}')
+    return False
+
+  key = client.request_start(worker_type, worker_key=worker_key, parameters=parameters)
+  console().print('[bold green]Requested:[/bold green] worker set to RUNNING (the daemon will start it)')
+  console().print(f'  [bold]Key:[/bold] {key}')
+  return True

crazy_workers-1.4.1/crazy_workers/cli/commands/status.py

@@ -0,0 +1,116 @@
+import json
+import os
+import re
+from datetime import datetime
+from rich.panel import Panel
+from rich.table import Table
+
+from ..ui import console
+
+
+def show_status(client, workers_dir):
+  """Observability hub: the target state store plus the worker table (desired vs actual)."""
+  console().print(_build_header(workers_dir))
+
+  workers = _merge_with_filesystem(client.list(), workers_dir)
+  if not workers:
+    console().print('[yellow]No workers found.[/yellow]')
+    return workers
+
+  console().print(_build_table(workers))
+  return workers
+
+
+def _build_header(workers_dir):
+  db_url = os.environ.get('CRAZY_WORKERS_DB_URL')
+  if db_url:
+    target = f'[green]shared DB[/green] [dim]({_redact(db_url)})[/dim]'
+  else:
+    target = '[dim]self-contained SQLite (.service/workers.db)[/dim]'
+  dir_label = workers_dir if workers_dir else '[dim](not set — scripts not listed)[/dim]'
+  body = f'[bold]Workers dir:[/bold] {dir_label}\n[bold]State store:[/bold] {target}'
+  return Panel.fit(body, border_style='cyan', title='[bold cyan]Crazy Workers status[/bold cyan]')
+
+
+def _redact(db_url):
+  """Hide the password in a SQLAlchemy URL for display."""
+  return re.sub(r'://([^:/@]+):[^@]*@', r'://\1:***@', db_url)
+
+
+def _merge_with_filesystem(db_workers, workers_dir):
+  """Append NEVER_STARTED rows for worker scripts that have no DB record yet."""
+  results = list(db_workers)
+  if not workers_dir:
+    # No dir resolved (shared-DB mode without CRAZY_WORKERS_DIR): nothing to scan.
+    return results
+  registered_types = {w['worker_type'] for w in results}
+  try:
+    available = sorted({f[:-3] for f in os.listdir(workers_dir) if f.endswith('.py') and f != '__init__.py'})
+  except OSError:
+    available = []
+  for worker_type in available:
+    if worker_type not in registered_types:
+      results.append(
+        {
+          'worker_key': None,
+          'worker_type': worker_type,
+          'parameters': {},
+          'desired_status': None,
+          'pid': None,
+          'status': 'NEVER_STARTED',
+          'last_started_at': None,
+          'last_stopped_at': None,
+        }
+      )
+  return results
+
+
+def _build_table(workers):
+  table = Table(
+    title='[bold cyan]Workers — desired vs actual[/bold cyan]', border_style='cyan', header_style='bold magenta'
+  )
+  table.add_column('#', justify='right', style='dim')
+  table.add_column('Key', style='bold')
+  table.add_column('Type')
+  table.add_column('Desired', justify='center')
+  table.add_column('Status', justify='center')
+  table.add_column('PID', justify='right', style='green')
+  table.add_column('Last Action', justify='center')
+  table.add_column('Params', overflow='ellipsis')
+
+  for i, w in enumerate(workers, 1):
+    status = w['status']
+    status_style = 'green' if status == 'RUNNING' else 'yellow'
+    if status in ('CRASHED', 'FAILED'):
+      status_style = 'bold red'
+    elif status == 'STOPPED':
+      status_style = 'dim'
+    elif status == 'NEVER_STARTED':
+      status_style = 'cyan'
+
+    desired = w.get('desired_status') or '-'
+    desired_style = 'green' if desired == 'RUNNING' else 'dim'
+
+    last_action = '-'
+    if status == 'RUNNING' and w.get('last_started_at'):
+      dt = datetime.fromisoformat(w['last_started_at'])
+      last_action = f'[green]Started {dt.strftime("%H:%M:%S")}[/green]'
+    elif w.get('last_stopped_at'):
+      dt = datetime.fromisoformat(w['last_stopped_at'])
+      last_action = f'[dim]Stopped {dt.strftime("%H:%M:%S")}[/dim]'
+
+    params_str = json.dumps(w['parameters']) if w['parameters'] else '-'
+    if len(params_str) > 30:
+      params_str = params_str[:27] + '...'
+
+    table.add_row(
+      str(i),
+      w['worker_key'] or '-',
+      w['worker_type'],
+      f'[{desired_style}]{desired}[/{desired_style}]',
+      f'[{status_style}]{status}[/{status_style}]',
+      str(w['pid']) if w['pid'] else '-',
+      last_action,
+      params_str,
+    )
+  return table

crazy_workers-1.4.1/crazy_workers/cli/commands/stopper.py

@@ -0,0 +1,28 @@
+from rich.prompt import IntPrompt
+
+from ..ui import console, err_console
+
+
+def stop_worker(client, worker_key):
+  """Request a worker to stop. The daemon performs the actual termination."""
+  if not worker_key:
+    # Interactive mode: offer the workers a stop is meaningful for.
+    candidates = [w for w in client.list() if w['desired_status'] == 'RUNNING']
+
+    if not candidates:
+      console().print('[yellow]No workers desired RUNNING to stop.[/yellow]')
+      return False
+
+    console().print('\n[bold cyan]Select a worker to stop:[/bold cyan]')
+    for i, w in enumerate(candidates, 1):
+      console().print(f'  [bold]{i})[/bold] {w["worker_key"]} [dim]({w["worker_type"]}, {w["status"]})[/dim]')
+
+    choice = IntPrompt.ask('Enter the number', choices=[str(i) for i in range(1, len(candidates) + 1)])
+    worker_key = candidates[choice - 1]['worker_key']
+
+  if client.request_stop(worker_key):
+    console().print(f'[bold green]Requested:[/bold green] worker {worker_key} set to STOPPED (the daemon will stop it)')
+    return True
+
+  err_console().print(f'[bold red]Error:[/bold red] Worker {worker_key} not found')
+  return False

{crazy_workers-1.3.0 → crazy_workers-1.4.1}/crazy_workers/cli/discovery.py

@@ -45,7 +45,15 @@ def save_to_env(key, value):
   os.replace(tmp, '.env')
 
 
-def resolve_workers_dir(flag_dir):
+def resolve_workers_dir(flag_dir, required=True):
+  """Locate the workers directory (where the ``.py`` scripts live).
+
+  ``required=False`` is used by commands that only talk to a shared DB (e.g.
+  ``status``/``stop``/``params`` with ``CRAZY_WORKERS_DB_URL`` set): they do not
+  need the scripts, so we must never block on the interactive prompt. In that
+  mode the dir is best-effort — returned from the flag/env/``workers`` fallback
+  if available, otherwise ``None``.
+  """
   load_env()
 
   # 1. Flag priority
@@ -65,6 +73,10 @@ def resolve_workers_dir(flag_dir):
       err_console().print(f'[bold red]Error:[/bold red] Directory "{env_dir}" (from CRAZY_WORKERS_DIR) does not exist.')
       sys.exit(1)
 
+  # When the dir is not needed, never prompt or exit — report it as unknown.
+  if not required:
+    return 'workers' if os.path.isdir('workers') else None
+
   # 3. Interactive Prompt
   if sys.stdin.isatty():
     console().print('[bold yellow]CRAZY_WORKERS_DIR not set in environment.[/bold yellow]')
@@ -72,11 +84,16 @@ def resolve_workers_dir(flag_dir):
     if user_input:
       if os.path.isdir(user_input):
         abs_path = os.path.abspath(user_input)
-        try:
-          save_to_env('CRAZY_WORKERS_DIR', abs_path)
-          console().print(f'[bold green]Saved CRAZY_WORKERS_DIR={abs_path} to .env[/bold green]')
-        except Exception as e:
-          err_console().print(f'[bold red]Failed to save configuration:[/bold red] {e}')
+        # When pointed at a shared DB the cwd is the consumer app's, not ours —
+        # don't rewrite its .env (see T2). The env var lives only for this run.
+        if os.environ.get('CRAZY_WORKERS_DB_URL'):
+          os.environ.setdefault('CRAZY_WORKERS_DIR', abs_path)
+        else:
+          try:
+            save_to_env('CRAZY_WORKERS_DIR', abs_path)
+            console().print(f'[bold green]Saved CRAZY_WORKERS_DIR={abs_path} to .env[/bold green]')
+          except Exception as e:
+            err_console().print(f'[bold red]Failed to save configuration:[/bold red] {e}')
         return abs_path
       else:
         err_console().print(f'[bold red]Error:[/bold red] "{user_input}" is not a valid directory.')

crazy_workers-1.4.1/crazy_workers/cli/main.py

@@ -0,0 +1,121 @@
+import argparse
+import json
+import os
+import sys
+from rich.panel import Panel
+
+from ..client import WorkerClient
+from .commands import show_params, show_status, start_worker, stop_worker
+from .discovery import resolve_workers_dir
+from .ui import console, err_console
+
+
+def _db_url():
+  return os.environ.get('CRAZY_WORKERS_DB_URL')
+
+
+def _build_client(workers_dir):
+  """A control-plane client over the shared DB, or the local self-contained SQLite.
+
+  With CRAZY_WORKERS_DB_URL set the CLI talks to the same DB as the daemon and
+  issues no DDL (the daemon or host owns the schema). Otherwise it falls back to
+  the local ``.service/workers.db``, the self-contained mode.
+  """
+  db_url = _db_url()
+  if db_url:
+    return WorkerClient(db_url=db_url, create_tables=False)
+  service_dir = os.path.join(workers_dir, '.service')
+  os.makedirs(service_dir, exist_ok=True)
+  sqlite_path = os.path.join(service_dir, 'workers.db')
+  return WorkerClient(db_url=f'sqlite:///{sqlite_path}', create_tables=True)
+
+
+def _build_parser():
+  def formatter(prog):
+    return argparse.HelpFormatter(prog, max_help_position=32)
+
+  parser = argparse.ArgumentParser(description='Crazy Workers CLI', formatter_class=formatter)
+  parser.add_argument('--workers-dir', help='Directory containing worker scripts')
+
+  subparsers = parser.add_subparsers(dest='command', help='Commands')
+
+  subparsers.add_parser('status', help='Show workers (desired vs actual) and the target DB')
+
+  start_parser = subparsers.add_parser('start', help='Request a worker to run (interactive if type missing)')
+  start_parser.add_argument('worker_type', nargs='?', help='The type (filename) of worker to start')
+  start_parser.add_argument('--key', help='Optional custom key for the worker')
+  start_parser.add_argument('--params', help='JSON string of parameters for the worker')
+
+  stop_parser = subparsers.add_parser('stop', help='Request a worker to stop (interactive if key missing)')
+  stop_parser.add_argument('worker_key', nargs='?', help='The key of the worker to stop')
+
+  params_parser = subparsers.add_parser('params', help='Show parameters for a worker')
+  params_parser.add_argument('worker_key', nargs='?', help='The key of the worker')
+
+  daemon_parser = subparsers.add_parser('daemon', help='Run the reconcile loop (owns the worker processes)')
+  daemon_parser.add_argument('--interval', type=float, default=2.0, help='Seconds between reconcile passes')
+
+  return parser
+
+
+def main():
+  parser = _build_parser()
+  args = parser.parse_args()
+
+  if not args.command:
+    console().print(
+      Panel.fit(
+        '[bold cyan]Crazy Workers CLI[/bold cyan]\n[dim]Manage your background processes with ease[/dim]',
+        border_style='cyan',
+      )
+    )
+    parser.print_help()
+    sys.exit(1)
+
+  # Only `start` (lists/validates the worker scripts) and the daemon (owns them)
+  # truly need the workers dir. With a shared DB, status/stop/params work without
+  # it, so we must not block on the interactive prompt (see CRAZY_WORKERS_DB_URL).
+  needs_dir = args.command in ('start', 'daemon') or _db_url() is None
+  workers_dir = resolve_workers_dir(args.workers_dir, required=needs_dir)
+
+  # The daemon is the process owner, not a client — it builds its own manager.
+  if args.command == 'daemon':
+    from ..daemon.runner import main as daemon_main
+
+    argv = ['--workers-dir', workers_dir, '--interval', str(args.interval)]
+    db_url = _db_url()
+    if db_url:
+      argv += ['--db-url', db_url]
+    sys.exit(daemon_main(argv))
+
+  try:
+    with _build_client(workers_dir) as client:
+      if args.command == 'status':
+        show_status(client, workers_dir)
+      elif args.command == 'start':
+        params = _parse_params(args.params)
+        if not start_worker(client, workers_dir, args.worker_type, worker_key=args.key, parameters=params):
+          sys.exit(1)
+      elif args.command == 'stop':
+        if not stop_worker(client, args.worker_key):
+          sys.exit(1)
+      elif args.command == 'params':
+        if not show_params(client, args.worker_key):
+          sys.exit(1)
+  except ValueError as e:
+    err_console().print(f'[bold red]Error:[/bold red] {e}')
+    sys.exit(1)
+
+
+def _parse_params(raw):
+  if not raw:
+    return None
+  try:
+    return json.loads(raw)
+  except json.JSONDecodeError:
+    err_console().print('[bold red]Error:[/bold red] Invalid JSON in --params')
+    sys.exit(1)
+
+
+if __name__ == '__main__':
+  main()

crazy_workers-1.4.1/crazy_workers/client.py

@@ -0,0 +1,69 @@
+"""Control-plane client: writes desired state only, never spawns processes.
+
+Used by anything that is NOT the daemon (HTTP API, CLI, scripts). It shares the
+daemon's database; the daemon reconciles desired -> actual. A client touches the
+``workers`` table and nothing else — no OS processes, no boot hooks, no recovery.
+
+Three ways to point it at a database, mirroring :class:`Storage`:
+
+- ``engine``: reuse an existing SQLAlchemy engine (e.g. the host backend's).
+- ``db_url``: any SQLAlchemy URL.
+- neither: the caller must pass one — unlike WorkerManager, the client has no
+  workers_dir and therefore no implicit SQLite location.
+"""
+
+from .database.schema import DesiredStatus, Worker
+from .database.storage import Storage
+
+
+class WorkerClient:
+  def __init__(self, db_url=None, engine=None, create_tables=False):
+    self.storage = Storage(db_url=db_url, engine=engine, create_tables=create_tables)
+
+  def request_start(self, worker_type, worker_key=None, parameters=None):
+    """Declare that ``worker_key`` should be RUNNING (upserting its spec).
+
+    Returns the resolved worker_key. The worker is not started here; the daemon
+    notices the desired state and starts it.
+    """
+    worker_key = worker_key or worker_type
+    with self.storage.session_scope() as session:
+      worker = session.query(Worker).filter_by(worker_key=worker_key).first()
+      if not worker:
+        worker = Worker(worker_key=worker_key, worker_type=worker_type)
+        session.add(worker)
+      worker.worker_type = worker_type
+      worker.parameters = parameters or {}
+      worker.desired_status = DesiredStatus.RUNNING
+    return worker_key
+
+  def request_stop(self, worker_key):
+    """Declare that ``worker_key`` should be STOPPED.
+
+    Returns False if no such worker exists. The actual stop (and last_stopped_at)
+    is performed by the daemon when it reconciles.
+    """
+    with self.storage.session_scope() as session:
+      worker = session.query(Worker).filter_by(worker_key=worker_key).first()
+      if not worker:
+        return False
+      worker.desired_status = DesiredStatus.STOPPED
+    return True
+
+  def list(self):
+    with self.storage.session_scope() as session:
+      return [w.to_dict() for w in session.query(Worker).all()]
+
+  def get(self, worker_key):
+    with self.storage.session_scope() as session:
+      worker = session.query(Worker).filter_by(worker_key=worker_key).first()
+      return worker.to_dict() if worker else None
+
+  def dispose(self):
+    self.storage.dispose()
+
+  def __enter__(self):
+    return self
+
+  def __exit__(self, exc_type, exc_val, exc_tb):
+    self.dispose()

{crazy_workers-1.3.0 → crazy_workers-1.4.1}/crazy_workers/core/manager/__init__.py

@@ -19,7 +19,7 @@ class WorkerManager:
     workers_dir='workers',
     create_dir=True,
     backend=None,
-    auto_boot=True,
+    auto_boot=False,
     boot_provider=None,
     db_url=None,
     engine=None,
@@ -38,9 +38,12 @@ class WorkerManager:
     # The backend is the only component that touches OS processes. The default
     # spawns real subprocesses; tests can swap in a fake one (see for_testing).
     self.backend = backend or SubprocessBackend()
-    # When True, starting a worker transparently installs the per-user OS
-    # boot-restore hook (see crazy_workers.boot). boot_provider is an injection
-    # seam for tests; None lets the platform default be auto-detected.
+    # Opt-in legacy behaviour: when True, starting a worker transparently
+    # installs the per-user OS boot-restore hook (see crazy_workers.boot). The
+    # default is now False — in the reconciler model, surviving a reboot is the
+    # deployment's job (a systemd unit / container that runs the daemon), not a
+    # per-worker hook. boot_provider is an injection seam for tests; None lets
+    # the platform default be auto-detected.
     self.auto_boot = auto_boot
     self._boot_provider = boot_provider
     # Environment variables injected into every spawned worker — e.g. the host

{crazy_workers-1.3.0 → crazy_workers-1.4.1}/crazy_workers/core/manager/starter.py

@@ -1,6 +1,7 @@
 import logging
 import os
 import re
+from datetime import datetime, timezone
 from sqlalchemy import func
 from sqlalchemy.exc import IntegrityError
 
@@ -60,7 +61,7 @@ def _check_already_running(manager, worker, session):
       return True
     else:
       logger.warning(f'Worker {worker.worker_key} found in RUNNING state but PID {worker.pid} is dead. Cleaning up.')
-      worker.status = WorkerStatus.CRASHED
+      _mark_crashed(worker)
       session.commit()
   return False
 
@@ -123,7 +124,7 @@ def _spawn_worker_process(manager, worker, worker_path, parameters, env, session
   )
 
   if handle is None:
-    worker.status = WorkerStatus.CRASHED
+    _mark_crashed(worker)
     worker.pid = None
     session.commit()
     return False, 'Worker process failed to start'
@@ -131,6 +132,8 @@ def _spawn_worker_process(manager, worker, worker_path, parameters, env, session
   worker.pid = handle.pid
   worker.status = WorkerStatus.RUNNING
   worker.last_started_at = func.now()
+  # A clean start clears the crash backoff so the next failure starts over.
+  worker.restart_count = 0
   session.commit()
 
   manager._active_processes[worker.worker_key] = handle
@@ -138,6 +141,15 @@ def _spawn_worker_process(manager, worker, worker_path, parameters, env, session
   return True, worker.to_dict()
 
 
+def _mark_crashed(worker):
+  # Record a death for crash backoff: bump the restart counter and stamp the
+  # exit time in UTC (Python-side, so the reconciler's backoff math does not
+  # depend on the DB dialect's now()/timezone semantics).
+  worker.status = WorkerStatus.CRASHED
+  worker.last_exit_at = datetime.now(timezone.utc)
+  worker.restart_count = (worker.restart_count or 0) + 1
+
+
 def _ensure_boot_restore(manager):
   # Best-effort and never raising: a freshly started worker transparently
   # registers the OS boot-restore hook so it survives a reboot.

crazy_workers-1.4.1/crazy_workers/daemon/__init__.py

@@ -0,0 +1,16 @@
+"""The reconcile daemon: the single owner of worker processes for a context.
+
+Clients (HTTP API, CLI, scripts) only write desired state to the shared DB; the
+daemon runs a loop that drives the observed state toward it — starting,
+stopping and crash-restarting processes. Exactly one daemon owns a given
+workers_dir/DB at a time (enforced by a lock in ``crazy_workers.daemon.runner``).
+
+Run it with ``python -m crazy_workers.daemon`` (a thin ``__main__`` shim over
+:func:`crazy_workers.daemon.runner.main`).
+"""
+
+from .reconciler import Reconciler
+from .runner import main
+
+
+__all__ = ['Reconciler', 'main']

crazy_workers-1.4.1/crazy_workers/daemon/__main__.py

@@ -0,0 +1,7 @@
+import sys
+
+from .runner import main
+
+
+if __name__ == '__main__':  # pragma: no cover
+  sys.exit(main())

crazy_workers-1.4.1/crazy_workers/daemon/reconciler.py

@@ -0,0 +1,118 @@
+import logging
+import time
+from datetime import datetime, timedelta, timezone
+
+from ..database.schema import DesiredStatus, Worker, WorkerStatus
+
+
+logger = logging.getLogger('crazy_workers')
+
+_BACKOFF_BASE_SECONDS = 1
+_BACKOFF_MAX_SECONDS = 60
+# Cap the exponent so a long-crashed worker doesn't compute an astronomically
+# large intermediate before min() clamps it.
+_BACKOFF_MAX_EXPONENT = 16
+
+
+class Reconciler:
+  """Single-owner loop: drives actual worker state toward desired state.
+
+  Owns every worker process for one workers_dir/DB. Clients never spawn; they
+  only set desired_status in the shared DB and this loop makes it so.
+
+  | desired  | alive | action                              |
+  |----------|-------|-------------------------------------|
+  | RUNNING  | no    | start (skipped while in backoff)    |
+  | RUNNING  | yes   | noop (reconcile observed status)    |
+  | STOPPED  | yes   | stop                                |
+  | STOPPED  | no    | noop                                |
+  """
+
+  def __init__(self, manager, interval=2.0):
+    self.manager = manager
+    self.interval = interval
+    self._stop = False
+
+  def run_forever(self):
+    logger.info('Reconciler started (interval=%ss)', self.interval)
+    while not self._stop:
+      try:
+        self.reconcile_once()
+      except Exception:
+        logger.exception('Reconcile pass failed; continuing.')
+      # Sleep in small slices so a SIGTERM-triggered stop is honoured promptly
+      # instead of after a full interval.
+      self._interruptible_sleep(self.interval)
+    logger.info('Reconciler stopped.')
+
+  def stop(self):
+    self._stop = True
+
+  def _interruptible_sleep(self, seconds):
+    deadline = time.monotonic() + seconds
+    while not self._stop and time.monotonic() < deadline:
+      time.sleep(min(0.2, deadline - time.monotonic()))
+
+  def reconcile_once(self):
+    """One pass over every worker. Returns the actions taken (for tests/observability)."""
+    actions = []
+    for row in self._load_snapshot():
+      action = self._reconcile_worker(row)
+      if action:
+        actions.append((row['worker_key'], action))
+    return actions
+
+  def _load_snapshot(self):
+    # Read everything we need into plain dicts and release the session before
+    # touching processes — start/stop open their own short-lived sessions.
+    with self.manager.storage.session_scope() as session:
+      return [
+        {
+          'worker_key': w.worker_key,
+          'worker_type': w.worker_type,
+          'parameters': w.parameters,
+          'pid': w.pid,
+          'desired': w.desired_status,
+          'status': w.status,
+          'restart_count': w.restart_count,
+          'last_exit_at': w.last_exit_at,
+        }
+        for w in session.query(Worker).all()
+      ]
+
+  def _reconcile_worker(self, row):
+    alive = self.manager.backend.is_alive(pid=row['pid'], worker_key=row['worker_key'])
+
+    if row['desired'] == DesiredStatus.RUNNING and not alive:
+      if self._in_backoff(row):
+        return None
+      logger.info('Reconcile: starting %s', row['worker_key'])
+      self.manager.start_worker(row['worker_type'], row['worker_key'], row['parameters'])
+      return 'start'
+    if row['desired'] == DesiredStatus.STOPPED and alive:
+      logger.info('Reconcile: stopping %s', row['worker_key'])
+      self.manager.stop_worker(row['worker_key'])
+      return 'stop'
+    if row['desired'] == DesiredStatus.RUNNING and alive and row['status'] != WorkerStatus.RUNNING:
+      # Process is up but the observed status drifted (e.g. left STARTING). Heal it.
+      self._mark_running(row['worker_key'])
+      return 'mark_running'
+    return None
+
+  def _in_backoff(self, row):
+    if not row['last_exit_at'] or row['status'] != WorkerStatus.CRASHED:
+      return False
+    exponent = min(row['restart_count'], _BACKOFF_MAX_EXPONENT)
+    delay = min(_BACKOFF_BASE_SECONDS * (2**exponent), _BACKOFF_MAX_SECONDS)
+    last_exit = row['last_exit_at']
+    # last_exit_at is stored as UTC wall-clock; coerce naive values read back
+    # from the DB to aware UTC so the comparison never mixes naive and aware.
+    if last_exit.tzinfo is None:
+      last_exit = last_exit.replace(tzinfo=timezone.utc)
+    return datetime.now(timezone.utc) < last_exit + timedelta(seconds=delay)
+
+  def _mark_running(self, worker_key):
+    with self.manager.storage.session_scope() as session:
+      worker = session.query(Worker).filter_by(worker_key=worker_key).first()
+      if worker:
+        worker.status = WorkerStatus.RUNNING

crazy_workers-1.4.1/crazy_workers/daemon/runner.py

@@ -0,0 +1,78 @@
+import argparse
+import logging
+import os
+import signal
+
+from ..core.manager import WorkerManager
+from ..core.recovery import RecoveryLock
+from .reconciler import Reconciler
+
+
+logger = logging.getLogger('crazy_workers')
+
+
+def main(argv=None):
+  parser = argparse.ArgumentParser(prog='crazy_workers.daemon', description='Run the reconcile loop.')
+  parser.add_argument('--workers-dir', required=True, help='Directory containing worker scripts')
+  parser.add_argument(
+    '--db-url',
+    default=os.environ.get('CRAZY_WORKERS_DB_URL'),
+    help='Shared DB URL (defaults to $CRAZY_WORKERS_DB_URL, else the local SQLite under .service/)',
+  )
+  parser.add_argument('--interval', type=float, default=2.0, help='Seconds between reconcile passes')
+  parser.add_argument('--log-level', default=os.environ.get('CRAZY_WORKERS_LOG_LEVEL', 'INFO'))
+  args = parser.parse_args(argv)
+
+  logging.basicConfig(
+    level=getattr(logging, args.log_level.upper(), logging.INFO),
+    format='%(asctime)s %(levelname)s %(name)s: %(message)s',
+  )
+
+  # Fail loudly on a misconfigured path rather than silently creating an empty
+  # workers dir (and then never finding any worker script in it).
+  if not os.path.isdir(args.workers_dir):
+    logger.error('Workers directory %s does not exist', args.workers_dir)
+    return 2
+
+  # The daemon owns the host-local runtime area (.service, logs, and the SQLite
+  # file in self-contained mode), so it materialises them (create_dir=True).
+  # auto_recover is off: a reconcile pass already restarts dead RUNNING workers.
+  manager = WorkerManager(
+    args.workers_dir,
+    create_dir=True,
+    auto_boot=False,
+    auto_recover=False,
+    db_url=args.db_url,
+    create_tables=args.db_url is None,
+  )
+
+  lock = RecoveryLock(os.path.join(manager.service_dir, 'daemon.lock'))
+  if not lock.acquire():
+    logger.error('Another crazy_workers daemon already owns %s; exiting.', args.workers_dir)
+    manager.dispose()
+    return 1
+
+  reconciler = Reconciler(manager, interval=args.interval)
+  _install_signal_handlers(reconciler)
+
+  try:
+    reconciler.run_forever()
+  finally:
+    lock.release()
+    manager.dispose()
+  return 0
+
+
+def _install_signal_handlers(reconciler):
+  def _handle(signum, _frame):
+    logger.info('Received signal %s; shutting down.', signum)
+    reconciler.stop()
+
+  for signame in ('SIGTERM', 'SIGINT'):
+    sig = getattr(signal, signame, None)
+    if sig is not None:
+      try:
+        signal.signal(sig, _handle)
+      except (ValueError, OSError):
+        # Not the main thread, or unsupported on this platform — best effort.
+        pass

{crazy_workers-1.3.0 → crazy_workers-1.4.1}/crazy_workers/database/schema.py

@@ -5,6 +5,12 @@ from sqlalchemy.orm import DeclarativeBase
 
 
 class WorkerStatus(enum.Enum):
+  """Observed status of a worker, owned by whoever runs the processes.
+
+  In the reconciler model this is daemon-owned; in embedded mode the
+  WorkerManager writes it directly.
+  """
+
   NEVER_STARTED = 'NEVER_STARTED'
   STARTING = 'STARTING'
   RUNNING = 'RUNNING'
@@ -12,6 +18,17 @@ class WorkerStatus(enum.Enum):
   CRASHED = 'CRASHED'
 
 
+class DesiredStatus(enum.Enum):
+  """What a client wants a worker to be doing. Client-owned.
+
+  Clients (HTTP API, CLI, scripts) only ever write this; the daemon reconciles
+  the observed ``status`` toward it.
+  """
+
+  RUNNING = 'RUNNING'
+  STOPPED = 'STOPPED'
+
+
 class Base(DeclarativeBase):
   pass
 
@@ -23,8 +40,18 @@ class Worker(Base):
   worker_key = Column(String(255), unique=True, nullable=False)
   worker_type = Column(String(255), nullable=False)  # Name of the .py file
   parameters = Column(JSON, nullable=False, default={})
+
+  # Desired state — written by clients (control plane).
+  desired_status = Column(Enum(DesiredStatus), nullable=False, default=DesiredStatus.STOPPED)
+
+  # Observed state — written by whoever owns the processes (the daemon, or the
+  # WorkerManager in embedded mode).
   pid = Column(Integer, nullable=True)
   status = Column(Enum(WorkerStatus), default=WorkerStatus.STOPPED)
+  # Crash backoff bookkeeping: restart_count grows on each failed/crashed spawn
+  # and resets on a successful start; last_exit_at timestamps the latest death.
+  restart_count = Column(Integer, nullable=False, default=0)
+  last_exit_at = Column(DateTime, nullable=True)
   last_started_at: datetime = Column(DateTime, nullable=True)
   last_stopped_at: datetime = Column(DateTime, nullable=True)
   created_at = Column(DateTime, server_default=func.now())
@@ -35,8 +62,11 @@ class Worker(Base):
       'worker_key': self.worker_key,
       'worker_type': self.worker_type,
       'parameters': self.parameters,
+      'desired_status': self.desired_status.value if self.desired_status else None,
       'pid': self.pid,
       'status': self.status.value,
+      'restart_count': self.restart_count,
+      'last_exit_at': self.last_exit_at.isoformat() if self.last_exit_at else None,
       'last_started_at': self.last_started_at.isoformat() if self.last_started_at else None,
       'last_stopped_at': self.last_stopped_at.isoformat() if self.last_stopped_at else None,
     }

{crazy_workers-1.3.0 → crazy_workers-1.4.1}/crazy_workers/database/storage.py

@@ -39,8 +39,13 @@ class Storage:
       self.engine = create_engine(url, connect_args=connect_args)
       self._owns_engine = True
 
-    if self.engine.dialect.name == 'sqlite':
+    # The tuning below registers engine-level listeners; on a shared engine more
+    # than one Storage may wrap it, so install exactly once per engine — a second
+    # 'begin' listener would emit a second BEGIN IMMEDIATE (transaction within a
+    # transaction) and fail.
+    if self.engine.dialect.name == 'sqlite' and not getattr(self.engine, '_cw_sqlite_tuned', False):
       self._install_sqlite_tuning()
+      self.engine._cw_sqlite_tuned = True
 
     self.Session = sessionmaker(bind=self.engine)
     if create_tables:

{crazy_workers-1.3.0 → crazy_workers-1.4.1}/crazy_workers.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: crazy-workers
-Version: 1.3.0
+Version: 1.4.1
 Summary: A Python library for managing background worker processes with persistent state, automatic recovery, and a CLI.
 Author: GioVanni Colasanto
 License: MIT

{crazy_workers-1.3.0 → crazy_workers-1.4.1}/crazy_workers.egg-info/SOURCES.txt

@@ -3,6 +3,7 @@ README.md
 pyproject.toml
 crazy_workers/__init__.py
 crazy_workers/_bootstrap.py
+crazy_workers/client.py
 crazy_workers.egg-info/PKG-INFO
 crazy_workers.egg-info/SOURCES.txt
 crazy_workers.egg-info/dependency_links.txt
@@ -35,6 +36,10 @@ crazy_workers/core/manager/lister.py
 crazy_workers/core/manager/recoverer.py
 crazy_workers/core/manager/starter.py
 crazy_workers/core/manager/stopper.py
+crazy_workers/daemon/__init__.py
+crazy_workers/daemon/__main__.py
+crazy_workers/daemon/reconciler.py
+crazy_workers/daemon/runner.py
 crazy_workers/database/__init__.py
 crazy_workers/database/schema.py
 crazy_workers/database/storage.py

{crazy_workers-1.3.0 → crazy_workers-1.4.1}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "crazy-workers"
-version = "1.3.0"
+version = "1.4.1"
 description = "A Python library for managing background worker processes with persistent state, automatic recovery, and a CLI."
 readme = "README.md"
 authors = [{ name = "GioVanni Colasanto" }]

crazy_workers-1.3.0/crazy_workers/__init__.py

@@ -1,5 +0,0 @@
-from .core.manager import WorkerManager
-from .database.schema import WorkerStatus
-
-
-__all__ = ['WorkerManager', 'WorkerStatus']

crazy_workers-1.3.0/crazy_workers/cli/commands/starter.py

@@ -1,36 +0,0 @@
-import os
-from rich.prompt import IntPrompt
-
-from ..ui import console, err_console
-
-
-def start_worker(manager, worker_type, worker_key=None, parameters=None):
-
-  if not worker_type:
-    # Interactive mode: list .py files in workers_dir
-    try:
-      files = [f[:-3] for f in os.listdir(manager.workers_dir) if f.endswith('.py')]
-    except Exception as e:
-      err_console().print(f'[bold red]Error reading workers directory:[/bold red] {e}')
-      return False
-
-    if not files:
-      console().print(f'[yellow]No worker scripts found in {manager.workers_dir}[/yellow]')
-      return False
-
-    console().print('\n[bold cyan]Select a worker type to start:[/bold cyan]')
-    for i, f in enumerate(files, 1):
-      console().print(f'  [bold]{i})[/bold] {f}')
-
-    choice = IntPrompt.ask('Enter the number', choices=[str(i) for i in range(1, len(files) + 1)])
-    worker_type = files[choice - 1]
-
-  success, result = manager.start_worker(worker_type, worker_key=worker_key, parameters=parameters)
-  if success:
-    console().print('[bold green]Success:[/bold green] Worker started')
-    console().print(f'  [bold]Key:[/bold] {result["worker_key"]}')
-    console().print(f'  [bold]PID:[/bold] {result["pid"]}')
-  else:
-    err_console().print(f'[bold red]Error:[/bold red] {result}')
-    return False
-  return True

crazy_workers-1.3.0/crazy_workers/cli/commands/status.py

@@ -1,82 +0,0 @@
-import json
-from datetime import datetime
-from rich.panel import Panel
-from rich.table import Table
-
-from ...boot import boot_state
-from ..ui import console
-
-
-def show_status(manager):
-  """Observability hub: boot-restore state plus the worker table."""
-  console().print(_build_header(manager))
-
-  workers = manager.list_workers()
-  if not workers:
-    console().print('[yellow]No workers found in database.[/yellow]')
-    return workers
-
-  console().print(_build_table(workers))
-  return workers
-
-
-def _build_header(manager):
-  state = boot_state(manager.workers_dir, provider=manager._boot_provider)
-  if state.mechanism == 'disabled':
-    boot_line = '[dim]boot-restore: disabled[/dim]'
-  elif not state.supported:
-    boot_line = '[dim]boot-restore: not supported on this platform[/dim]'
-  elif state.installed:
-    boot_line = f'[green]boot-restore: enabled[/green] [dim]({state.mechanism}, {state.detail})[/dim]'
-  else:
-    reason = f' — {state.detail}' if state.detail else ''
-    boot_line = f'[yellow]boot-restore: not installed[/yellow][dim]{reason}[/dim]'
-
-  body = f'[bold]Workers dir:[/bold] {manager.workers_dir}\n{boot_line}'
-  return Panel.fit(body, border_style='cyan', title='[bold cyan]Crazy Workers status[/bold cyan]')
-
-
-def _build_table(workers):
-  table = Table(
-    title='[bold cyan]Active & Registered Workers[/bold cyan]', border_style='cyan', header_style='bold magenta'
-  )
-  table.add_column('#', justify='right', style='dim')
-  table.add_column('Key', style='bold')
-  table.add_column('Type')
-  table.add_column('Status', justify='center')
-  table.add_column('PID', justify='right', style='green')
-  table.add_column('Last Action', justify='center')
-  table.add_column('Params', overflow='ellipsis')
-
-  for i, w in enumerate(workers, 1):
-    status = w['status']
-    status_style = 'green' if status == 'RUNNING' else 'yellow'
-    if status in ['CRASHED', 'FAILED']:
-      status_style = 'bold red'
-    elif status == 'STOPPED':
-      status_style = 'dim'
-    elif status == 'NEVER_STARTED':
-      status_style = 'cyan'
-
-    last_action = '-'
-    if status == 'RUNNING' and w.get('last_started_at'):
-      dt = datetime.fromisoformat(w['last_started_at'])
-      last_action = f'[green]Started {dt.strftime("%H:%M:%S")}[/green]'
-    elif w.get('last_stopped_at'):
-      dt = datetime.fromisoformat(w['last_stopped_at'])
-      last_action = f'[dim]Stopped {dt.strftime("%H:%M:%S")}[/dim]'
-
-    params_str = json.dumps(w['parameters']) if w['parameters'] else '-'
-    if len(params_str) > 30:
-      params_str = params_str[:27] + '...'
-
-    table.add_row(
-      str(i),
-      w['worker_key'] or '-',
-      w['worker_type'],
-      f'[{status_style}]{status}[/{status_style}]',
-      str(w['pid']) if w['pid'] else '-',
-      last_action,
-      params_str,
-    )
-  return table

crazy_workers-1.3.0/crazy_workers/cli/commands/stopper.py

@@ -1,30 +0,0 @@
-from rich.prompt import IntPrompt
-
-from ..ui import console, err_console
-
-
-def stop_worker(manager, worker_key):
-
-  if not worker_key:
-    # Interactive mode
-    workers = manager.list_workers()
-    running_workers = [w for w in workers if w['status'] == 'RUNNING']
-
-    if not running_workers:
-      console().print('[yellow]No running workers to stop.[/yellow]')
-      return False
-
-    console().print('\n[bold cyan]Select a worker to stop:[/bold cyan]')
-    for i, w in enumerate(running_workers, 1):
-      console().print(f'  [bold]{i})[/bold] {w["worker_key"]} [dim]({w["worker_type"]})[/dim]')
-
-    choice = IntPrompt.ask('Enter the number', choices=[str(i) for i in range(1, len(running_workers) + 1)])
-    worker_key = running_workers[choice - 1]['worker_key']
-
-  success, message = manager.stop_worker(worker_key)
-  if success:
-    console().print(f'[bold green]Success:[/bold green] {message}')
-  else:
-    err_console().print(f'[bold red]Error:[/bold red] {message}')
-    return False
-  return True

crazy_workers-1.3.0/crazy_workers/cli/main.py

@@ -1,80 +0,0 @@
-import argparse
-import sys
-from rich.panel import Panel
-
-from ..core.manager import WorkerManager
-from .commands import show_params, show_status, start_worker, stop_worker
-from .discovery import resolve_workers_dir
-from .ui import console, err_console
-
-
-def main():
-
-  def formatter(prog):
-    return argparse.HelpFormatter(prog, max_help_position=32)
-
-  parser = argparse.ArgumentParser(description='Crazy Workers CLI', formatter_class=formatter)
-  parser.add_argument('--workers-dir', help='Directory containing worker scripts')
-
-  subparsers = parser.add_subparsers(dest='command', help='Commands')
-
-  # Status command
-  subparsers.add_parser('status', help='Show workers and boot-restore status')
-
-  # Start command
-  start_parser = subparsers.add_parser('start', help='Start a worker (interactive if type missing)')
-  start_parser.add_argument('worker_type', nargs='?', help='The type (filename) of worker to start')
-  start_parser.add_argument('--key', help='Optional custom key for the worker')
-  start_parser.add_argument('--params', help='JSON string of parameters for the worker')
-
-  # Stop command
-  stop_parser = subparsers.add_parser('stop', help='Stop a worker (interactive if key missing)')
-  stop_parser.add_argument('worker_key', nargs='?', help='The key of the worker to stop')
-
-  # Params command
-  params_parser = subparsers.add_parser('params', help='Show parameters for a worker')
-  params_parser.add_argument('worker_key', nargs='?', help='The key of the worker')
-
-  args = parser.parse_args()
-
-  if not args.command:
-    console().print(
-      Panel.fit(
-        '[bold cyan]Crazy Workers CLI[/bold cyan]\n[dim]Manage your background processes with ease[/dim]',
-        border_style='cyan',
-      )
-    )
-    parser.print_help()
-    sys.exit(1)
-
-  workers_dir = resolve_workers_dir(args.workers_dir)
-  try:
-    with WorkerManager(workers_dir, create_dir=False, auto_recover=False) as manager:
-      if args.command == 'status':
-        show_status(manager)
-      elif args.command == 'start':
-        import json
-
-        params = None
-        if args.params:
-          try:
-            params = json.loads(args.params)
-          except json.JSONDecodeError:
-            err_console().print('[bold red]Error:[/bold red] Invalid JSON in --params')
-            sys.exit(1)
-
-        if not start_worker(manager, args.worker_type, worker_key=args.key, parameters=params):
-          sys.exit(1)
-      elif args.command == 'stop':
-        if not stop_worker(manager, args.worker_key):
-          sys.exit(1)
-      elif args.command == 'params':
-        if not show_params(manager, args.worker_key):
-          sys.exit(1)
-  except ValueError as e:
-    err_console().print(f'[bold red]Error:[/bold red] {e}')
-    sys.exit(1)
-
-
-if __name__ == '__main__':
-  main()