crazy-workers 0.1.0__tar.gz

crazy_workers-0.1.0/LICENSE

@@ -0,0 +1,19 @@
+Copyright (c) 2024 GioVanni Colasanto
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

crazy_workers-0.1.0/PKG-INFO

@@ -0,0 +1,247 @@
+Metadata-Version: 2.4
+Name: crazy-workers
+Version: 0.1.0
+Summary: A Python library for managing background worker processes with persistent state, automatic recovery, and a CLI.
+Author: GioVanni Colasanto
+License: MIT
+Project-URL: Homepage, https://github.com/Vanni-broUser/crazy-workers
+Project-URL: Bug Tracker, https://github.com/Vanni-broUser/crazy-workers/issues
+Project-URL: Source, https://github.com/Vanni-broUser/crazy-workers
+Keywords: workers,background,processes,process-manager,task-runner,cli,psutil
+Classifier: Development Status :: 4 - Beta
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Topic :: Software Development :: Libraries :: Python Modules
+Classifier: Topic :: System :: Systems Administration
+Classifier: Topic :: Utilities
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: sqlalchemy>=2.0.0
+Requires-Dist: psutil>=5.9.0
+Requires-Dist: rich>=13.0.0
+Provides-Extra: dev
+Requires-Dist: ruff; extra == "dev"
+Requires-Dist: pytest>=7.0; extra == "dev"
+Requires-Dist: coverage; extra == "dev"
+Requires-Dist: flask; extra == "dev"
+Dynamic: license-file
+
+# Crazy Workers
+
+A Python library for managing background worker processes with persistent state, automatic crash recovery, and a built-in CLI.
+
+[![Python 3.10+](https://img.shields.io/badge/python-3.10+-blue.svg)](https://www.python.org/)
+[![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](LICENSE)
+
+## Features
+
+- **Persistent State** — SQLite database tracks worker status, PIDs, and parameters across restarts.
+- **Process Management** — Start, stop, and monitor background Python scripts as independent OS processes.
+- **Automatic Recovery** — Detects crashed workers and restarts them on application boot.
+- **Child Process Control** — On stop, terminates unmanaged subprocesses while preserving independently-managed nested workers.
+- **CLI Interface** — Manage workers from the terminal with interactive prompts and auto-discovery (see [CLI.md](CLI.md)).
+- **Security** — Built-in protection against path traversal in worker type and key names.
+- **Observability** — Per-worker file logging; all service files (DB, lock, logs) live in a `.service/` folder inside your workers directory.
+- **Zombie Protection** — Distinguishes active processes from zombies using `psutil`.
+- **Gunicorn-safe** — File-based lock prevents concurrent recovery runs across multiple workers.
+
+## Installation
+
+```bash
+pip install crazy-workers
+```
+
+Or from source:
+
+```bash
+git clone https://github.com/Vanni-broUser/crazy-workers
+cd crazy-workers
+pip install .
+```
+
+## Quick Start
+
+### 1. Create a worker script
+
+```python
+# workers/my_worker.py
+import json, sys, time
+
+params = json.loads(sys.argv[1]) if len(sys.argv) > 1 else {}
+duration = params.get('duration', 60)
+
+for _ in range(duration):
+    time.sleep(1)
+```
+
+### 2. Manage it from Python
+
+```python
+from crazy_workers import WorkerManager
+
+manager = WorkerManager('workers')
+
+# Start
+success, result = manager.start_worker(
+    'my_worker',
+    worker_key='job_1',
+    parameters={'duration': 30},
+)
+print(result['pid'])   # OS process ID
+print(result['status'])  # 'RUNNING'
+
+# List
+for w in manager.list_workers():
+    print(w['worker_key'], w['status'])
+
+# Stop
+manager.stop_worker('job_1')
+
+# Recover crashed workers (call on app startup)
+restarted = manager.recover_workers()
+
+manager.dispose()  # releases DB connection; does NOT kill workers
+```
+
+### 3. Or from the CLI
+
+```bash
+crazy-workers list
+crazy-workers start my_worker --key job_1 --params '{"duration": 30}'
+crazy-workers stop job_1
+crazy-workers restore
+```
+
+See [CLI.md](CLI.md) for full CLI documentation.
+
+## API Reference
+
+### `WorkerManager(workers_dir, create_dir=True)`
+
+| Parameter | Type | Default | Description |
+|-----------|------|---------|-------------|
+| `workers_dir` | `str` | `'workers'` | Directory containing worker `.py` scripts |
+| `create_dir` | `bool` | `True` | Create `workers_dir` and `.service/` if they don't exist |
+
+### `start_worker(worker_type, worker_key=None, parameters=None, env=None)`
+
+| Parameter | Type | Default | Description |
+|-----------|------|---------|-------------|
+| `worker_type` | `str` | — | Filename (without `.py`) of the worker script |
+| `worker_key` | `str` | `worker_type` | Unique identifier; allows multiple instances of the same type |
+| `parameters` | `dict` | `{}` | JSON-serializable dict passed as `sys.argv[1]` to the worker |
+| `env` | `dict` | `None` | Extra environment variables injected into the worker process |
+
+Returns `(bool, dict | str)` — `(True, worker_dict)` on success, `(False, error_message)` on failure.
+
+### `stop_worker(worker_key)`
+
+Gracefully terminates the worker (SIGTERM → SIGKILL after timeout). Returns `(bool, str)`.
+
+### `list_workers()`
+
+Returns a list of worker dicts including RUNNING, STOPPED, CRASHED, and NEVER_STARTED (filesystem-discovered) workers.
+
+### `recover_workers()`
+
+Restarts any worker whose DB status is RUNNING but whose process is dead. Uses a file lock to prevent concurrent recovery. Returns a list of restarted keys.
+
+### `dispose()`
+
+Closes the database connection and clears internal process references. Does **not** kill background workers — they continue running independently.
+
+## Worker Script Contract
+
+A worker receives its parameters as a JSON string in `sys.argv[1]`:
+
+```python
+import json, sys
+
+params = json.loads(sys.argv[1]) if len(sys.argv) > 1 else {}
+# ... do work ...
+```
+
+## Project Structure
+
+```
+crazy_workers/       # Library package
+  core/              # WorkerManager, process engine, recovery lock
+  cli/               # CLI entry point, commands, discovery
+  database/          # SQLAlchemy schema and SQLite storage
+example_app/         # Flask demo application
+  app.py
+  workers/           # Example worker scripts
+tests/
+  core/              # Unit tests for core modules
+  cli/               # Unit tests for CLI modules
+  database/          # Unit tests for storage layer
+  integration/       # Full-stack integration tests (real processes)
+  app/               # Tests for the example Flask app
+```
+
+## Flask Integration
+
+```python
+from crazy_workers import WorkerManager
+
+def create_app():
+    app = Flask(__name__)
+    manager = WorkerManager('workers')
+
+    @app.route('/workers/start', methods=['POST'])
+    def start():
+        data = request.json
+        success, result = manager.start_worker(
+            data['worker_type'],
+            worker_key=data.get('worker_key'),
+            parameters=data.get('parameters', {}),
+        )
+        return (jsonify(result), 200) if success else (jsonify({'error': result}), 400)
+
+    manager.recover_workers()  # restart any crashed workers on boot
+    return app
+```
+
+See `example_app/app.py` for a complete example.
+
+## Gunicorn / Multi-Process Servers
+
+When using a pre-fork server like Gunicorn:
+
+- **Recovery is atomic** — a file lock (`.service/workers.db.recovery.lock`) ensures `recover_workers()` runs once even when multiple workers boot simultaneously.
+- **Workers outlive their parent** — if a Gunicorn worker is recycled, background processes keep running. The next recovery cycle re-attaches or restarts them.
+
+## Development
+
+### Setup
+
+```bash
+git clone https://github.com/Vanni-broUser/crazy-workers
+cd crazy-workers
+pip install -e .[dev]
+```
+
+### Commands
+
+```bash
+# Lint and format
+ruff check . --fix && ruff format .
+
+# Run tests
+pytest
+
+# Run tests with coverage
+coverage run -m pytest && coverage report
+```
+
+### Standards
+
+See [AI.md](AI.md) for the full coding and testing standards used in this project.
+

crazy_workers-0.1.0/README.md

@@ -0,0 +1,212 @@
+# Crazy Workers
+
+A Python library for managing background worker processes with persistent state, automatic crash recovery, and a built-in CLI.
+
+[![Python 3.10+](https://img.shields.io/badge/python-3.10+-blue.svg)](https://www.python.org/)
+[![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](LICENSE)
+
+## Features
+
+- **Persistent State** — SQLite database tracks worker status, PIDs, and parameters across restarts.
+- **Process Management** — Start, stop, and monitor background Python scripts as independent OS processes.
+- **Automatic Recovery** — Detects crashed workers and restarts them on application boot.
+- **Child Process Control** — On stop, terminates unmanaged subprocesses while preserving independently-managed nested workers.
+- **CLI Interface** — Manage workers from the terminal with interactive prompts and auto-discovery (see [CLI.md](CLI.md)).
+- **Security** — Built-in protection against path traversal in worker type and key names.
+- **Observability** — Per-worker file logging; all service files (DB, lock, logs) live in a `.service/` folder inside your workers directory.
+- **Zombie Protection** — Distinguishes active processes from zombies using `psutil`.
+- **Gunicorn-safe** — File-based lock prevents concurrent recovery runs across multiple workers.
+
+## Installation
+
+```bash
+pip install crazy-workers
+```
+
+Or from source:
+
+```bash
+git clone https://github.com/Vanni-broUser/crazy-workers
+cd crazy-workers
+pip install .
+```
+
+## Quick Start
+
+### 1. Create a worker script
+
+```python
+# workers/my_worker.py
+import json, sys, time
+
+params = json.loads(sys.argv[1]) if len(sys.argv) > 1 else {}
+duration = params.get('duration', 60)
+
+for _ in range(duration):
+    time.sleep(1)
+```
+
+### 2. Manage it from Python
+
+```python
+from crazy_workers import WorkerManager
+
+manager = WorkerManager('workers')
+
+# Start
+success, result = manager.start_worker(
+    'my_worker',
+    worker_key='job_1',
+    parameters={'duration': 30},
+)
+print(result['pid'])   # OS process ID
+print(result['status'])  # 'RUNNING'
+
+# List
+for w in manager.list_workers():
+    print(w['worker_key'], w['status'])
+
+# Stop
+manager.stop_worker('job_1')
+
+# Recover crashed workers (call on app startup)
+restarted = manager.recover_workers()
+
+manager.dispose()  # releases DB connection; does NOT kill workers
+```
+
+### 3. Or from the CLI
+
+```bash
+crazy-workers list
+crazy-workers start my_worker --key job_1 --params '{"duration": 30}'
+crazy-workers stop job_1
+crazy-workers restore
+```
+
+See [CLI.md](CLI.md) for full CLI documentation.
+
+## API Reference
+
+### `WorkerManager(workers_dir, create_dir=True)`
+
+| Parameter | Type | Default | Description |
+|-----------|------|---------|-------------|
+| `workers_dir` | `str` | `'workers'` | Directory containing worker `.py` scripts |
+| `create_dir` | `bool` | `True` | Create `workers_dir` and `.service/` if they don't exist |
+
+### `start_worker(worker_type, worker_key=None, parameters=None, env=None)`
+
+| Parameter | Type | Default | Description |
+|-----------|------|---------|-------------|
+| `worker_type` | `str` | — | Filename (without `.py`) of the worker script |
+| `worker_key` | `str` | `worker_type` | Unique identifier; allows multiple instances of the same type |
+| `parameters` | `dict` | `{}` | JSON-serializable dict passed as `sys.argv[1]` to the worker |
+| `env` | `dict` | `None` | Extra environment variables injected into the worker process |
+
+Returns `(bool, dict | str)` — `(True, worker_dict)` on success, `(False, error_message)` on failure.
+
+### `stop_worker(worker_key)`
+
+Gracefully terminates the worker (SIGTERM → SIGKILL after timeout). Returns `(bool, str)`.
+
+### `list_workers()`
+
+Returns a list of worker dicts including RUNNING, STOPPED, CRASHED, and NEVER_STARTED (filesystem-discovered) workers.
+
+### `recover_workers()`
+
+Restarts any worker whose DB status is RUNNING but whose process is dead. Uses a file lock to prevent concurrent recovery. Returns a list of restarted keys.
+
+### `dispose()`
+
+Closes the database connection and clears internal process references. Does **not** kill background workers — they continue running independently.
+
+## Worker Script Contract
+
+A worker receives its parameters as a JSON string in `sys.argv[1]`:
+
+```python
+import json, sys
+
+params = json.loads(sys.argv[1]) if len(sys.argv) > 1 else {}
+# ... do work ...
+```
+
+## Project Structure
+
+```
+crazy_workers/       # Library package
+  core/              # WorkerManager, process engine, recovery lock
+  cli/               # CLI entry point, commands, discovery
+  database/          # SQLAlchemy schema and SQLite storage
+example_app/         # Flask demo application
+  app.py
+  workers/           # Example worker scripts
+tests/
+  core/              # Unit tests for core modules
+  cli/               # Unit tests for CLI modules
+  database/          # Unit tests for storage layer
+  integration/       # Full-stack integration tests (real processes)
+  app/               # Tests for the example Flask app
+```
+
+## Flask Integration
+
+```python
+from crazy_workers import WorkerManager
+
+def create_app():
+    app = Flask(__name__)
+    manager = WorkerManager('workers')
+
+    @app.route('/workers/start', methods=['POST'])
+    def start():
+        data = request.json
+        success, result = manager.start_worker(
+            data['worker_type'],
+            worker_key=data.get('worker_key'),
+            parameters=data.get('parameters', {}),
+        )
+        return (jsonify(result), 200) if success else (jsonify({'error': result}), 400)
+
+    manager.recover_workers()  # restart any crashed workers on boot
+    return app
+```
+
+See `example_app/app.py` for a complete example.
+
+## Gunicorn / Multi-Process Servers
+
+When using a pre-fork server like Gunicorn:
+
+- **Recovery is atomic** — a file lock (`.service/workers.db.recovery.lock`) ensures `recover_workers()` runs once even when multiple workers boot simultaneously.
+- **Workers outlive their parent** — if a Gunicorn worker is recycled, background processes keep running. The next recovery cycle re-attaches or restarts them.
+
+## Development
+
+### Setup
+
+```bash
+git clone https://github.com/Vanni-broUser/crazy-workers
+cd crazy-workers
+pip install -e .[dev]
+```
+
+### Commands
+
+```bash
+# Lint and format
+ruff check . --fix && ruff format .
+
+# Run tests
+pytest
+
+# Run tests with coverage
+coverage run -m pytest && coverage report
+```
+
+### Standards
+
+See [AI.md](AI.md) for the full coding and testing standards used in this project.
+

crazy_workers-0.1.0/crazy_workers/__init__.py

@@ -0,0 +1,5 @@
+from .core.manager import WorkerManager
+from .database.schema import WorkerStatus
+
+
+__all__ = ['WorkerManager', 'WorkerStatus']

crazy_workers-0.1.0/crazy_workers/_bootstrap.py

@@ -0,0 +1,33 @@
+"""
+Thin launcher invoked by WorkerManager for every worker subprocess.
+Configures logging once so individual worker scripts don't have to.
+
+Invocation (managed internally by WorkerManager):
+    python -m crazy_workers._bootstrap <worker_path> <json_params>
+"""
+
+import logging
+import os
+import runpy
+import sys
+
+
+def main():
+  logging.basicConfig(
+    level=logging.INFO,
+    format='%(asctime)s - %(levelname)s - %(message)s',
+    stream=sys.stderr,
+    force=True,
+  )
+
+  # Restore sys.argv so the worker sees [worker_path, json_params]
+  sys.argv = sys.argv[1:]
+
+  worker_path = sys.argv[0]
+  sys.path.insert(0, os.path.dirname(os.path.abspath(worker_path)))
+
+  runpy.run_path(worker_path, run_name='__main__')
+
+
+if __name__ == '__main__':
+  main()

crazy_workers-0.1.0/crazy_workers/cli/__init__.py

@@ -0,0 +1,4 @@
+from .main import main
+
+
+__all__ = ['main']

crazy_workers-0.1.0/crazy_workers/cli/commands/__init__.py

@@ -0,0 +1,8 @@
+from .lister import list_workers
+from .params import show_params
+from .restorer import restore_workers
+from .starter import start_worker
+from .stopper import stop_worker
+
+
+__all__ = ['list_workers', 'show_params', 'start_worker', 'stop_worker', 'restore_workers']

crazy_workers-0.1.0/crazy_workers/cli/commands/lister.py

@@ -0,0 +1,57 @@
+import json
+from datetime import datetime
+from rich.table import Table
+
+from ..ui import console
+
+
+def list_workers(manager):
+  workers = manager.list_workers()
+  if not workers:
+    console().print('[yellow]No workers found in database.[/yellow]')
+    return []
+  else:
+    table = Table(
+      title='[bold cyan]Active & Registered Workers[/bold cyan]', border_style='cyan', header_style='bold magenta'
+    )
+    table.add_column('#', justify='right', style='dim')
+    table.add_column('Key', style='bold')
+    table.add_column('Type')
+    table.add_column('Status', justify='center')
+    table.add_column('PID', justify='right', style='green')
+    table.add_column('Last Action', justify='center')
+    table.add_column('Params', overflow='ellipsis')
+
+    for i, w in enumerate(workers, 1):
+      status = w['status']
+      status_style = 'green' if status == 'RUNNING' else 'yellow'
+      if status in ['CRASHED', 'FAILED']:
+        status_style = 'bold red'
+      elif status == 'STOPPED':
+        status_style = 'dim'
+      elif status == 'NEVER_STARTED':
+        status_style = 'cyan'
+
+      last_action = '-'
+      if status == 'RUNNING' and w.get('last_started_at'):
+        dt = datetime.fromisoformat(w['last_started_at'])
+        last_action = f'[green]Started {dt.strftime("%H:%M:%S")}[/green]'
+      elif w.get('last_stopped_at'):
+        dt = datetime.fromisoformat(w['last_stopped_at'])
+        last_action = f'[dim]Stopped {dt.strftime("%H:%M:%S")}[/dim]'
+
+      params_str = json.dumps(w['parameters']) if w['parameters'] else '-'
+      if len(params_str) > 30:
+        params_str = params_str[:27] + '...'
+
+      table.add_row(
+        str(i),
+        w['worker_key'] or '-',
+        w['worker_type'],
+        f'[{status_style}]{status}[/{status_style}]',
+        str(w['pid']) if w['pid'] else '-',
+        last_action,
+        params_str,
+      )
+    console().print(table)
+    return workers

crazy_workers-0.1.0/crazy_workers/cli/commands/params.py

@@ -0,0 +1,37 @@
+import json
+from rich.prompt import IntPrompt
+
+from ..ui import console, err_console
+
+
+def show_params(manager, worker_key):
+
+  workers = manager.list_workers()
+  if not workers:
+    console().print('[yellow]No workers found.[/yellow]')
+    return False
+
+  if not worker_key:
+    # Interactive mode
+    active_workers = [w for w in workers if w['worker_key'] is not None]
+
+    if not active_workers:
+      console().print('[yellow]No registered workers to show parameters for.[/yellow]')
+      return False
+
+    console().print('\n[bold cyan]Select a worker to show parameters:[/bold cyan]')
+    for i, w in enumerate(active_workers, 1):
+      status_style = 'green' if w['status'] == 'RUNNING' else 'dim'
+      console().print(f'  [bold]{i})[/bold] {w["worker_key"]} [{status_style}]({w["status"]})[/{status_style}]')
+
+    choice = IntPrompt.ask('Enter the number', choices=[str(i) for i in range(1, len(active_workers) + 1)])
+    selected_worker = active_workers[choice - 1]
+  else:
+    selected_worker = next((w for w in workers if w['worker_key'] == worker_key), None)
+    if not selected_worker:
+      err_console().print(f'[bold red]Error:[/bold red] Worker {worker_key} not found')
+      return False
+
+  console().print(f'\n[bold cyan]Parameters for worker:[/bold cyan] {selected_worker["worker_key"]}')
+  console().print_json(json.dumps(selected_worker['parameters']))
+  return True

crazy_workers-0.1.0/crazy_workers/cli/commands/restorer.py

@@ -0,0 +1,14 @@
+from ..ui import console
+
+
+def restore_workers(manager):
+  restarted = manager.recover_workers()
+
+  if restarted:
+    console().print(f'[bold green]Successfully restored {len(restarted)} workers:[/bold green]')
+    for key in restarted:
+      console().print(f'  - {key}')
+    return True
+  else:
+    console().print('[yellow]No workers needed restoration.[/yellow]')
+    return True

crazy_workers-0.1.0/crazy_workers/cli/commands/starter.py

@@ -0,0 +1,36 @@
+import os
+from rich.prompt import IntPrompt
+
+from ..ui import console, err_console
+
+
+def start_worker(manager, worker_type, worker_key=None, parameters=None):
+
+  if not worker_type:
+    # Interactive mode: list .py files in workers_dir
+    try:
+      files = [f[:-3] for f in os.listdir(manager.workers_dir) if f.endswith('.py')]
+    except Exception as e:
+      err_console().print(f'[bold red]Error reading workers directory:[/bold red] {e}')
+      return False
+
+    if not files:
+      console().print(f'[yellow]No worker scripts found in {manager.workers_dir}[/yellow]')
+      return False
+
+    console().print('\n[bold cyan]Select a worker type to start:[/bold cyan]')
+    for i, f in enumerate(files, 1):
+      console().print(f'  [bold]{i})[/bold] {f}')
+
+    choice = IntPrompt.ask('Enter the number', choices=[str(i) for i in range(1, len(files) + 1)])
+    worker_type = files[choice - 1]
+
+  success, result = manager.start_worker(worker_type, worker_key=worker_key, parameters=parameters)
+  if success:
+    console().print('[bold green]Success:[/bold green] Worker started')
+    console().print(f'  [bold]Key:[/bold] {result["worker_key"]}')
+    console().print(f'  [bold]PID:[/bold] {result["pid"]}')
+  else:
+    err_console().print(f'[bold red]Error:[/bold red] {result}')
+    return False
+  return True