parallelazy 0.0.1__py3-none-any.whl

parallelazy/examples/example_worker_pool.py

@@ -0,0 +1,188 @@
+"""
+Worker Pool Example: Multiple instances of the same plugin.
+
+This example shows how plugins can:
+1. Create multiple instances of the same plugin
+2. Load balance work across worker instances
+3. Track metrics per worker
+4. Coordinate work distribution
+"""
+
+import logging
+import time
+from typing import Any, Dict
+
+from parallelazy.plugin import Plugin, PluginDescriptor
+from parallelazy.plugin_worker import PluginWorker
+from parallelazy.plugin_manager import PluginManager
+
+logger = logging.getLogger(__name__)
+
+
+class TaskQueue(Plugin):
+    """
+    Task distributor that publishes tasks to workers.
+    """
+
+    @classmethod
+    def Descriptor(cls, options: Dict[str, Any] = None) -> PluginDescriptor:
+        return super().Descriptor(
+            version="1.0.0",
+            author="ModularNet Team",
+            description="Task queue distributor",
+            options=options or {}
+        )
+
+    def __init__(self, worker: PluginWorker, plugin_id: str, options: Dict[str, Any]):
+        super().__init__(worker, plugin_id, options)
+        self.task_id = 0
+
+    def on_start(self) -> None:
+        logger.info(f"[{self.plugin_id}] TaskQueue started")
+        self.env_set("queue.total_tasks", 0)
+
+    def main(self):
+        self.task_id += 1
+        # Generate a task
+        task = {
+            "id": self.task_id,
+            "operation": "process",
+            "complexity": (self.task_id % 3) + 1  # 1-3
+        }
+        # Publish to workers
+        self.topic_publish("tasks.queue", task)
+        self.env_set("queue.total_tasks", self.task_id, wait=False)
+
+    def on_stop(self) -> None:
+        logger.info(f"[{self.plugin_id}] TaskQueue stopped, created {self.task_id} tasks")
+
+
+class Worker(Plugin):
+    """
+    Worker that processes tasks from the queue.
+    Multiple instances can be created.
+    """
+
+    @classmethod
+    def Descriptor(cls, options: Dict[str, Any] = None) -> PluginDescriptor:
+        return super().Descriptor(
+            version="1.0.0",
+            author="ModularNet Team",
+            description="Worker pool member",
+            options=options or {}
+        )
+
+    def __init__(self, worker: PluginWorker, plugin_id: str, options: Dict[str, Any]):
+        super().__init__(worker, plugin_id, options)
+        self.tasks_processed = 0
+        self.total_time = 0
+
+    def on_start(self) -> None:
+        logger.info(f"[{self.plugin_id}] Worker started")
+        self.topic_subscribe("tasks.queue", self.on_task)
+        # Each worker tracks its own metrics
+        self.env_set(f"worker.{self.plugin_id}.processed", 0)
+        self.env_set(f"worker.{self.plugin_id}.total_time", 0)
+
+    def on_task(self, task: Dict[str, Any]):
+        """Process a task from the queue."""
+        self.tasks_processed += 1
+        complexity = task["complexity"]
+        # Simulate work based on complexity
+        time_taken = complexity * 0.1
+        self.total_time += time_taken
+
+        logger.info(
+            f"[{self.plugin_id}] Task #{task['id']}: complexity={complexity}, time={time_taken:.1f}s"
+        )
+
+        self.env_set(f"worker.{self.plugin_id}.processed", self.tasks_processed, wait=False)
+        self.env_set(f"worker.{self.plugin_id}.total_time", self.total_time, wait=False)
+
+    def main(self):
+        pass
+
+    def on_stop(self) -> None:
+        logger.info(
+            f"[{self.plugin_id}] Worker stopped: "
+            f"processed={self.tasks_processed}, total_time={self.total_time:.1f}s"
+        )
+
+
+def main():
+    """Run the worker pool example."""
+    manager = PluginManager()
+
+    # Create queue
+    queue_desc = TaskQueue.Descriptor()
+    manager.load_plugin(queue_desc)
+    manager.start_plugin(queue_desc.name)
+
+    # Create a pool of workers
+    num_workers = 3
+    worker_descs = []
+    for i in range(num_workers):
+        worker_desc = Worker.Descriptor()
+        worker_desc.name = f"worker-{i}"  # Unique name per instance
+        manager.load_plugin(worker_desc)
+        manager.start_plugin(worker_desc.name)
+        worker_descs.append(worker_desc)
+
+    # Let them run
+    print(f"Queue and {num_workers} workers running... Press Ctrl+C to stop.")
+    time.sleep(5)
+    print("Stopping workers...")
+
+    # Stop all workers
+    for worker_desc in worker_descs:
+        manager.stop_plugin(worker_desc.name)
+
+    manager.stop_plugin(queue_desc.name)
+
+    time.sleep(1)
+
+    # Show worker metrics
+    print("\nWorker pool metrics:")
+    all_vars = manager.env_get()
+    for name, value in sorted(all_vars.items()):
+        if name.startswith("worker."):
+            print(f"  {name}: {value}")
+
+    # VERIFICATION
+    print("\n=== VERIFICATION ===")
+    print("PREDICTED BEHAVIOR:")
+    print("1. TaskQueue generates ~5 tasks in 5 seconds")
+    print("2. 3 workers load-balance tasks via pub/sub")
+    print("3. Each worker tracks its own processed count")
+    print("4. Total processed across all workers >= queue total\n")
+
+    total_tasks = all_vars.get("queue.total_tasks", 0)
+    assert total_tasks > 0, "Queue should generate tasks"
+    print(f"✓ Queue generated: {total_tasks} tasks")
+
+    # Count workers and their processed tasks
+    workers_found = {}
+    for name, value in all_vars.items():
+        if name.startswith("worker.") and name.endswith(".processed"):
+            worker_name = name.replace("worker.", "").replace(".processed", "")
+            workers_found[worker_name] = value
+
+    assert len(workers_found) == num_workers, f"Should have {num_workers} workers, found {len(workers_found)}"
+    total_processed = sum(workers_found.values())
+    assert total_processed > 0, "Workers should process tasks"
+    
+    print(f"✓ Workers found: {num_workers}")
+    print(f"✓ Total tasks processed across all workers: {total_processed}")
+    for worker_id, count in sorted(workers_found.items()):
+        print(f"  - {worker_id}: {count} tasks")
+    print("✓ All worker pool assertions passed!")
+
+    # Clean up
+    for worker_desc in worker_descs:
+        manager.unload_plugin(worker_desc.name)
+
+    manager.unload_plugin(queue_desc.name)
+
+
+if __name__ == "__main__":
+    main()

parallelazy/plugin.py

@@ -0,0 +1,223 @@
+"""
+Base Plugin Interface: Abstract base class for all plugins.
+
+Defines the lifecycle, subscription, and service interfaces that all
+plugins must implement or can override.
+"""
+
+from typing import Any, Dict, Optional, TYPE_CHECKING, Self
+from enum import Enum
+
+from parallelazy.utils import camel_to_kebab
+
+
+if TYPE_CHECKING:
+    from parallelazy.plugin_worker import PluginWorker 
+
+
+class PluginState(Enum):
+    """Plugin lifecycle states."""
+    INITIALIZED = "initialized"
+    LOADED = "loaded"
+    RUNNING = "running"
+    STOPPED = "stopped"
+    ERROR = "error"
+
+    def __str__(self): return str(self.value)
+
+"""
+Plugin Descriptor: Serializable metadata for dynamic plugin loading.
+
+Describes a plugin's identity, location, and configuration options.
+"""
+
+from dataclasses import dataclass, field, asdict
+from typing import Any, Dict, Optional
+import json
+
+
+@dataclass
+class PluginDescriptor:
+    """
+    Serializable descriptor for plugin metadata and instantiation.
+    
+    Attributes:
+        name: Unique plugin identifier
+        version: Semantic version (e.g., "1.0.0")
+        package: Full package path (e.g., "modularnet.plugins.my_plugin")
+        class_name: Plugin class name in the package
+        description: Human-readable description
+        author: Plugin author
+        options: Default configuration options for the plugin
+        dependencies: List of required plugin names
+    """
+    name: str
+    version: str
+    package: str
+    class_name: str
+    description: str = ""
+    author: str = ""
+    options: Dict[str, Any] = field(default_factory=dict)
+    dependencies: list = field(default_factory=list)
+    
+    def to_dict(self) -> Dict[str, Any]:
+        """Convert descriptor to dictionary (JSON-serializable)."""
+        return asdict(self)
+    
+    def to_json(self) -> str:
+        """Serialize descriptor to JSON."""
+        return json.dumps(self.to_dict(), indent=2)
+    
+    @classmethod
+    def from_dict(cls, data: Dict[str, Any]) -> "PluginDescriptor":
+        """Create descriptor from dictionary."""
+        return cls(**data)
+    
+    @classmethod
+    def from_json(cls, json_str: str) -> "PluginDescriptor":
+        """Create descriptor from JSON string."""
+        return cls.from_dict(json.loads(json_str))
+    
+    def __str__(self) -> str:
+        return f"{self.name}@{self.version} ({self.package}.{self.class_name})"
+    
+    def __repr__(self) -> str:
+        return f"PluginDescriptor({self})"
+
+
+class Plugin:
+    """
+    Base class for all ModularNet plugins.
+    
+    Plugins run in isolated processes and communicate via:
+    1. Control channel: Load/unload/configure/service commands
+    2. Data plane: Pub/sub topics for high-bandwidth data exchange
+    3. Env plane: Shared environment variables with change notifications
+    4. Manager proxy: Direct access to manager services (get/set env, publish, invoke)
+    
+    Subclasses must implement:
+    - on_start: Initialize plugin resources (optional)
+    - on_stop: Clean up resources (optional)
+    - main: Main execution loop (optional)
+    - on_error: Handle errors (optional)
+
+
+    Direct manager access via self.manager:
+        model = self.env_get("model")
+        self.env_set("status", "ready")
+        self.topic_publish("topic", data)
+    """
+
+    _registry: dict[str, type[Self]] = {}
+        
+    def __init_subclass__(cls, plugin_name: str | None = None, **kwargs):
+        super().__init_subclass__(**kwargs)
+        
+        # Determine registry key: custom plugin_name or default class name
+        key = plugin_name or cls.__name__
+        
+        if key in cls._registry:
+            raise ValueError(f"Plugin '{key}' is already registered.")
+            
+        cls._registry[key] = cls
+
+
+    @classmethod
+    def default_options(cls) -> Dict[str, Any]:
+        return {}
+
+    @classmethod
+    def Descriptor(cls, version:str, author:str, description:str=None, name:str=None, options: Dict[str, Any] = None) -> PluginDescriptor:
+        all_options = cls.default_options().copy()
+        all_options.update(options or {})
+
+
+        return PluginDescriptor(
+            version=version,
+            author=author,
+            options=all_options,
+            name= camel_to_kebab(name or cls.__name__),
+            package=f"{cls.__module__}",
+            class_name=cls.__name__,
+            description=description or f"{cls.__doc__}",
+            
+        )
+
+    def __init__(self, worker: "PluginWorker", plugin_id: str, options: Optional[Dict[str, Any]] = None):
+        """
+        Initialize plugin.
+        
+        Args:
+            plugin_id: Unique identifier for this plugin instance
+            config: Configuration options dictionary
+            worker: PluginWorker instance for managing the plugin
+        """
+        all_options = self.default_options().copy()
+        all_options.update(options or {})
+
+        self._worker = worker
+        self.plugin_id = plugin_id
+        self.state = PluginState.INITIALIZED
+        self.options = all_options
+        self._is_running = False
+    
+    
+    def main(self): pass
+    
+    def on_start(self) -> None:
+        """
+        Called when plugin is started.
+        
+        Initialize resources, open connections, start background tasks, etc.
+        Raise exception on failure.
+        """
+        pass
+    
+    def on_stop(self) -> None:
+        """
+        Called when plugin is stopped.
+        
+        Clean up resources, close connections, stop background tasks, etc.
+        """
+        pass
+    
+    def on_error(self, error: Exception, source: str=None) -> None:
+        """ Called when an error occurs in the plugin."""
+        pass
+    
+    @property
+    def is_running(self) -> bool:
+        """Whether plugin is currently running."""
+        return self._is_running
+    
+    def stop(self) -> None:
+        """Stop the plugin."""
+        
+        self._worker.stop()
+
+    def topic_subscribe(self, topic:str, callback:str) -> None:
+        """Subscribe to a data topic or environment variable."""
+        self._worker.topic_subscribe(topic, callback)
+
+    def topic_publish(self, topic:str, data:Any) -> None:
+        """Publish data to a topic."""
+        self._worker.topic_publish(topic, data)
+
+    def env_set(self, name:str, value:Any, wait: bool = True) -> None:
+        """Set an environment variable."""
+        self._worker.env_set(name, value, wait=wait)
+
+    def env_get(self, name:str=None, default:Any=None) -> dict|Any:
+        """Get an environment variable."""
+        if name is None:
+            return self._worker.env_get_all() 
+
+        return self._worker.env_get(name) or default
+    
+    def env_subscribe(self, name:str, callback:str) -> None:
+        """Subscribe to changes in an environment variable."""
+        self._worker.env_subscribe(name, callback)
+    
+    def plugin_call(self, plugin_id: str, method_name: str, *args, **kwargs) -> Any:
+        """Call a method provided by the manager or another plugin."""
+        return self._worker.plugin_call(plugin_id, method_name, *args, **kwargs)