parallelazy 0.0.1__py3-none-any.whl

parallelazy/examples/example_env_variables.py

@@ -0,0 +1,158 @@
+"""
+Environment Variables Example: Demonstrate shared state management.
+
+This example shows how plugins can:
+1. Set and get environment variables in the manager
+2. Subscribe to changes in environment variables
+3. Use environment variables for cross-plugin communication
+"""
+
+import logging
+import time
+from typing import Any, Dict
+
+from parallelazy.plugin import Plugin, PluginDescriptor
+from parallelazy.plugin_worker import PluginWorker
+from parallelazy.plugin_manager import PluginManager
+
+logger = logging.getLogger(__name__)
+
+
+class ConfigWriter(Plugin):
+    """
+    Plugin that periodically updates shared configuration.
+    Simulates a configuration server updating settings.
+    """
+
+    @classmethod
+    def Descriptor(cls, options: Dict[str, Any] = None) -> PluginDescriptor:
+        return super().Descriptor(
+            version="1.0.0",
+            author="ModularNet Team",
+            description="Writes configuration to environment variables",
+            options=options or {}
+        )
+    
+    def __init__(self, worker: PluginWorker, plugin_id: str, options: Dict[str, Any]):
+        super().__init__(worker, plugin_id, options)
+        self.iteration = 0
+
+    def on_start(self) -> None:
+        logger.info(f"[{self.plugin_id}] ConfigWriter started")
+        self.env_set("config.batch_size", 32)
+        self.env_set("config.learning_rate", 0.001)
+        self.env_set("config.model_type", "resnet50")
+
+    def main(self):
+        self.iteration += 1
+        # Periodically update configuration
+        if self.iteration % 10 == 0:
+            new_lr = 0.001 * (0.9 ** (self.iteration // 10))
+            self.env_set("config.learning_rate", new_lr)
+            logger.info(f"[{self.plugin_id}] Updated learning rate to {new_lr:.6f}")
+
+    def on_stop(self) -> None:
+        logger.info(f"[{self.plugin_id}] ConfigWriter stopped after {self.iteration} iterations")
+
+
+
+class ConfigReader(Plugin):
+    """
+    Plugin that subscribes to configuration changes.
+    Simulates a worker that reacts to configuration updates.
+    """
+
+    @classmethod
+    def Descriptor(cls, options: Dict[str, Any] = None) -> PluginDescriptor:
+        return super().Descriptor(
+            version="1.0.0",
+            author="ModularNet Team",
+            description="Reads and reacts to configuration changes",
+            options=options or {}
+        )
+
+    def __init__(self, worker: PluginWorker, plugin_id: str, options: Dict[str, Any]):
+        super().__init__(worker, plugin_id, options)
+        self.current_config = {}
+
+    def on_start(self) -> None:
+        logger.info(f"[{self.plugin_id}] ConfigReader started")
+        # Subscribe to all config changes
+        self.env_subscribe("config.batch_size", self.on_config_change)
+        self.env_subscribe("config.learning_rate", self.on_config_change)
+        self.env_subscribe("config.model_type", self.on_config_change)
+
+    def on_config_change(self, value: Any):
+        # The callback receives the new value, not the name
+        # (the name is implied by which variable we subscribed to)
+        logger.info(f"[{self.plugin_id}] Config changed: {value}")
+
+    def main(self):
+        # Read current config without subscribing
+        lr = self.env_get("config.learning_rate", 0.001)
+
+    def on_stop(self) -> None:
+        logger.info(f"[{self.plugin_id}] ConfigReader stopped")
+        logger.info(f"[{self.plugin_id}] Final config: {self.current_config}")
+
+
+def main():
+    """Run the environment variables example."""
+    manager = PluginManager()
+
+    # Create and load plugins
+    writer_desc = ConfigWriter.Descriptor()
+    reader_desc = ConfigReader.Descriptor()
+
+    manager.load_plugin(writer_desc)
+    manager.load_plugin(reader_desc)
+
+    manager.start_plugin(writer_desc.name)
+    manager.start_plugin(reader_desc.name)
+
+    # Let them run for a bit
+    print("Plugins running... Press Ctrl+C to stop.")
+    time.sleep(8)
+    print("Stopping plugins...")
+
+    manager.stop_plugin(writer_desc.name)
+    manager.stop_plugin(reader_desc.name)
+
+    # Show final state
+    time.sleep(1)
+    all_vars = manager.env_get()
+    print("\nFinal environment state:")
+    for name, value in all_vars.items():
+        if name.startswith("config."):
+            print(f"  {name}: {value}")
+
+    # VERIFICATION
+    print("\n=== VERIFICATION ===")
+    print("PREDICTED BEHAVIOR:")
+    print("1. ConfigWriter sets initial config (batch_size=32, learning_rate=0.001, model_type=resnet50)")
+    print("2. ConfigReader subscribes and receives updates")
+    print("3. Learning rate decays over iterations")
+    print("4. Both plugins update same environment variables\n")
+    
+    batch_size = all_vars.get("config.batch_size")
+    learning_rate = all_vars.get("config.learning_rate")
+    model_type = all_vars.get("config.model_type")
+    
+    assert batch_size is not None, "config.batch_size should be set"
+    assert learning_rate is not None, "config.learning_rate should be set"
+    assert model_type is not None, "config.model_type should be set"
+    assert batch_size == 32, f"batch_size should be 32, got {batch_size}"
+    assert model_type == "resnet50", f"model_type should be resnet50, got {model_type}"
+    assert learning_rate < 0.001, f"learning_rate should decay below 0.001, got {learning_rate}"
+    
+    print(f"✓ config.batch_size: {batch_size} (unchanged)")
+    print(f"✓ config.learning_rate: {learning_rate} (decayed from 0.001)")
+    print(f"✓ config.model_type: {model_type} (unchanged)")
+    print("✓ All assertions passed!")
+
+    manager.unload_plugin(writer_desc.name)
+    manager.unload_plugin(reader_desc.name)
+
+
+if __name__ == "__main__":
+    main()

parallelazy/examples/example_error_handling.py

@@ -0,0 +1,266 @@
+"""
+Error Handling Example: Resilient plugins with error recovery.
+
+This example shows how plugins can:
+1. Handle errors gracefully
+2. Retry operations
+3. Report errors to manager
+4. Implement fallback strategies
+"""
+
+import logging
+import time
+import random
+from typing import Any, Dict
+
+from parallelazy.plugin import Plugin, PluginDescriptor
+from parallelazy.plugin_worker import PluginWorker
+from parallelazy.plugin_manager import PluginManager
+
+logger = logging.getLogger(__name__)
+
+
+class UnreliableService(Plugin):
+    """
+    Plugin that simulates an unreliable external service.
+    Randomly fails to simulate real-world conditions.
+    """
+
+    @classmethod
+    def Descriptor(cls, options: Dict[str, Any] = None) -> PluginDescriptor:
+        return super().Descriptor(
+            version="1.0.0",
+            author="ModularNet Team",
+            description="Unreliable service simulator",
+            options=options or {}
+        )
+
+    def __init__(self, worker: PluginWorker, plugin_id: str, options: Dict[str, Any]):
+        super().__init__(worker, plugin_id, options)
+        self.success_count = 0
+        self.failure_count = 0
+        self.failure_rate = 0.3  # 30% failure rate
+
+    def on_start(self) -> None:
+        logger.info(f"[{self.plugin_id}] UnreliableService started")
+        self.env_set("service.status", "ready")
+        self.env_set("service.success_count", 0)
+        self.env_set("service.failure_count", 0)
+
+    def main(self):
+        pass
+
+    def call_service(self, request_id: int) -> Dict[str, Any]:
+        """
+        Simulates a service call that may fail.
+        """
+        if random.random() < self.failure_rate:
+            self.failure_count += 1
+            logger.error(f"[{self.plugin_id}] Service call #{request_id} FAILED")
+            self.env_set("service.failure_count", self.failure_count, wait=False)
+            raise Exception(f"Service temporarily unavailable (request #{request_id})")
+        else:
+            self.success_count += 1
+            result = {"request_id": request_id, "data": f"response_{request_id}"}
+            logger.info(f"[{self.plugin_id}] Service call #{request_id} SUCCESS")
+            self.env_set("service.success_count", self.success_count, wait=False)
+            return result
+
+    def on_stop(self) -> None:
+        logger.info(
+            f"[{self.plugin_id}] UnreliableService stopped: "
+            f"success={self.success_count}, failures={self.failure_count}"
+        )
+        self.env_set("service.final_status", "stopped")
+
+
+class ResilientClient(Plugin):
+    """
+    Plugin that calls unreliable service with retry logic.
+    Demonstrates error handling and recovery strategies.
+    """
+
+    @classmethod
+    def Descriptor(cls, options: Dict[str, Any] = None) -> PluginDescriptor:
+        return super().Descriptor(
+            version="1.0.0",
+            author="ModularNet Team",
+            description="Resilient service client with retry logic",
+            options=options or {}
+        )
+
+    def __init__(self, worker: PluginWorker, plugin_id: str, options: Dict[str, Any]):
+        super().__init__(worker, plugin_id, options)
+        self.requests_sent = 0
+        self.requests_succeeded = 0
+        self.requests_failed = 0
+        self.service_id = "unreliable-service"
+
+    def on_start(self) -> None:
+        logger.info(f"[{self.plugin_id}] ResilientClient started")
+        self.env_set("client.requests_sent", 0)
+        self.env_set("client.requests_succeeded", 0)
+        self.env_set("client.requests_failed", 0)
+
+    def main(self):
+        self.requests_sent += 1
+        if self.requests_sent % 3 == 0:
+            self._call_with_retry(self.requests_sent)
+
+    def _call_with_retry(self, request_id: int, max_retries: int = 3):
+        """Call service with retry logic."""
+        retry_count = 0
+        last_error = None
+
+        while retry_count < max_retries:
+            try:
+                logger.info(
+                    f"[{self.plugin_id}] Calling service (request #{request_id}, "
+                    f"attempt {retry_count + 1}/{max_retries})"
+                )
+                result = self.plugin_call(self.service_id, "call_service", request_id)
+                logger.info(f"[{self.plugin_id}] Request #{request_id} succeeded: {result}")
+                self.requests_succeeded += 1
+                self.env_set("client.requests_succeeded", self.requests_succeeded, wait=False)
+                return result
+
+            except Exception as e:
+                last_error = e
+                retry_count += 1
+                logger.warning(
+                    f"[{self.plugin_id}] Request #{request_id} failed (attempt {retry_count}/{max_retries}): {e}"
+                )
+                if retry_count < max_retries:
+                    # Brief backoff before retry
+                    time.sleep(0.1 * retry_count)
+
+        # All retries exhausted
+        logger.error(f"[{self.plugin_id}] Request #{request_id} FAILED after {max_retries} attempts: {last_error}")
+        self.requests_failed += 1
+        self.env_set("client.requests_failed", self.requests_failed, wait=False)
+        return None
+
+    def on_stop(self) -> None:
+        logger.info(
+            f"[{self.plugin_id}] ResilientClient stopped: "
+            f"sent={self.requests_sent}, succeeded={self.requests_succeeded}, "
+            f"failed={self.requests_failed}"
+        )
+        self.env_set("client.requests_sent", self.requests_sent)
+        self.env_set("client.requests_succeeded", self.requests_succeeded)
+        self.env_set("client.requests_failed", self.requests_failed)
+
+
+class HealthMonitor(Plugin):
+    """
+    Plugin that monitors the health of other plugins.
+    """
+
+    @classmethod
+    def Descriptor(cls, options: Dict[str, Any] = None) -> PluginDescriptor:
+        return super().Descriptor(
+            version="1.0.0",
+            author="ModularNet Team",
+            description="Health monitor for system resilience",
+            options=options or {}
+        )
+
+    def __init__(self, worker: PluginWorker, plugin_id: str, options: Dict[str, Any]):
+        super().__init__(worker, plugin_id, options)
+        self.checks = 0
+
+    def on_start(self) -> None:
+        logger.info(f"[{self.plugin_id}] HealthMonitor started")
+        # Subscribe to service status
+        self.env_subscribe("service.status", self.on_service_status_change)
+        self.env_set("monitor.checks", 0)
+
+    def on_service_status_change(self, name: str, value: Any):
+        logger.info(f"[{self.plugin_id}] Service status changed: {value}")
+
+    def main(self):
+        self.checks += 1
+        if self.checks % 4 == 0:
+            # Check service metrics
+            success = self.env_get("service.success_count", 0)
+            failures = self.env_get("service.failure_count", 0)
+            total = success + failures
+            if total > 0:
+                failure_rate = failures / total
+                status = "UNHEALTHY" if failure_rate > 0.5 else "HEALTHY"
+                logger.info(
+                    f"[{self.plugin_id}] Health check: {status} (success_rate={1-failure_rate:.1%})"
+                )
+
+    def on_stop(self) -> None:
+        logger.info(f"[{self.plugin_id}] HealthMonitor stopped after {self.checks} checks")
+
+
+def main():
+    """Run the error handling example."""
+    manager = PluginManager()
+
+    # Create plugins
+    service_desc = UnreliableService.Descriptor()
+    service_desc.name = "unreliable-service"
+
+    client_desc = ResilientClient.Descriptor()
+    monitor_desc = HealthMonitor.Descriptor()
+
+    manager.load_plugin(service_desc)
+    manager.load_plugin(client_desc)
+    manager.load_plugin(monitor_desc)
+
+    manager.start_plugin(service_desc.name)
+    manager.start_plugin(client_desc.name)
+    manager.start_plugin(monitor_desc.name)
+
+    print("Error handling example running... Press Ctrl+C to stop.")
+    time.sleep(6)
+    print("Stopping plugins...")
+
+    manager.stop_plugin(monitor_desc.name)
+    manager.stop_plugin(client_desc.name)
+    manager.stop_plugin(service_desc.name)
+
+    time.sleep(1)
+
+    # Show final metrics
+    print("\nFinal metrics:")
+    all_vars = manager.env_get()
+    for name, value in sorted(all_vars.items()):
+        if "service" in name or "client" in name:
+            print(f"  {name}: {value}")
+
+    # VERIFICATION
+    print("\n=== VERIFICATION ===")
+    print("PREDICTED BEHAVIOR:")
+    print("1. UnreliableService fails ~30% of the time")
+    print("2. ResilientClient retries up to 3 times (exponential backoff)")
+    print("3. Client succeeds on some retries, fails on others")
+    print("4. Both success and failure metrics tracked\n")
+
+    service_success = all_vars.get("service.success_count", 0)
+    service_failures = all_vars.get("service.failure_count", 0)
+    client_success = all_vars.get("client.requests_succeeded", 0)
+    client_failures = all_vars.get("client.requests_failed", 0)
+
+    assert service_success > 0, "Service should have successes"
+    assert service_failures > 0, "Service should have failures (30% rate)"
+    assert client_success > 0, "Client should retry and succeed sometimes"
+    
+    service_total = service_success + service_failures
+    failure_rate = service_failures / service_total if service_total > 0 else 0
+    
+    print(f"✓ Service stats: success={service_success}, failures={service_failures}, failure_rate={failure_rate:.1%}")
+    print(f"✓ Client resilience: succeeded={client_success}, failed={client_failures}")
+    print(f"✓ Retry strategy working (some requests succeeded despite service failures)")
+    print("✓ All error handling assertions passed!")
+
+    manager.unload_plugin(service_desc.name)
+    manager.unload_plugin(client_desc.name)
+    manager.unload_plugin(monitor_desc.name)
+
+
+if __name__ == "__main__":
+    main()

parallelazy/examples/example_graceful_shutdown.py

@@ -0,0 +1,139 @@
+"""
+Graceful Shutdown Example: Verify clean plugin termination.
+
+This minimal example demonstrates:
+1. Plugin lifecycle (load, start, stop, unload)
+2. Graceful shutdown with on_stop() callback
+3. Process cleanup and resource release
+"""
+
+import logging
+import sys
+import time
+from typing import Any, Dict
+
+from parallelazy.plugin import Plugin, PluginDescriptor
+from parallelazy.plugin_worker import PluginWorker
+from parallelazy.plugin_manager import PluginManager
+
+# Configure logging for clear shutdown verification
+logging.basicConfig(
+    level=logging.DEBUG,
+    format='%(asctime)s - %(name)s - %(levelname)s - %(message)s',
+    handlers=[logging.StreamHandler(sys.stdout)]
+)
+logger = logging.getLogger(__name__)
+
+
+class SimpleWorker(Plugin):
+    """
+    Minimal plugin that logs lifecycle events.
+    Perfect for verifying graceful shutdown behavior.
+    """
+
+    @classmethod
+    def Descriptor(cls, options: Dict[str, Any] = None) -> PluginDescriptor:
+        return super().Descriptor(
+            version="1.0.0",
+            author="ModularNet Team",
+            description="Simple worker for testing graceful shutdown",
+            options=options or {}
+        )
+
+    def __init__(self, worker: PluginWorker, plugin_id: str, options: Dict[str, Any]):
+        super().__init__(worker, plugin_id, options)
+        self.iteration = 0
+
+    def on_start(self) -> None:
+        logger.info(f"[{self.plugin_id}] ✓ on_start() called - plugin initialized")
+        self.iteration = 0
+
+    def main(self):
+        """Main loop - runs repeatedly until on_stop() is called."""
+        self.iteration += 1
+        logger.debug(f"[{self.plugin_id}] Iteration {self.iteration}")
+        time.sleep(0.1)  # Brief sleep to simulate work
+
+    def on_stop(self) -> None:
+        """Graceful shutdown callback - called before plugin terminates."""
+        logger.info(f"[{self.plugin_id}] ✓ on_stop() called - cleaning up after {self.iteration} iterations")
+        logger.info(f"[{self.plugin_id}] ✓ Graceful shutdown complete")
+
+
+def main():
+    """Run the graceful shutdown verification."""
+    print("\n" + "=" * 70)
+    print("GRACEFUL SHUTDOWN VERIFICATION")
+    print("=" * 70 + "\n")
+
+    # Step 1: Create manager
+    print("[STEP 1] Creating PluginManager...")
+    manager = PluginManager(timeout_start=5.0)
+    print("✓ Manager created\n")
+
+    # Step 2: Load plugin
+    print("[STEP 2] Loading plugin...")
+    descriptor = SimpleWorker.Descriptor()
+    if not manager.load_plugin(descriptor):
+        logger.error("Failed to load plugin")
+        return False
+    print("✓ Plugin loaded\n")
+
+    # Step 3: Start plugin
+    print("[STEP 3] Starting plugin...")
+    if not manager.start_plugin(descriptor.name):
+        logger.error("Failed to start plugin")
+        return False
+    print("✓ Plugin started\n")
+
+    # Step 4: Let it run briefly
+    print("[STEP 4] Running for 2 seconds...")
+    time.sleep(2)
+    print("✓ Plugin has been running\n")
+
+    # Step 5: Stop plugin (graceful shutdown)
+    print("[STEP 5] Stopping plugin (graceful shutdown)...")
+    if not manager.stop_plugin(descriptor.name):
+        logger.error("Failed to stop plugin")
+        return False
+    print("✓ Plugin stopped gracefully\n")
+
+    # Step 6: Unload plugin
+    print("[STEP 6] Unloading plugin...")
+    if not manager.unload_plugin(descriptor.name):
+        logger.error("Failed to unload plugin")
+        return False
+    print("✓ Plugin unloaded\n")
+
+    # Step 7: Verify shutdown
+    print("[STEP 7] Verification...")
+    if descriptor.name in manager._plugins:
+        logger.error("Plugin still in manager after unload!")
+        return False
+    print("✓ Plugin successfully cleaned up from manager\n")
+
+    # Summary
+    print("=" * 70)
+    print("GRACEFUL SHUTDOWN VERIFICATION PASSED ✓")
+    print("=" * 70)
+    print("\nExpected behavior verified:")
+    print("  1. on_start() called when plugin started")
+    print("  2. Plugin ran for ~2 seconds (main loop)")
+    print("  3. on_stop() called when stop_plugin() invoked")
+    print("  4. Process terminated cleanly")
+    print("  5. Resources cleaned up after unload_plugin()")
+    print("\n")
+
+    return True
+
+
+if __name__ == "__main__":
+    try:
+        success = main()
+        sys.exit(0 if success else 1)
+    except KeyboardInterrupt:
+        print("\nInterrupted by user")
+        sys.exit(1)
+    except Exception as e:
+        logger.error(f"Unexpected error: {e}", exc_info=True)
+        sys.exit(1)