parallelazy 0.0.1__py3-none-any.whl

parallelazy/__init__.py

@@ -0,0 +1,43 @@
+"""
+ModularNet Plugin System: Lightweight multiprocessing-based plugin architecture.
+
+Provides modular plugin execution with pub/sub messaging, environment variables,
+and service invocation.
+
+Main Components:
+- PluginDescriptor: Serializable plugin metadata
+- Plugin: Abstract base class for plugins
+- PluginManager: Plugin orchestration and management
+- PluginWorker: Plugin subprocess worker
+- PluginMessages: Unified IPC message protocol (Request, Response)
+- DataExchange: Unified subscription model (topics + env vars)
+- ManagerProxy: Client-side proxy for direct manager access from plugins
+"""
+
+from parallelazy.plugin import Plugin, PluginState, PluginDescriptor
+from parallelazy.environment_manager import EnvironmentManager
+from parallelazy.plugin_manager import PluginManager
+
+from parallelazy.plugin_data import (
+    Request, Response,
+    Subscription, 
+    PluginStatus, ProcessStatus
+)
+from parallelazy.plugin_rpc import PluginRPC
+
+__all__ = [
+    "PluginDescriptor",
+    "Plugin",
+    "PluginState",
+    "EnvironmentManager",
+    "EnvUpdate",
+    "PluginManager",
+    "Request",
+    "Response",
+    "Channel",
+    "Subscription",
+    "SubscriptionType",
+    "PluginStatus",
+    "ProcessStatus",
+    "PluginRPC",
+]

parallelazy/environment_manager.py

@@ -0,0 +1,55 @@
+"""
+Environment State System: Manages shared environment variables across plugins.
+
+Simple thread-safe system for reading/writing global environment variables
+and broadcasting updates to interested plugins.
+"""
+
+from typing import Any, Dict
+from threading import Lock
+import logging
+
+logger = logging.getLogger(__name__)
+
+
+
+class EnvironmentManager:
+    """Thread-safe manager for shared environment variables."""
+    
+    def __init__(self):
+        """Initialize environment manager."""
+        self._env: Dict[str, Any] = {}
+        self._lock = Lock()
+    
+    def get(self, key: str, default: Any = None) -> Any:
+        """Get environment variable value."""
+        with self._lock:
+            return self._env.get(key, default)
+    
+    def set(self, key: str, value: Any) -> None:
+        """Set environment variable."""
+        with self._lock:
+            self._env[key] = value
+    
+    def get_all(self) -> Dict[str, Any]:
+        """Get all environment variables."""
+        with self._lock:
+            return self._env.copy()
+    
+    def set_all(self, env_dict: Dict[str, Any]) -> None:
+        """Set multiple environment variables."""
+        for key, value in env_dict.items():
+            self.set(key, value)
+    
+    def clear(self, key: str=None) -> None:
+        """Clear a specific environment variable."""
+        with self._lock:
+            if key in self._env:
+                del self._env[key]
+            else:
+                self._env.clear()
+    
+    def __contains__(self, key: str) -> bool:
+        """Check if key exists in environment."""
+        with self._lock:
+            return key in self._env

parallelazy/examples/example_configuration.py

@@ -0,0 +1,194 @@
+"""
+Configuration & Options Example: Plugin customization at load time.
+
+This example shows how plugins can:
+1. Accept configuration options when loaded
+2. Use options to customize behavior
+3. Override configuration dynamically
+4. Validate configuration options
+"""
+
+import logging
+import time
+from typing import Any, Dict
+
+from parallelazy.plugin import Plugin, PluginDescriptor
+from parallelazy.plugin_worker import PluginWorker
+from parallelazy.plugin_manager import PluginManager
+
+logger = logging.getLogger(__name__)
+
+
+class Sensor(Plugin):
+    """
+    Plugin that simulates a sensor with configurable sampling.
+    """
+
+    @classmethod
+    def Descriptor(cls, options: Dict[str, Any] = None) -> PluginDescriptor:
+        default_options = {
+            "sample_rate": 10,  # Hz
+            "noise_level": 0.1,
+            "sensor_type": "temperature",
+        }
+        if options:
+            default_options.update(options)
+
+        return super().Descriptor(
+            version="1.0.0",
+            author="ModularNet Team",
+            description="Configurable sensor plugin",
+            options=default_options
+        )
+
+    def __init__(self, worker: PluginWorker, plugin_id: str, options: Dict[str, Any]):
+        super().__init__(worker, plugin_id, options)
+        self.sample_count = 0
+        self.sample_rate = options.get("sample_rate", 10)
+        self.noise_level = options.get("noise_level", 0.1)
+        self.sensor_type = options.get("sensor_type", "temperature")
+
+    def on_start(self) -> None:
+        logger.info(f"[{self.plugin_id}] Sensor started with options:")
+        logger.info(f"  - type: {self.sensor_type}")
+        logger.info(f"  - sample_rate: {self.sample_rate} Hz")
+        logger.info(f"  - noise_level: {self.noise_level}")
+        self.env_set("sensor.config", {
+            "type": self.sensor_type,
+            "sample_rate": self.sample_rate,
+            "noise_level": self.noise_level
+        })
+        self.env_set("sensor.samples", 0)
+
+    def main(self):
+        self.sample_count += 1
+        # Generate simulated sensor reading
+        base_value = 20 + (self.sample_count % 10)  # Temperature 20-30°C
+        import random
+        noise = random.gauss(0, self.noise_level)
+        reading = base_value + noise
+
+        if self.sample_count % 5 == 0:
+            logger.info(f"[{self.plugin_id}] Sample #{self.sample_count}: {reading:.2f}")
+            self.topic_publish("sensor.readings", {
+                "sample": self.sample_count,
+                "value": reading,
+                "type": self.sensor_type
+            })
+            self.env_set("sensor.samples", self.sample_count, wait=False)
+
+    def on_stop(self) -> None:
+        logger.info(f"[{self.plugin_id}] Sensor stopped, collected {self.sample_count} samples")
+
+
+class SensorLogger(Plugin):
+    """
+    Plugin that logs sensor readings with configurable verbosity.
+    """
+
+    @classmethod
+    def Descriptor(cls, options: Dict[str, Any] = None) -> PluginDescriptor:
+        default_options = {
+            "verbose": False,
+            "log_every_n": 5,
+        }
+        if options:
+            default_options.update(options)
+
+        return super().Descriptor(
+            version="1.0.0",
+            author="ModularNet Team",
+            description="Sensor logger with configurable output",
+            options=default_options
+        )
+
+    def __init__(self, worker: PluginWorker, plugin_id: str, options: Dict[str, Any]):
+        super().__init__(worker, plugin_id, options)
+        self.verbose = options.get("verbose", False)
+        self.log_every_n = options.get("log_every_n", 5)
+        self.readings = []
+
+    def on_start(self) -> None:
+        logger.info(f"[{self.plugin_id}] SensorLogger started (verbose={self.verbose})")
+        self.topic_subscribe("sensor.readings", self.on_reading)
+
+    def on_reading(self, reading: Dict[str, Any]):
+        self.readings.append(reading)
+        if self.verbose:
+            logger.info(f"[{self.plugin_id}] Received: {reading}")
+        elif len(self.readings) % self.log_every_n == 0:
+            avg_value = sum(r["value"] for r in self.readings) / len(self.readings)
+            logger.info(f"[{self.plugin_id}] {len(self.readings)} readings, avg={avg_value:.2f}")
+            self.env_set("logger.avg_value", avg_value, wait=False)
+
+    def main(self):
+        pass
+
+    def on_stop(self) -> None:
+        logger.info(f"[{self.plugin_id}] SensorLogger stopped, logged {len(self.readings)} readings")
+
+
+def main():
+    """Run the configuration example."""
+    manager = PluginManager()
+
+    # Load sensor with custom options
+    sensor_desc = Sensor.Descriptor(options={
+        "sample_rate": 20,
+        "noise_level": 0.5,
+        "sensor_type": "humidity"
+    })
+
+    # Load logger with verbose mode
+    logger_desc = SensorLogger.Descriptor(options={
+        "verbose": False,
+        "log_every_n": 3
+    })
+
+    manager.load_plugin(sensor_desc)
+    manager.load_plugin(logger_desc)
+
+    manager.start_plugin(sensor_desc.name)
+    manager.start_plugin(logger_desc.name)
+
+    print("Sensor and logger running... Press Ctrl+C to stop.")
+    time.sleep(5)
+    print("Stopping plugins...")
+
+    manager.stop_plugin(logger_desc.name)
+    manager.stop_plugin(sensor_desc.name)
+
+    # Show configuration
+    print("\nFinal state:")
+    all_vars = manager.env_get()
+    for name, value in all_vars.items():
+        if name.startswith("sensor.") or name.startswith("logger."):
+            print(f"  {name}: {value}")
+
+    # VERIFICATION
+    print("\n=== VERIFICATION ===")
+    print("PREDICTED BEHAVIOR:")
+    print("1. Sensor loads with custom options (sample_rate=20, noise_level=0.5, type=humidity)")
+    print("2. Logger loads with verbose=False, log_every_n=3")
+    print("3. Sensor publishes readings, logger subscribes and aggregates")
+    print("4. Config persists in environment\n")
+
+    sensor_config = all_vars.get("sensor.config")
+    logger_avg = all_vars.get("logger.avg_value")
+
+    assert sensor_config is not None, "sensor.config should be set"
+    assert sensor_config.get("type") == "humidity", f"Sensor type should be humidity, got {sensor_config.get('type')}"
+    assert sensor_config.get("sample_rate") == 20, f"Sample rate should be 20, got {sensor_config.get('sample_rate')}"
+    assert sensor_config.get("noise_level") == 0.5, f"Noise level should be 0.5, got {sensor_config.get('noise_level')}"
+    
+    print(f"✓ Sensor config: type={sensor_config['type']}, rate={sensor_config['sample_rate']}Hz, noise={sensor_config['noise_level']}")
+    if logger_avg is not None:
+        print(f"✓ Logger computed average: {logger_avg:.2f}")
+    print("✓ All configuration assertions passed!")
+
+    manager.unload_plugin(sensor_desc.name)
+    manager.unload_plugin(logger_desc.name)
+
+
+if __name__ == "__main__":
+    main()

parallelazy/examples/example_count.py

@@ -0,0 +1,84 @@
+"""
+Basic test plugin for lifecycle testing.
+
+A minimal plugin that exercises the load/unload/start/stop/status lifecycle
+without any external dependencies.
+"""
+
+import logging
+from typing import Any, Dict
+
+from parallelazy.plugin import Plugin, PluginDescriptor
+from parallelazy.plugin_worker import PluginWorker
+import time
+from parallelazy.plugin_manager import PluginManager
+
+
+
+logger = logging.getLogger(__name__)
+
+class ExamplePluginCount(Plugin):
+    """
+    Minimal test plugin that exercises core lifecycle.
+    
+    Features:
+    - Simple on_start/on_stop implementation
+    - Tracks call count
+    - Can be used to verify manager lifecycle
+    """
+
+    @classmethod
+    def Descriptor(cls, options: Dict[str, Any] = None) -> PluginDescriptor:
+        """Create plugin descriptor."""
+        return super().Descriptor(
+            version="1.0.0",
+            author="ModularNet Team",
+            description="Basic test plugin for lifecycle verification",
+            options=options or {}
+        )
+
+    def __init__(
+        self,
+        worker: PluginWorker,
+        plugin_id: str,
+        options: Dict[str, Any],
+    ):
+        super().__init__(worker, plugin_id, options)
+        self.count = None
+
+    def main(self):
+        self.count +=1
+        self.env_set("count", self.count, wait=False)
+
+
+    def on_start(self) -> None:
+        logger.info(f"[{self.plugin_id}] ExamplePluginCount starting...")
+        self.count = 0
+        self.env_set("count", self.count)
+
+    def on_stop(self) -> None:
+        logger.info(f"[{self.plugin_id}] Total loop: {self.count}")
+        self.env_set("count", self.count)
+        
+
+def main():
+    """Example main function to run the plugin standalone for testing."""
+    manager = PluginManager()
+    descriptor = ExamplePluginCount.Descriptor()
+    manager.load_plugin(descriptor)
+    manager.start_plugin(descriptor.name)
+
+    # Let it run for a few seconds
+    time.sleep(3)
+
+    
+    manager.unload_plugin(descriptor.name)
+
+    all_vars = manager.env_get()
+
+    print("All environment variables:")
+    for name, value in all_vars.items():
+        print(f"  {name}: {value}")
+
+if __name__ == "__main__":    main()
+

parallelazy/examples/example_data_pipeline.py

@@ -0,0 +1,226 @@
+"""
+Data Pipeline Example: Multi-stage processing with pub/sub.
+
+This example shows how plugins can:
+1. Form a data pipeline using pub/sub topics
+2. Process data in stages
+3. Coordinate using environment variables
+4. Track progress across the pipeline
+"""
+
+import logging
+import time
+from typing import Any, Dict
+
+from parallelazy.plugin import Plugin, PluginDescriptor
+from parallelazy.plugin_worker import PluginWorker
+from parallelazy.plugin_manager import PluginManager
+
+logger = logging.getLogger(__name__)
+
+
+class DataSource(Plugin):
+    """
+    Stage 1: Generate data and publish to topic.from parallelazy.plugin import Plugin, PluginDescriptor
+from parallelazy.plugin_worker import PluginWorker
+from parallelazy.plugin_manager import PluginManager
+
+    """
+
+    @classmethod
+    def Descriptor(cls, options: Dict[str, Any] = None) -> PluginDescriptor:
+        return super().Descriptor(
+            version="1.0.0",
+            author="ModularNet Team",
+            description="Data source for pipeline",
+            options=options or {}
+        )
+
+    def __init__(self, worker: PluginWorker, plugin_id: str, options: Dict[str, Any]):
+        super().__init__(worker, plugin_id, options)
+        self.count = 0
+
+    def on_start(self) -> None:
+        logger.info(f"[{self.plugin_id}] DataSource started")
+        self.env_set("pipeline.stage1.count", 0)
+
+    def main(self):
+        self.count += 1
+        # Generate raw data
+        raw_data = {
+            "id": self.count,
+            "raw_value": self.count * 10
+        }
+        # Publish to pipeline topic
+        self.topic_publish("pipeline.raw", raw_data)
+        self.env_set("pipeline.stage1.count", self.count, wait=False)
+
+    def on_stop(self) -> None:
+        logger.info(f"[{self.plugin_id}] DataSource stopped, produced {self.count} items")
+
+
+class DataProcessor(Plugin):
+    """
+    Stage 2: Process data from raw topic and publish to processed topic.
+    """
+
+    @classmethod
+    def Descriptor(cls, options: Dict[str, Any] = None) -> PluginDescriptor:
+        return super().Descriptor(
+            version="1.0.0",
+            author="ModularNet Team",
+            description="Data processor for pipeline",
+            options=options or {}
+        )
+
+    def __init__(self, worker: PluginWorker, plugin_id: str, options: Dict[str, Any]):
+        super().__init__(worker, plugin_id, options)
+        self.count = 0
+
+    def on_start(self) -> None:
+        logger.info(f"[{self.plugin_id}] DataProcessor started")
+        self.topic_subscribe("pipeline.raw", self.on_raw_data)
+        self.env_set("pipeline.stage2.count", 0)
+
+    def on_raw_data(self, data: Dict[str, Any]):
+        self.count += 1
+        # Process the raw data
+        processed = {
+            "id": data["id"],
+            "raw_value": data["raw_value"],
+            "processed_value": data["raw_value"] * 2 + 5
+        }
+        logger.info(f"[{self.plugin_id}] Processed item #{data['id']}: {processed['processed_value']}")
+        # Publish processed data
+        self.topic_publish("pipeline.processed", processed)
+        self.env_set("pipeline.stage2.count", self.count, wait=False)
+
+    def main(self):
+        pass
+
+    def on_stop(self) -> None:
+        logger.info(f"[{self.plugin_id}] DataProcessor stopped, processed {self.count} items")
+
+
+class DataAggregator(Plugin):
+    """
+    Stage 3: Aggregate processed data and store statistics.
+    """
+
+    @classmethod
+    def Descriptor(cls, options: Dict[str, Any] = None) -> PluginDescriptor:
+        return super().Descriptor(
+            version="1.0.0",
+            author="ModularNet Team",
+            description="Data aggregator for pipeline",
+            options=options or {}
+        )
+
+    def __init__(self, worker: PluginWorker, plugin_id: str, options: Dict[str, Any]):
+        super().__init__(worker, plugin_id, options)
+        self.count = 0
+        self.sum_value = 0
+        self.max_value = 0
+
+    def on_start(self) -> None:
+        logger.info(f"[{self.plugin_id}] DataAggregator started")
+        self.topic_subscribe("pipeline.processed", self.on_processed_data)
+        self.env_set("pipeline.stage3.count", 0)
+        self.env_set("pipeline.stats.sum", 0)
+        self.env_set("pipeline.stats.max", 0)
+
+    def on_processed_data(self, data: Dict[str, Any]):
+        self.count += 1
+        value = data["processed_value"]
+        self.sum_value += value
+        self.max_value = max(self.max_value, value)
+
+        if self.count % 5 == 0:
+            logger.info(
+                f"[{self.plugin_id}] Aggregated {self.count} items | "
+                f"Sum: {self.sum_value} | Max: {self.max_value}"
+            )
+            self.env_set("pipeline.stage3.count", self.count, wait=False)
+            self.env_set("pipeline.stats.sum", self.sum_value, wait=False)
+            self.env_set("pipeline.stats.max", self.max_value, wait=False)
+
+    def main(self):
+        pass
+
+    def on_stop(self) -> None:
+        logger.info(f"[{self.plugin_id}] DataAggregator stopped")
+        logger.info(f"[{self.plugin_id}] Final stats: count={self.count}, sum={self.sum_value}, max={self.max_value}")
+        self.env_set("pipeline.stats.final_count", self.count)
+        self.env_set("pipeline.stats.final_sum", self.sum_value)
+        self.env_set("pipeline.stats.final_max", self.max_value)
+
+
+def main():
+    """Run the data pipeline example."""
+    manager = PluginManager()
+
+    # Create and load pipeline plugins in order
+    source_desc = DataSource.Descriptor()
+    processor_desc = DataProcessor.Descriptor()
+    aggregator_desc = DataAggregator.Descriptor()
+
+    manager.load_plugin(source_desc)
+    manager.load_plugin(processor_desc)
+    manager.load_plugin(aggregator_desc)
+
+    manager.start_plugin(source_desc.name)
+    manager.start_plugin(processor_desc.name)
+    manager.start_plugin(aggregator_desc.name)
+
+    # Let the pipeline run
+    print("Data pipeline running... Press Ctrl+C to stop.")
+    time.sleep(3)
+    print("Stopping pipeline...")
+
+    manager.stop_plugin(source_desc.name)
+    manager.stop_plugin(processor_desc.name)
+    manager.stop_plugin(aggregator_desc.name)
+
+    time.sleep(1)
+
+    # Show final state
+    print("\nFinal pipeline statistics:")
+    all_vars = manager.env_get()
+    for name, value in all_vars.items():
+        if name.startswith("pipeline."):
+            print(f"  {name}: {value}")
+
+    # VERIFICATION
+    print("\n=== VERIFICATION ===")
+    print("PREDICTED BEHAVIOR:")
+    print("1. DataSource generates ~3 items in 3 seconds")
+    print("2. DataProcessor receives all items and doubles+offsets values")
+    print("3. DataAggregator aggregates: sum and max of processed values")
+    print("4. All 3 stages see the same count\n")
+
+    count1 = all_vars.get("pipeline.stage1.count", 0)
+    count2 = all_vars.get("pipeline.stage2.count", 0)
+    count3 = all_vars.get("pipeline.stage3.count", 0)
+    stats_sum = all_vars.get("pipeline.stats.sum", 0)
+    stats_max = all_vars.get("pipeline.stats.max", 0)
+
+    assert count1 > 0, "Stage 1 should generate items"
+    assert count2 > 0, "Stage 2 should process items"
+    assert count3 > 0, "Stage 3 should aggregate items"
+    assert count1 == count2, f"Stage counts should match: stage1={count1}, stage2={count2}"
+    assert count1 == count3, f"Stage counts should match: stage1={count1}, stage3={count3}"
+    assert stats_sum > 0, "Sum should be positive"
+    assert stats_max > 0, "Max should be positive"
+
+    print(f"✓ Items generated: {count1}")
+    print(f"✓ Items processed through all 3 stages: {count1} == {count2} == {count3}")
+    print(f"✓ Aggregator computed: sum={stats_sum}, max={stats_max}")
+    print("✓ All pipeline assertions passed!")
+
+    manager.unload_plugin(source_desc.name)
+    manager.unload_plugin(processor_desc.name)
+    manager.unload_plugin(aggregator_desc.name)
+
+
+if __name__ == "__main__":
+    main()