fred-oss 0.11.0__tar.gz → 0.13.0__tar.gz

{fred_oss-0.11.0/src/main/fred_oss.egg-info → fred_oss-0.13.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: fred-oss
-Version: 0.11.0
+Version: 0.13.0
 Summary: FREDOSS
 Home-page: https://fred.fahera.mx
 Author: Fahera Research, Education, and Development

fred_oss-0.13.0/src/main/fred/version

@@ -0,0 +1 @@
+0.13.0

{fred_oss-0.11.0 → fred_oss-0.13.0}/src/main/fred/worker/runner/client.py

@@ -9,6 +9,7 @@ from fred.settings import (
 from fred.worker.runner.utils import (
     get_request_queue_name_from_payload,
     get_response_queue_name_from_payload,
+    get_redis_configs_from_payload,
 )
 
 from redis import Redis
@@ -24,14 +25,7 @@ class RunnerClient:
 
     @classmethod
     def auto(cls, **kwargs) -> "RunnerClient":
-        redis_configs = {
-            "host": kwargs.get("host") or get_environ_variable(name="REDIS_HOST", default="localhost"),
-            "port": int(kwargs.get("port") or get_environ_variable(name="REDIS_PORT", default=6379)),
-            "password": get_environ_variable(name="REDIS_PASSWORD", default=None),
-            "db": int(kwargs.get("db") or get_environ_variable(name="REDIS_DB", default=0)),
-            "decode_responses": True,
-            **kwargs.pop("redis_configs", {}),
-        }
+        redis_configs = get_redis_configs_from_payload(payload=kwargs, keep=False)
         redis_instance = Redis(**redis_configs)
         req_queue = get_request_queue_name_from_payload(payload=kwargs, keep=False) 
         res_queue = get_response_queue_name_from_payload(payload=kwargs, keep=False) or (

fred_oss-0.13.0/src/main/fred/worker/runner/info.py

@@ -0,0 +1,54 @@
+import uuid
+from typing import Optional
+from dataclasses import dataclass
+
+from fred.settings import logger_manager
+
+logger = logger_manager.get_logger(name=__name__)
+
+
+@dataclass(frozen=True, slots=True)
+class RunnerInfo:
+    runner_id: str
+    created_at: str
+    runner_inner_handler_classname: str
+    runner_inner_handler_classpath: str
+    lifetime: int
+    timeout: int
+
+    @classmethod
+    def create(
+        cls,
+        runner_inner_handler_classname: str,
+        runner_inner_handler_classpath: str,
+        runner_id: Optional[str] = None,
+        created_at: Optional[str] = None,
+        lifetime: int = 3600,  # Default to 1 hour
+        timeout: int = 10,  # Default to 10 seconds
+    ) -> "RunnerInfo":
+        from fred.utils.dateops import datetime_utcnow
+        return cls(
+            runner_id=runner_id or str(uuid.uuid4()),
+            runner_inner_handler_classname=runner_inner_handler_classname,
+            runner_inner_handler_classpath=runner_inner_handler_classpath,
+            created_at=created_at or datetime_utcnow().isoformat(),
+            lifetime=lifetime,
+            timeout=timeout,
+        )
+
+    def get_start_event(self, **kwargs) -> dict:
+        runner_id = kwargs.pop("runner_id", self.runner_id) 
+        payload = {
+            "lifetime": self.lifetime,
+            "timeout": self.timeout,
+            "runner_configs": {
+                "id": runner_id,
+                "import_pattern": self.runner_inner_handler_classpath,
+                "handler_classname": self.runner_inner_handler_classname,
+            },
+            **kwargs,
+        }
+        return {
+            "input": payload,
+            "id": runner_id,
+        }

fred_oss-0.13.0/src/main/fred/worker/runner/plugins/_local.py

@@ -0,0 +1,26 @@
+from dataclasses import dataclass
+
+from fred.settings import logger_manager
+from fred.worker.runner.info import RunnerInfo
+from fred.worker.runner.handler import RunnerHandler
+from fred.worker.runner.plugins.interface import PluginInterface
+
+logger = logger_manager.get_logger(name=__name__)
+
+
+@dataclass(frozen=True, slots=True)
+class LocalPlugin(PluginInterface):
+
+    def _execute(
+            self,
+            runner_info: RunnerInfo,
+            outer_handler: RunnerHandler,
+            **kwargs
+        ):
+        """Execute the runner locally using the provided outer handler.
+        Args:
+            runner_info (RunnerInfo): Information about the runner to execute.
+            outer_handler (RunnerHandler): The outer handler to use for execution.
+            **kwargs: Additional keyword arguments to pass to the execution method implemented by the subclass.
+        """
+        outer_handler.run(event=runner_info.get_start_event(**kwargs))

fred_oss-0.13.0/src/main/fred/worker/runner/plugins/catalog.py

@@ -0,0 +1,15 @@
+import enum
+from dataclasses import dataclass
+
+from fred.settings import logger_manager
+from fred.worker.runner.plugins._local import LocalPlugin
+
+logger = logger_manager.get_logger(name=__name__)
+
+
+class PluginCatalog(enum.Enum):
+    """Enum for the different plugins available in FRED."""
+
+    LOCAL = LocalPlugin.auto()
+    RUNPOD = None  # Placeholder for future RunPod plugin
+    LAMBDA = None  # Placeholder for future AWS Lambda plugin

fred_oss-0.13.0/src/main/fred/worker/runner/plugins/interface.py

@@ -0,0 +1,152 @@
+from dataclasses import dataclass
+from threading import Thread
+from typing import Optional
+
+from redis import Redis
+
+from fred.settings import logger_manager
+from fred.worker.runner.info import RunnerInfo
+from fred.worker.runner.handler import RunnerHandler
+from fred.worker.runner.utils import get_redis_configs_from_payload
+from fred.utils.dateops import datetime_utcnow
+
+logger = logger_manager.get_logger(name=__name__)
+
+
+@dataclass(frozen=True, slots=True)
+class PluginInterface:
+    """Interface for the different plugins available for FRED Runners."""
+    redis: Redis
+
+    @classmethod
+    def auto(cls, **kwargs) -> "PluginInterface":
+        """Auto-instantiate the plugin interface with a Redis instance.
+        If a Redis instance is not provided, it will create one using configurations
+        extracted from the provided keyword arguments.
+        Args:
+            **kwargs: Keyword arguments that may contain Redis configurations or an existing Redis instance.
+        Returns:
+            PluginInterface: An instance of the PluginInterface with a Redis connection.
+        """
+        redis = kwargs.pop("instance", None) or Redis(**get_redis_configs_from_payload(payload=kwargs, keep=False))
+        return cls(redis=redis)
+
+    def _execute(
+            self,
+            runner_info: RunnerInfo,
+            outer_handler: RunnerHandler,
+            **kwargs
+        ):
+        raise NotImplementedError("This method should be overridden by subclasses.")
+
+    def _monitor(self, runner_info: RunnerInfo, **kwargs):
+        raise NotImplementedError("This method should be overridden by subclasses.")
+
+    def monitor(
+            self,
+            runner_info: RunnerInfo,
+            blocking: bool = False,
+            timeout: Optional[int] = None,
+            **kwargs
+        ) -> Optional[Thread]:
+        """Start monitoring the runner in a separate thread.
+        Args:
+            runner_info (RunnerInfo): Information about the runner to monitor.
+            blocking (bool, optional): Whether to block the main thread until monitoring is complete. Defaults to False.
+            timeout (Optional[int], optional): Timeout in seconds for the monitoring thread. Defaults to None.
+            **kwargs: Additional keyword arguments to pass to the monitoring method.
+        Returns:
+            Optional[Thread]: The thread object for the monitoring thread if not blocking, else None."""
+        # Start monitoring in a separate thread
+        thread = Thread(
+            target=self._monitor,
+            kwargs={
+                "runner_info": runner_info,
+                "blocking": blocking,
+                **kwargs
+            },
+            daemon=True,
+        )
+        thread.start()
+        # If blocking is True, wait for the thread to finish or timeout
+        return thread.join(timeout=timeout) if blocking else thread
+
+    def _execute_wrapper(
+            self,
+            runner_info: RunnerInfo,
+            outer_handler: RunnerHandler,
+            **kwargs
+        ) -> str:
+        """Wrapper method to handle execution and include error logging.
+
+        Since we don't control the implementation of the _execute method in subclasses, we
+        need to wrap it to add error handling and logging.
+
+        Args:
+            runner_info (RunnerInfo): Information about the runner to execute.
+            outer_handler (RunnerHandler): The outer handler to use for execution.
+            **kwargs: Additional keyword arguments to pass to the execution method implemented by the subclass.
+        """
+        runner_id = runner_info.runner_id
+        self.redis.set(f"runner:{runner_id}:status", f"STARTING:{datetime_utcnow().isoformat()}")
+        try:
+            self._execute(
+                    runner_info=runner_info,
+                    outer_handler=outer_handler,
+                    **kwargs
+                )
+        except Exception as e:
+            self.redis.set(f"runner:{runner_id}:status", f"ERROR:{datetime_utcnow().isoformat()}")
+            logger.error(f"Error executing runner '{runner_id}': {e}")
+            raise
+        return runner_id
+
+    def execute(
+            self,
+            runner_info: RunnerInfo,
+            outer_handler: RunnerHandler,
+            wait_for_exec: bool = False,
+            timeout: Optional[int] = None,
+            enable_monitor: bool = False,
+            wait_for_monitor: bool = False,
+            **kwargs
+        ) -> Thread:
+        """Execute the runner using the specified plugin.
+        This method starts the execution of the runner in a separate thread. It can also
+        optionally start a monitoring thread to monitor the runner's status.
+        Args:
+            runner_info (RunnerInfo): Information about the runner to execute.
+            outer_handler (RunnerHandler): The outer handler to use for execution.
+            wait_for_exec (bool, optional): Whether to wait for the execution thread to complete. Defaults to False.
+            timeout (Optional[int], optional): Timeout in seconds for the execution thread. Defaults to None.
+            enable_monitor (bool, optional): Whether to enable monitoring of the runner. Defaults to False.
+            wait_for_monitor (bool, optional): Whether to wait for the monitoring thread to complete. Defaults to False.
+            **kwargs: Additional keyword arguments to pass to the execution and monitoring methods.
+        Returns:
+            Thread: The thread object for the execution.
+        """
+        runner_id = runner_info.runner_id
+        logger.info(f"Starting thread runner '{runner_id}' using plugin '{self.__class__.__name__}'.")
+        thread = Thread(
+            target=self._execute_wrapper,
+            kwargs={
+                "runner_info": runner_info,
+                "outer_handler": outer_handler,
+                **kwargs
+            },
+            daemon=True,
+        )
+        thread.start()
+        if enable_monitor:
+            self.monitor(
+                runner_info=runner_info,
+                blocking=wait_for_monitor,
+                timeout=timeout,
+                **kwargs
+            )
+        else:
+            logger.info(f"Monitoring disabled for runner '{runner_id}'.")
+        if wait_for_exec:
+            logger.info(f"Waiting for execution of runner '{runner_id}' to complete.")
+            thread.join()
+        return thread

{fred_oss-0.11.0 → fred_oss-0.13.0}/src/main/fred/worker/runner/utils.py

@@ -47,3 +47,28 @@ def get_response_queue_name_from_payload(
         default=None,
         keep=keep,
     )
+
+def get_redis_configs_from_payload(
+        payload: dict,
+        keep: bool = False,
+    ) -> dict:
+    for host_key in ["host", "redis_host"]:
+        if (host := payload.get(host_key) if keep else payload.pop(host_key, None)):
+            break
+    for port_key in ["port", "redis_port"]:
+        if (port := payload.get(port_key) if keep else payload.pop(port_key, None)):
+            break
+    for password_key in ["password", "redis_password"]:
+        if (password := payload.get(password_key) if keep else payload.pop(password_key, None)):
+            break
+    for db_key in ["db", "redis_db"]:
+        if (db := payload.get(db_key) if keep else payload.pop(db_key, None)):
+            break
+    return {
+        "host": host or get_environ_variable(name="REDIS_HOST", default="localhost"),
+        "port": int(port or get_environ_variable(name="REDIS_PORT", default=6379)),
+        "password": password or get_environ_variable(name="REDIS_PASSWORD", default=None),
+        "db": int(db or get_environ_variable(name="REDIS_DB", default=0)),
+        "decode_responses": True,
+        **(payload.get("redis_configs", {}) if keep else payload.pop("redis_configs", {})),
+    }

{fred_oss-0.11.0 → fred_oss-0.13.0/src/main/fred_oss.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: fred-oss
-Version: 0.11.0
+Version: 0.13.0
 Summary: FREDOSS
 Home-page: https://fred.fahera.mx
 Author: Fahera Research, Education, and Development

{fred_oss-0.11.0 → fred_oss-0.13.0}/src/main/fred_oss.egg-info/SOURCES.txt

@@ -30,7 +30,12 @@ src/main/fred/worker/interface.py
 src/main/fred/worker/runner/__init__.py
 src/main/fred/worker/runner/client.py
 src/main/fred/worker/runner/handler.py
+src/main/fred/worker/runner/info.py
 src/main/fred/worker/runner/utils.py
+src/main/fred/worker/runner/plugins/__init__.py
+src/main/fred/worker/runner/plugins/_local.py
+src/main/fred/worker/runner/plugins/catalog.py
+src/main/fred/worker/runner/plugins/interface.py
 src/main/fred_oss.egg-info/PKG-INFO
 src/main/fred_oss.egg-info/SOURCES.txt
 src/main/fred_oss.egg-info/dependency_links.txt

fred_oss-0.11.0/src/main/fred/version

@@ -1 +0,0 @@
-0.11.0