fred-oss 0.53.0__tar.gz → 0.54.0__tar.gz

{fred_oss-0.53.0/src/main/fred_oss.egg-info → fred_oss-0.54.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: fred-oss
-Version: 0.53.0
+Version: 0.54.0
 Summary: FREDOSS
 Home-page: https://fred.fahera.mx
 Author: Fahera Research, Education, and Development

{fred_oss-0.53.0 → fred_oss-0.54.0}/src/main/fred/dao/comp/_keyval.py

@@ -1,5 +1,5 @@
 from dataclasses import dataclass
-from typing import Optional
+from typing import Iterator, Optional
 
 from fred.settings import logger_manager, get_environ_variable
 from fred.dao.service.catalog import ServiceCatalog
@@ -36,6 +36,51 @@ class FredKeyVal(ComponentInterface):
     """
     key: str
     
+    @classmethod
+    def keys(cls, pattern: Optional[str] = None, **kwargs) -> Iterator[str]:
+        """Returns a list of keys matching the given pattern.
+        The implementation of this method depends on the underlying service.
+        For example, if the service is Redis, it uses the KEYS command to get the
+        list of keys matching the pattern.
+        Args:
+            pattern (str): The pattern to match keys against. Defaults to "*".
+        Returns:
+            list[str]: A list of keys matching the pattern.
+        Raises:
+            NotImplementedError: If the method is not implemented for the current service.
+        """
+        match cls._cat:
+            case ServiceCatalog.REDIS:
+                return (
+                    key if isinstance(key, str) else key.decode("utf-8")
+                    for key in cls._srv.client.scan_iter(match=pattern, **kwargs)
+                )
+            case ServiceCatalog.STDLIB:
+                import fnmatch
+                pattern = pattern or "*"
+                return (
+                    key
+                    for key in cls._srv.client._memstore_keyval.keys()
+                    if fnmatch.fnmatch(key, pattern=pattern)
+                )
+            case ServiceCatalog.MINIO:
+                import fnmatch
+                pattern = pattern or "*"
+                bucket_name = (
+                    kwargs.get("bucket", None)
+                    or kwargs.get("minio_bucket", None)
+                    or get_environ_variable("MINIO_BUCKET")
+                )
+                if not bucket_name:
+                    raise ValueError("Missing bucket info to list keys in MinIO service.")
+                return (
+                    key
+                    for obj in cls._srv.objects(bucket_name, **kwargs)
+                    if fnmatch.fnmatch((key := obj.object_name), pattern=pattern)
+                )
+            case _:
+                raise NotImplementedError(f"Keys method not implemented for service {cls._nme}")
+
     def set(self, value: str, key: Optional[str] = None, **kwargs) -> None:
         """Sets a key-value pair in the store.
         The implementation of this method depends on the underlying service.

{fred_oss-0.53.0 → fred_oss-0.54.0}/src/main/fred/dao/comp/interface.py

@@ -7,6 +7,7 @@ SRV_REF_TYPE = str | ServiceInterface | ServiceCatalog
 
 class SrvCompanionMixin:
     _srv: ServiceInterface
+    _cat: ServiceCatalog
 
     @classmethod
     def _set_srv(cls, srv_ref: Optional[SRV_REF_TYPE] = None, **kwargs):
@@ -19,11 +20,16 @@ class SrvCompanionMixin:
         """
         match (srv_ref or "REDIS"):
             case str() as name:
-                cls._srv = ServiceCatalog[name.upper()].auto(**kwargs)
+                cls._cat = ServiceCatalog[name.upper()]
+                cls._srv = cls._cat.auto(**kwargs)
             case ServiceCatalog() as cat:
+                cls._cat = cat
                 cls._srv = cat.auto(**kwargs)
             case ServiceInterface() as instance:
                 cls._srv = instance
+                cls._cat = ServiceCatalog.from_classname(
+                    classname=instance.__class__.__name__
+                )
             case _:
                 raise ValueError(f"Invalid service '{srv_ref}' type: {type(srv_ref)}")
     
@@ -32,11 +38,6 @@ class SrvCompanionMixin:
         """Returns the class name of the current service instance."""
         return self._srv.__class__.__name__
 
-    @property
-    def _cat(self) -> ServiceCatalog:
-        """Returns the ServiceCatalog enum member corresponding to the current service instance."""
-        return ServiceCatalog.from_classname(self._nme)
-
 
 class ComponentInterface(SrvCompanionMixin):
 

fred_oss-0.54.0/src/main/fred/version

@@ -0,0 +1 @@
+0.54.0

fred_oss-0.54.0/src/main/fred/worker/runner/backend.py

@@ -0,0 +1,42 @@
+from dataclasses import dataclass
+
+from fred.settings import logger_manager
+from fred.dao.comp.catalog import FredKeyVal, FredQueue, CompCatalog
+from fred.dao.service.interface import ServiceInterface
+from fred.dao.service.catalog import ServiceCatalog
+
+logger = logger_manager.get_logger(name=__name__)
+
+
+@dataclass(frozen=True, slots=False)
+class RunnerBackend:
+    keyval: FredKeyVal
+    queue: FredQueue
+    # NOTE: Catalog and service instance references (for internal use)
+    # should be a temporal fix until we refactor the backend system
+    # according to a more precise usage-patter (still to be defined/identified).
+    _cat: ServiceCatalog  # Contains complementary service info and simple references
+    _srv: ServiceInterface  # Allows direct access to a client instance
+
+    @classmethod
+    def auto(cls, service_name: str, **kwargs) -> 'RunnerBackend':
+        match (srv_catalog := ServiceCatalog[service_name.upper()]):
+            case ServiceCatalog.REDIS:
+                from fred.dao.service.utils import get_redis_configs_from_payload
+                service_kwargs = get_redis_configs_from_payload(kwargs)
+            case ServiceCatalog.STDLIB:
+                service_kwargs = {}
+            case _:
+                logger.error(
+                    f"Unknown service '{service_name}'... "
+                    "will attempt to use provided kwargs as-is."
+                )
+                service_kwargs = kwargs
+        logger.info(f"Initializing RunnerBackend using service '{service_name}'")
+        srv_instance = srv_catalog.auto(**service_kwargs)
+        return cls(
+            keyval=CompCatalog.KEYVAL.value.mount(srv_ref=srv_instance),
+            queue=CompCatalog.QUEUE.value.mount(srv_ref=srv_instance),
+            _cat=srv_catalog,
+            _srv=srv_instance,
+        )

{fred_oss-0.53.0 → fred_oss-0.54.0}/src/main/fred/worker/runner/client.py

@@ -61,14 +61,34 @@ class RunnerClient:
         signal = RunnerSignal[signal.upper()] if isinstance(signal, str) else signal
         return signal.send(self.req_queue)
 
-    def runner_status(self, runner_id: str) -> RunnerStatus:
-        runner_status=self._runner_backend.keyval(
-                key=RunnerStatus.get_key(runner_id=runner_id)
-            )
-        if not (value := runner_status.get()):
+    def runner_info(self, runner_id: str) -> tuple[str, RunnerStatus]:
+        runner_status = self._runner_backend.keyval(
+            key=RunnerStatus.get_key(runner_id=runner_id)
+        )
+        if not (out := runner_status.get()):
             logger.warning(f"No status found for runner_id: '{runner_id}'")
-            return RunnerStatus.UNDEFINED
-        return RunnerStatus.parse_value(value=value)
+            return ("", RunnerStatus.UNDEFINED)
+        return RunnerStatus.parse_value(value=out)
+
+    def runner_status(self, runner_id: str) -> RunnerStatus:
+        _, status = self.runner_info(runner_id=runner_id)
+        return status
+
+    def runner_queue(self, runner_id: str) -> str:
+        queue_slug, _ = self.runner_info(runner_id=runner_id)
+        return queue_slug
+
+    def runners(self) -> dict[str, Optional[str]]:
+        return {
+            key: self._runner_backend.keyval(key=key).get()
+            for key in self._runner_backend.keyval.keys(pattern="frd:runner:*")
+        }
+
+    def futures(self) -> dict[str, Optional[str]]:
+        return {
+            key: self._runner_backend.keyval(key=key).get()
+            for key in self._runner_backend.keyval.keys(pattern="frd:future:*:status")
+        }
 
     def send(
             self,

{fred_oss-0.53.0 → fred_oss-0.54.0}/src/main/fred/worker/runner/handler.py

@@ -140,9 +140,11 @@ class RunnerHandler(HandlerInterface):
         runner_status = runner_backend.keyval(
             key=RunnerStatus.get_key(runner_id=runner_id)
         )
-        logger.info(f"Starting runner with ID '{runner_id}' using request-queue '{req_queue.name}'")
+        # Start the runner loop in a future and track its status
+        on_start_queue_size = req_queue.size()
+        logger.info(f"Starting runner '{runner_id}' using req-queue '{req_queue.name}': {on_start_queue_size}")
         runner_status.set(
-            value=RunnerStatus.STARTED.get_val(),
+            value=RunnerStatus.STARTED.get_val(spec.queue_slug, f"Q({on_start_queue_size})"),
             expire=None,
         )
         runner_loop = Future(
@@ -156,7 +158,7 @@ class RunnerHandler(HandlerInterface):
             future_id=runner_id,
         )
         runner_status.set(
-            value=RunnerStatus.RUNNING.get_val(),
+            value=RunnerStatus.RUNNING.get_val(spec.queue_slug, f"Q({req_queue.size()})"),
             expire=None,
         )
         results = {
@@ -180,7 +182,7 @@ class RunnerHandler(HandlerInterface):
                 }
         results["pending_requests"] = pending_requests = req_queue.size()
         runner_status.set(
-            value=RunnerStatus.STOPPED.get_val(str(pending_requests)),
+            value=RunnerStatus.STOPPED.get_val(spec.queue_slug, f"Q({pending_requests})"),
             expire=3600,  # Keep the stopped status for 1 hour
         )
         if pending_requests:

{fred_oss-0.53.0 → fred_oss-0.54.0}/src/main/fred/worker/runner/status.py

@@ -14,14 +14,14 @@ class RunnerStatus(Enum):
     @staticmethod
     def get_key(runner_id: str) -> str:
         """Get the status key for the specified runner ID."""
-        return f"runner:status:{runner_id}"
+        return f"frd:runner:{runner_id}"
     
-    def get_val(self, *include) -> str:
+    def get_val(self, queue_slug: str, *include) -> str:
         """Get the string representation of the status."""
-        return ":".join([self.name, datetime_utcnow().isoformat(), *include]) 
+        return ":".join([queue_slug, self.name, datetime_utcnow().isoformat(), *include]) 
     
     @classmethod
-    def parse_value(cls, value: str) -> "RunnerStatus":
+    def parse_value(cls, value: str) -> tuple[str, "RunnerStatus"]:
         """Parse the status from the stored value."""
-        val, *_ = value.split(":")
-        return cls[val.upper()]
+        queue_slug, status, *_ = value.split(":")
+        return (queue_slug, cls[status.upper()])

{fred_oss-0.53.0 → fred_oss-0.54.0/src/main/fred_oss.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: fred-oss
-Version: 0.53.0
+Version: 0.54.0
 Summary: FREDOSS
 Home-page: https://fred.fahera.mx
 Author: Fahera Research, Education, and Development

fred_oss-0.53.0/src/main/fred/version

@@ -1 +0,0 @@
-0.53.0

fred_oss-0.53.0/src/main/fred/worker/runner/backend.py

@@ -1,33 +0,0 @@
-from dataclasses import dataclass
-
-from fred.settings import logger_manager
-from fred.dao.comp.catalog import FredKeyVal, FredQueue
-from fred.dao.service.catalog import ServiceCatalog
-
-logger = logger_manager.get_logger(name=__name__)
-
-
-@dataclass(frozen=True, slots=False)
-class RunnerBackend:
-    keyval: FredKeyVal
-    queue: FredQueue
-    _cat: ServiceCatalog
-
-    @classmethod
-    def auto(cls, service_name: str, **kwargs) -> 'RunnerBackend':
-        service = ServiceCatalog[service_name.upper()]
-        match service:
-            case ServiceCatalog.REDIS:
-                from fred.dao.service.utils import get_redis_configs_from_payload
-                service_kwargs = get_redis_configs_from_payload(kwargs)
-            case ServiceCatalog.STDLIB:
-                service_kwargs = {}
-            case _:
-                logger.error(f"Unknown service '{service_name}'... will attempt to use provided kwargs as-is.")
-                service_kwargs = kwargs
-        components = service.component_catalog(**service_kwargs)
-        return cls(
-            keyval=components.KEYVAL.value,
-            queue=components.QUEUE.value,
-            _cat=service,
-        )