pydocket 0.6.4__py3-none-any.whl → 0.7.1__py3-none-any.whl

docket/annotations.py

@@ -2,6 +2,8 @@ import abc
 import inspect
 from typing import Any, Iterable, Mapping, Self
 
+from .instrumentation import CACHE_SIZE
+
 
 class Annotation(abc.ABC):
     _cache: dict[tuple[type[Self], inspect.Signature], Mapping[str, Self]] = {}
@@ -10,6 +12,7 @@ class Annotation(abc.ABC):
     def annotated_parameters(cls, signature: inspect.Signature) -> Mapping[str, Self]:
         key = (cls, signature)
         if key in cls._cache:
+            CACHE_SIZE.set(len(cls._cache), {"cache": "annotation"})
             return cls._cache[key]
 
         annotated: dict[str, Self] = {}
@@ -30,6 +33,7 @@ class Annotation(abc.ABC):
                     annotated[param_name] = arg_type()
 
         cls._cache[key] = annotated
+        CACHE_SIZE.set(len(cls._cache), {"cache": "annotation"})
         return annotated
 
 

docket/cli.py

@@ -358,6 +358,32 @@ def strike(
     asyncio.run(run())
 
 
+@app.command(help="Clear all pending and scheduled tasks from the docket")
+def clear(
+    docket_: Annotated[
+        str,
+        typer.Option(
+            "--docket",
+            help="The name of the docket",
+            envvar="DOCKET_NAME",
+        ),
+    ] = "docket",
+    url: Annotated[
+        str,
+        typer.Option(
+            help="The URL of the Redis server",
+            envvar="DOCKET_URL",
+        ),
+    ] = "redis://localhost:6379/0",
+) -> None:
+    async def run() -> None:
+        async with Docket(name=docket_, url=url) as docket:
+            cleared_count = await docket.clear()
+            print(f"Cleared {cleared_count} tasks from docket '{docket_}'")
+
+    asyncio.run(run())
+
+
 @app.command(help="Restores a task or parameters to the Docket")
 def restore(
     function: Annotated[

docket/dependencies.py

@@ -3,7 +3,7 @@ import logging
 import time
 from contextlib import AsyncExitStack, asynccontextmanager
 from contextvars import ContextVar
-from datetime import timedelta
+from datetime import datetime, timedelta, timezone
 from types import TracebackType
 from typing import (
     TYPE_CHECKING,
@@ -14,12 +14,14 @@ from typing import (
     Callable,
     Counter,
     Generic,
+    NoReturn,
     TypeVar,
     cast,
 )
 
 from .docket import Docket
 from .execution import Execution, TaskFunction, get_signature
+from .instrumentation import CACHE_SIZE
 
 if TYPE_CHECKING:  # pragma: no cover
     from .worker import Worker
@@ -188,6 +190,10 @@ def TaskLogger() -> logging.LoggerAdapter[logging.Logger]:
     return cast(logging.LoggerAdapter[logging.Logger], _TaskLogger())
 
 
+class ForcedRetry(Exception):
+    """Raised when a task requests a retry via `in_` or `at`"""
+
+
 class Retry(Dependency):
     """Configures linear retries for a task.  You can specify the total number of
     attempts (or `None` to retry indefinitely), and the delay between attempts.
@@ -222,6 +228,17 @@ class Retry(Dependency):
         retry.attempt = execution.attempt
         return retry
 
+    def at(self, when: datetime) -> NoReturn:
+        now = datetime.now(timezone.utc)
+        diff = when - now
+        diff = diff if diff.total_seconds() >= 0 else timedelta(0)
+
+        self.in_(diff)
+
+    def in_(self, when: timedelta) -> NoReturn:
+        self.delay: timedelta = when
+        raise ForcedRetry()
+
 
 class ExponentialRetry(Retry):
     """Configures exponential retries for a task.  You can specify the total number
@@ -251,7 +268,6 @@ class ExponentialRetry(Retry):
             maximum_delay: The maximum delay between attempts.
         """
         super().__init__(attempts=attempts, delay=minimum_delay)
-        self.minimum_delay = minimum_delay
         self.maximum_delay = maximum_delay
 
     async def __aenter__(self) -> "ExponentialRetry":
@@ -259,14 +275,14 @@ class ExponentialRetry(Retry):
 
         retry = ExponentialRetry(
             attempts=self.attempts,
-            minimum_delay=self.minimum_delay,
+            minimum_delay=self.delay,
             maximum_delay=self.maximum_delay,
         )
         retry.attempt = execution.attempt
 
         if execution.attempt > 1:
             backoff_factor = 2 ** (execution.attempt - 1)
-            calculated_delay = self.minimum_delay * backoff_factor
+            calculated_delay = self.delay * backoff_factor
 
             if calculated_delay > self.maximum_delay:
                 retry.delay = self.maximum_delay
@@ -400,6 +416,7 @@ def get_dependency_parameters(
     function: TaskFunction | DependencyFunction[Any],
 ) -> dict[str, Dependency]:
     if function in _parameter_cache:
+        CACHE_SIZE.set(len(_parameter_cache), {"cache": "parameter"})
         return _parameter_cache[function]
 
     dependencies: dict[str, Dependency] = {}
@@ -413,6 +430,7 @@ def get_dependency_parameters(
         dependencies[parameter] = param.default
 
     _parameter_cache[function] = dependencies
+    CACHE_SIZE.set(len(_parameter_cache), {"cache": "parameter"})
     return dependencies
 
 

docket/docket.py

@@ -743,3 +743,46 @@ class Docket:
                 workers.append(WorkerInfo(worker_name, last_seen, task_names))
 
         return workers
+
+    async def clear(self) -> int:
+        """Clear all pending and scheduled tasks from the docket.
+
+        This removes all tasks from the stream (immediate tasks) and queue
+        (scheduled tasks), along with their associated parked data. Running
+        tasks are not affected.
+
+        Returns:
+            The total number of tasks that were cleared.
+        """
+        with tracer.start_as_current_span(
+            "docket.clear",
+            attributes=self.labels(),
+        ):
+            async with self.redis() as redis:
+                async with redis.pipeline() as pipeline:
+                    # Get counts before clearing
+                    pipeline.xlen(self.stream_key)
+                    pipeline.zcard(self.queue_key)
+                    pipeline.zrange(self.queue_key, 0, -1)
+
+                    stream_count: int
+                    queue_count: int
+                    scheduled_keys: list[bytes]
+                    stream_count, queue_count, scheduled_keys = await pipeline.execute()
+
+                    # Clear all data
+                    # Trim stream to 0 messages instead of deleting it to preserve consumer group
+                    if stream_count > 0:
+                        pipeline.xtrim(self.stream_key, maxlen=0, approximate=False)
+                    pipeline.delete(self.queue_key)
+
+                    # Clear parked task data and known task keys
+                    for key_bytes in scheduled_keys:
+                        key = key_bytes.decode()
+                        pipeline.delete(self.parked_task_key(key))
+                        pipeline.delete(self.known_task_key(key))
+
+                    await pipeline.execute()
+
+                    total_cleared = stream_count + queue_count
+                    return total_cleared

docket/execution.py

@@ -19,7 +19,7 @@ import opentelemetry.context
 from opentelemetry import propagate, trace
 
 from .annotations import Logged
-from .instrumentation import message_getter
+from .instrumentation import CACHE_SIZE, message_getter
 
 logger: logging.Logger = logging.getLogger(__name__)
 
@@ -32,10 +32,12 @@ _signature_cache: dict[Callable[..., Any], inspect.Signature] = {}
 
 def get_signature(function: Callable[..., Any]) -> inspect.Signature:
     if function in _signature_cache:
+        CACHE_SIZE.set(len(_signature_cache), {"cache": "signature"})
         return _signature_cache[function]
 
     signature = inspect.signature(function)
     _signature_cache[function] = signature
+    CACHE_SIZE.set(len(_signature_cache), {"cache": "signature"})
     return signature
 
 

docket/instrumentation.py

@@ -117,6 +117,12 @@ SCHEDULE_DEPTH = meter.create_gauge(
     unit="1",
 )
 
+CACHE_SIZE = meter.create_gauge(
+    "docket_cache_size",
+    description="Size of internal docket caches",
+    unit="1",
+)
+
 Message = dict[bytes, bytes]
 
 

docket/worker.py

@@ -15,7 +15,7 @@ from typing import (
 )
 
 from opentelemetry import trace
-from opentelemetry.trace import Tracer
+from opentelemetry.trace import Status, StatusCode, Tracer
 from redis.asyncio import Redis
 from redis.exceptions import ConnectionError, LockError
 
@@ -531,7 +531,7 @@ class Worker:
                 "code.function.name": execution.function.__name__,
             },
             links=execution.incoming_span_links(),
-        ):
+        ) as span:
             try:
                 async with resolved_dependencies(self, execution) as dependencies:
                     # Preemptively reschedule the perpetual task for the future, or clear
@@ -576,6 +576,8 @@ class Worker:
                     duration = log_context["duration"] = time.time() - start
                     TASKS_SUCCEEDED.add(1, counter_labels)
 
+                    span.set_status(Status(StatusCode.OK))
+
                     rescheduled = await self._perpetuate_if_requested(
                         execution, dependencies, timedelta(seconds=duration)
                     )
@@ -584,10 +586,13 @@ class Worker:
                     logger.info(
                         "%s [%s] %s", arrow, ms(duration), call, extra=log_context
                     )
-            except Exception:
+            except Exception as e:
                 duration = log_context["duration"] = time.time() - start
                 TASKS_FAILED.add(1, counter_labels)
 
+                span.record_exception(e)
+                span.set_status(Status(StatusCode.ERROR, str(e)))
+
                 retried = await self._retry_if_requested(execution, dependencies)
                 if not retried:
                     retried = await self._perpetuate_if_requested(

{pydocket-0.6.4.dist-info → pydocket-0.7.1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: pydocket
-Version: 0.6.4
+Version: 0.7.1
 Summary: A distributed background task system for Python functions
 Project-URL: Homepage, https://github.com/chrisguidry/docket
 Project-URL: Bug Tracker, https://github.com/chrisguidry/docket/issues

pydocket-0.7.1.dist-info/RECORD

@@ -0,0 +1,16 @@
+docket/__init__.py,sha256=sY1T_NVsXQNOmOhOnfYmZ95dcE_52Ov6DSIVIMZp-1w,869
+docket/__main__.py,sha256=wcCrL4PjG51r5wVKqJhcoJPTLfHW0wNbD31DrUN0MWI,28
+docket/annotations.py,sha256=wttix9UOeMFMAWXAIJUfUw5GjESJZsACb4YXJCozP7Q,2348
+docket/cli.py,sha256=XG_mbjcqNRO0F0hh6l3AwH9bIZv9xJofZaeaAj9nChc,21608
+docket/dependencies.py,sha256=GBwyEY198JFrfm7z5GkLbd84hv7sJktKBMJXv4veWig,17007
+docket/docket.py,sha256=Cw7QB1d0eDwSgwn0Rj26WjFsXSe7MJtfsUBBHGalL7A,26262
+docket/execution.py,sha256=r_2RGC1qhtAcBUg7E6wewLEgftrf3hIxNbH0HnYPbek,14961
+docket/instrumentation.py,sha256=ogvzrfKbWsdPGfdg4hByH3_r5d3b5AwwQkSrmXw0hRg,5492
+docket/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+docket/tasks.py,sha256=RIlSM2omh-YDwVnCz6M5MtmK8T_m_s1w2OlRRxDUs6A,1437
+docket/worker.py,sha256=CY5Z9p8FZw-6WUwp7Ws4A0V7IFTmonSnBmYP-Cp8Fdw,28079
+pydocket-0.7.1.dist-info/METADATA,sha256=00KHm5Er2R6dmjHLTYBUF13kKAeCRPHmDTdAcv5oRcQ,5335
+pydocket-0.7.1.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+pydocket-0.7.1.dist-info/entry_points.txt,sha256=4WOk1nUlBsUT5O3RyMci2ImuC5XFswuopElYcLHtD5k,47
+pydocket-0.7.1.dist-info/licenses/LICENSE,sha256=YuVWU_ZXO0K_k2FG8xWKe5RGxV24AhJKTvQmKfqXuyk,1087
+pydocket-0.7.1.dist-info/RECORD,,

pydocket-0.6.4.dist-info/RECORD

@@ -1,16 +0,0 @@
-docket/__init__.py,sha256=sY1T_NVsXQNOmOhOnfYmZ95dcE_52Ov6DSIVIMZp-1w,869
-docket/__main__.py,sha256=wcCrL4PjG51r5wVKqJhcoJPTLfHW0wNbD31DrUN0MWI,28
-docket/annotations.py,sha256=SFBrOMbpAh7P67u8fRTH-u3MVvJQxe0qYi92WAShAsw,2173
-docket/cli.py,sha256=WPm_URZ54h8gHjrsHKP8SXpRzdeepmyH_FhQHai-Qus,20899
-docket/dependencies.py,sha256=_31Fgn6A_4aWn5TJpXdbsPtimBVIPabNJkw49RRLJTc,16441
-docket/docket.py,sha256=5e101CGLZ2tWNcADo4cdewapmXab47ieMCeQr6d92YQ,24478
-docket/execution.py,sha256=6KozjnS96byvyCMTQ2-IkcIrPsqaPIVu2HZU0U4Be9E,14813
-docket/instrumentation.py,sha256=f-GG5VS6EdS2It30qxjVpzWUBOZQcTnat-3KzPwwDgQ,5367
-docket/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-docket/tasks.py,sha256=RIlSM2omh-YDwVnCz6M5MtmK8T_m_s1w2OlRRxDUs6A,1437
-docket/worker.py,sha256=tJfk2rlHODzHaWBzpBXT8h-Lo7RDQ6gb6HU8b3T9gFA,27878
-pydocket-0.6.4.dist-info/METADATA,sha256=R3ODtTRkrNkplBvC5-8pVsRjSLSfYKYHKXqZCT9Qr-w,5335
-pydocket-0.6.4.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-pydocket-0.6.4.dist-info/entry_points.txt,sha256=4WOk1nUlBsUT5O3RyMci2ImuC5XFswuopElYcLHtD5k,47
-pydocket-0.6.4.dist-info/licenses/LICENSE,sha256=YuVWU_ZXO0K_k2FG8xWKe5RGxV24AhJKTvQmKfqXuyk,1087
-pydocket-0.6.4.dist-info/RECORD,,