pydocket 0.11.1__py3-none-any.whl → 0.12.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of pydocket might be problematic. Click here for more details.
- docket/annotations.py +3 -1
- docket/cli.py +35 -8
- docket/dependencies.py +83 -22
- docket/docket.py +28 -3
- docket/execution.py +10 -11
- docket/instrumentation.py +8 -1
- docket/tasks.py +2 -2
- docket/worker.py +29 -19
- {pydocket-0.11.1.dist-info → pydocket-0.12.0.dist-info}/METADATA +21 -3
- pydocket-0.12.0.dist-info/RECORD +17 -0
- pydocket-0.11.1.dist-info/RECORD +0 -17
- {pydocket-0.11.1.dist-info → pydocket-0.12.0.dist-info}/WHEEL +0 -0
- {pydocket-0.11.1.dist-info → pydocket-0.12.0.dist-info}/entry_points.txt +0 -0
- {pydocket-0.11.1.dist-info → pydocket-0.12.0.dist-info}/licenses/LICENSE +0 -0
docket/annotations.py
CHANGED
docket/cli.py
CHANGED
|
@@ -25,7 +25,7 @@ app: typer.Typer = typer.Typer(
|
|
|
25
25
|
)
|
|
26
26
|
|
|
27
27
|
|
|
28
|
-
class LogLevel(enum.
|
|
28
|
+
class LogLevel(str, enum.Enum):
|
|
29
29
|
DEBUG = "DEBUG"
|
|
30
30
|
INFO = "INFO"
|
|
31
31
|
WARNING = "WARNING"
|
|
@@ -33,7 +33,7 @@ class LogLevel(enum.StrEnum):
|
|
|
33
33
|
CRITICAL = "CRITICAL"
|
|
34
34
|
|
|
35
35
|
|
|
36
|
-
class LogFormat(enum.
|
|
36
|
+
class LogFormat(str, enum.Enum):
|
|
37
37
|
RICH = "rich"
|
|
38
38
|
PLAIN = "plain"
|
|
39
39
|
JSON = "json"
|
|
@@ -111,7 +111,23 @@ def set_logging_format(format: LogFormat) -> None:
|
|
|
111
111
|
|
|
112
112
|
|
|
113
113
|
def set_logging_level(level: LogLevel) -> None:
|
|
114
|
-
logging.getLogger().setLevel(level)
|
|
114
|
+
logging.getLogger().setLevel(level.value)
|
|
115
|
+
|
|
116
|
+
|
|
117
|
+
def validate_url(url: str) -> str:
|
|
118
|
+
"""
|
|
119
|
+
Validate that the provided URL is compatible with the CLI.
|
|
120
|
+
|
|
121
|
+
The memory:// backend is not compatible with the CLI as it doesn't persist
|
|
122
|
+
across processes.
|
|
123
|
+
"""
|
|
124
|
+
if url.startswith("memory://"):
|
|
125
|
+
raise typer.BadParameter(
|
|
126
|
+
"The memory:// URL scheme is not supported by the CLI.\n"
|
|
127
|
+
"The memory backend does not persist across processes.\n"
|
|
128
|
+
"Please use a persistent backend like Redis or Valkey."
|
|
129
|
+
)
|
|
130
|
+
return url
|
|
115
131
|
|
|
116
132
|
|
|
117
133
|
def handle_strike_wildcard(value: str) -> str | None:
|
|
@@ -178,6 +194,7 @@ def worker(
|
|
|
178
194
|
typer.Option(
|
|
179
195
|
help="The URL of the Redis server",
|
|
180
196
|
envvar="DOCKET_URL",
|
|
197
|
+
callback=validate_url,
|
|
181
198
|
),
|
|
182
199
|
] = "redis://localhost:6379/0",
|
|
183
200
|
name: Annotated[
|
|
@@ -336,6 +353,7 @@ def strike(
|
|
|
336
353
|
typer.Option(
|
|
337
354
|
help="The URL of the Redis server",
|
|
338
355
|
envvar="DOCKET_URL",
|
|
356
|
+
callback=validate_url,
|
|
339
357
|
),
|
|
340
358
|
] = "redis://localhost:6379/0",
|
|
341
359
|
) -> None:
|
|
@@ -347,7 +365,7 @@ def strike(
|
|
|
347
365
|
value_ = interpret_python_value(value)
|
|
348
366
|
if parameter:
|
|
349
367
|
function_name = f"{function or '(all tasks)'}"
|
|
350
|
-
print(f"Striking {function_name} {parameter} {operator} {value_!r}")
|
|
368
|
+
print(f"Striking {function_name} {parameter} {operator.value} {value_!r}")
|
|
351
369
|
else:
|
|
352
370
|
print(f"Striking {function}")
|
|
353
371
|
|
|
@@ -373,6 +391,7 @@ def clear(
|
|
|
373
391
|
typer.Option(
|
|
374
392
|
help="The URL of the Redis server",
|
|
375
393
|
envvar="DOCKET_URL",
|
|
394
|
+
callback=validate_url,
|
|
376
395
|
),
|
|
377
396
|
] = "redis://localhost:6379/0",
|
|
378
397
|
) -> None:
|
|
@@ -425,6 +444,7 @@ def restore(
|
|
|
425
444
|
typer.Option(
|
|
426
445
|
help="The URL of the Redis server",
|
|
427
446
|
envvar="DOCKET_URL",
|
|
447
|
+
callback=validate_url,
|
|
428
448
|
),
|
|
429
449
|
] = "redis://localhost:6379/0",
|
|
430
450
|
) -> None:
|
|
@@ -436,7 +456,7 @@ def restore(
|
|
|
436
456
|
value_ = interpret_python_value(value)
|
|
437
457
|
if parameter:
|
|
438
458
|
function_name = f"{function or '(all tasks)'}"
|
|
439
|
-
print(f"Restoring {function_name} {parameter} {operator} {value_!r}")
|
|
459
|
+
print(f"Restoring {function_name} {parameter} {operator.value} {value_!r}")
|
|
440
460
|
else:
|
|
441
461
|
print(f"Restoring {function}")
|
|
442
462
|
|
|
@@ -468,6 +488,7 @@ def trace(
|
|
|
468
488
|
typer.Option(
|
|
469
489
|
help="The URL of the Redis server",
|
|
470
490
|
envvar="DOCKET_URL",
|
|
491
|
+
callback=validate_url,
|
|
471
492
|
),
|
|
472
493
|
] = "redis://localhost:6379/0",
|
|
473
494
|
message: Annotated[
|
|
@@ -511,6 +532,7 @@ def fail(
|
|
|
511
532
|
typer.Option(
|
|
512
533
|
help="The URL of the Redis server",
|
|
513
534
|
envvar="DOCKET_URL",
|
|
535
|
+
callback=validate_url,
|
|
514
536
|
),
|
|
515
537
|
] = "redis://localhost:6379/0",
|
|
516
538
|
message: Annotated[
|
|
@@ -554,6 +576,7 @@ def sleep(
|
|
|
554
576
|
typer.Option(
|
|
555
577
|
help="The URL of the Redis server",
|
|
556
578
|
envvar="DOCKET_URL",
|
|
579
|
+
callback=validate_url,
|
|
557
580
|
),
|
|
558
581
|
] = "redis://localhost:6379/0",
|
|
559
582
|
seconds: Annotated[
|
|
@@ -688,6 +711,7 @@ def snapshot(
|
|
|
688
711
|
typer.Option(
|
|
689
712
|
help="The URL of the Redis server",
|
|
690
713
|
envvar="DOCKET_URL",
|
|
714
|
+
callback=validate_url,
|
|
691
715
|
),
|
|
692
716
|
] = "redis://localhost:6379/0",
|
|
693
717
|
stats: Annotated[
|
|
@@ -746,10 +770,11 @@ def snapshot(
|
|
|
746
770
|
|
|
747
771
|
console.print(table)
|
|
748
772
|
|
|
749
|
-
# Display task statistics if requested
|
|
750
|
-
|
|
773
|
+
# Display task statistics if requested. On Linux the Click runner executes
|
|
774
|
+
# this CLI in a subprocess, so coverage cannot observe it. Mark as no cover.
|
|
775
|
+
if stats: # pragma: no cover
|
|
751
776
|
task_stats = get_task_stats(snapshot)
|
|
752
|
-
if task_stats:
|
|
777
|
+
if task_stats: # pragma: no cover
|
|
753
778
|
console.print() # Add spacing between tables
|
|
754
779
|
stats_table = Table(title="Task Count Statistics by Function")
|
|
755
780
|
stats_table.add_column("Function", style="cyan")
|
|
@@ -839,6 +864,7 @@ def list_workers(
|
|
|
839
864
|
typer.Option(
|
|
840
865
|
help="The URL of the Redis server",
|
|
841
866
|
envvar="DOCKET_URL",
|
|
867
|
+
callback=validate_url,
|
|
842
868
|
),
|
|
843
869
|
] = "redis://localhost:6379/0",
|
|
844
870
|
) -> None:
|
|
@@ -875,6 +901,7 @@ def workers_for_task(
|
|
|
875
901
|
typer.Option(
|
|
876
902
|
help="The URL of the Redis server",
|
|
877
903
|
envvar="DOCKET_URL",
|
|
904
|
+
callback=validate_url,
|
|
878
905
|
),
|
|
879
906
|
] = "redis://localhost:6379/0",
|
|
880
907
|
) -> None:
|
docket/dependencies.py
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import abc
|
|
2
|
+
import inspect
|
|
2
3
|
import logging
|
|
3
4
|
import time
|
|
4
5
|
from contextlib import AsyncExitStack, asynccontextmanager
|
|
@@ -12,6 +13,7 @@ from typing import (
|
|
|
12
13
|
AsyncGenerator,
|
|
13
14
|
Awaitable,
|
|
14
15
|
Callable,
|
|
16
|
+
ContextManager,
|
|
15
17
|
Counter,
|
|
16
18
|
Generic,
|
|
17
19
|
NoReturn,
|
|
@@ -161,7 +163,7 @@ def TaskArgument(parameter: str | None = None, optional: bool = False) -> Any:
|
|
|
161
163
|
|
|
162
164
|
|
|
163
165
|
class _TaskLogger(Dependency):
|
|
164
|
-
async def __aenter__(self) -> logging.LoggerAdapter[logging.Logger]:
|
|
166
|
+
async def __aenter__(self) -> "logging.LoggerAdapter[logging.Logger]":
|
|
165
167
|
execution = self.execution.get()
|
|
166
168
|
logger = logging.getLogger(f"docket.task.{execution.function.__name__}")
|
|
167
169
|
return logging.LoggerAdapter(
|
|
@@ -174,7 +176,7 @@ class _TaskLogger(Dependency):
|
|
|
174
176
|
)
|
|
175
177
|
|
|
176
178
|
|
|
177
|
-
def TaskLogger() -> logging.LoggerAdapter[logging.Logger]:
|
|
179
|
+
def TaskLogger() -> "logging.LoggerAdapter[logging.Logger]":
|
|
178
180
|
"""A dependency to access a logger for the currently executing task. The logger
|
|
179
181
|
will automatically inject contextual information such as the worker and docket
|
|
180
182
|
name, the task key, and the current execution attempt number.
|
|
@@ -183,11 +185,11 @@ def TaskLogger() -> logging.LoggerAdapter[logging.Logger]:
|
|
|
183
185
|
|
|
184
186
|
```python
|
|
185
187
|
@task
|
|
186
|
-
async def my_task(logger: LoggerAdapter[Logger] = TaskLogger()) -> None:
|
|
188
|
+
async def my_task(logger: "LoggerAdapter[Logger]" = TaskLogger()) -> None:
|
|
187
189
|
logger.info("Hello, world!")
|
|
188
190
|
```
|
|
189
191
|
"""
|
|
190
|
-
return cast(logging.LoggerAdapter[logging.Logger], _TaskLogger())
|
|
192
|
+
return cast("logging.LoggerAdapter[logging.Logger]", _TaskLogger())
|
|
191
193
|
|
|
192
194
|
|
|
193
195
|
class ForcedRetry(Exception):
|
|
@@ -403,7 +405,9 @@ class Timeout(Dependency):
|
|
|
403
405
|
|
|
404
406
|
R = TypeVar("R")
|
|
405
407
|
|
|
406
|
-
DependencyFunction = Callable[
|
|
408
|
+
DependencyFunction = Callable[
|
|
409
|
+
..., R | Awaitable[R] | ContextManager[R] | AsyncContextManager[R]
|
|
410
|
+
]
|
|
407
411
|
|
|
408
412
|
|
|
409
413
|
_parameter_cache: dict[
|
|
@@ -441,7 +445,10 @@ class _Depends(Dependency, Generic[R]):
|
|
|
441
445
|
stack: ContextVar[AsyncExitStack] = ContextVar("stack")
|
|
442
446
|
|
|
443
447
|
def __init__(
|
|
444
|
-
self,
|
|
448
|
+
self,
|
|
449
|
+
dependency: Callable[
|
|
450
|
+
[], R | Awaitable[R] | ContextManager[R] | AsyncContextManager[R]
|
|
451
|
+
],
|
|
445
452
|
) -> None:
|
|
446
453
|
self.dependency = dependency
|
|
447
454
|
|
|
@@ -473,33 +480,87 @@ class _Depends(Dependency, Generic[R]):
|
|
|
473
480
|
stack = self.stack.get()
|
|
474
481
|
arguments = await self._resolve_parameters(self.dependency)
|
|
475
482
|
|
|
476
|
-
|
|
483
|
+
raw_value: R | Awaitable[R] | ContextManager[R] | AsyncContextManager[R] = (
|
|
484
|
+
self.dependency(**arguments)
|
|
485
|
+
)
|
|
477
486
|
|
|
478
|
-
|
|
479
|
-
|
|
487
|
+
# Handle different return types from the dependency function
|
|
488
|
+
resolved_value: R
|
|
489
|
+
if isinstance(raw_value, AsyncContextManager):
|
|
490
|
+
# Async context manager: await enter_async_context
|
|
491
|
+
resolved_value = await stack.enter_async_context(raw_value)
|
|
492
|
+
elif isinstance(raw_value, ContextManager):
|
|
493
|
+
# Sync context manager: use enter_context (no await needed)
|
|
494
|
+
resolved_value = stack.enter_context(raw_value)
|
|
495
|
+
elif inspect.iscoroutine(raw_value) or isinstance(raw_value, Awaitable):
|
|
496
|
+
# Async function returning awaitable: await it
|
|
497
|
+
resolved_value = await cast(Awaitable[R], raw_value)
|
|
480
498
|
else:
|
|
481
|
-
value
|
|
499
|
+
# Sync function returning a value directly, use as-is
|
|
500
|
+
resolved_value = cast(R, raw_value)
|
|
482
501
|
|
|
483
|
-
cache[self.dependency] =
|
|
484
|
-
return
|
|
502
|
+
cache[self.dependency] = resolved_value
|
|
503
|
+
return resolved_value
|
|
485
504
|
|
|
486
505
|
|
|
487
506
|
def Depends(dependency: DependencyFunction[R]) -> R:
|
|
488
|
-
"""Include a user-defined function as a dependency. Dependencies may
|
|
489
|
-
|
|
490
|
-
|
|
491
|
-
|
|
507
|
+
"""Include a user-defined function as a dependency. Dependencies may be:
|
|
508
|
+
- Synchronous functions returning a value
|
|
509
|
+
- Asynchronous functions returning a value (awaitable)
|
|
510
|
+
- Synchronous context managers (using @contextmanager)
|
|
511
|
+
- Asynchronous context managers (using @asynccontextmanager)
|
|
492
512
|
|
|
493
|
-
|
|
513
|
+
If a dependency returns a context manager, it will be entered and exited around
|
|
514
|
+
the task, giving an opportunity to control the lifetime of a resource.
|
|
494
515
|
|
|
495
|
-
|
|
516
|
+
**Important**: Synchronous dependencies should NOT include blocking I/O operations
|
|
517
|
+
(file access, network calls, database queries, etc.). Use async dependencies for
|
|
518
|
+
any I/O. Sync dependencies are best for:
|
|
519
|
+
- Pure computations
|
|
520
|
+
- In-memory data structure access
|
|
521
|
+
- Configuration lookups from memory
|
|
522
|
+
- Non-blocking transformations
|
|
496
523
|
|
|
497
|
-
|
|
498
|
-
return "Hello, world!"
|
|
524
|
+
Examples:
|
|
499
525
|
|
|
500
|
-
|
|
501
|
-
|
|
526
|
+
```python
|
|
527
|
+
# Sync dependency - pure computation, no I/O
|
|
528
|
+
def get_config() -> dict:
|
|
529
|
+
# Access in-memory config, no I/O
|
|
530
|
+
return {"api_url": "https://api.example.com", "timeout": 30}
|
|
531
|
+
|
|
532
|
+
# Sync dependency - compute value from arguments
|
|
533
|
+
def build_query_params(
|
|
534
|
+
user_id: int = TaskArgument(),
|
|
535
|
+
config: dict = Depends(get_config)
|
|
536
|
+
) -> dict:
|
|
537
|
+
# Pure computation, no I/O
|
|
538
|
+
return {"user_id": user_id, "timeout": config["timeout"]}
|
|
539
|
+
|
|
540
|
+
# Async dependency - I/O operations
|
|
541
|
+
async def get_user(user_id: int = TaskArgument()) -> User:
|
|
542
|
+
# Network I/O - must be async
|
|
543
|
+
return await fetch_user_from_api(user_id)
|
|
544
|
+
|
|
545
|
+
# Async context manager - I/O resource management
|
|
546
|
+
from contextlib import asynccontextmanager
|
|
547
|
+
|
|
548
|
+
@asynccontextmanager
|
|
549
|
+
async def get_db_connection():
|
|
550
|
+
# I/O operations - must be async
|
|
551
|
+
conn = await db.connect()
|
|
552
|
+
try:
|
|
553
|
+
yield conn
|
|
554
|
+
finally:
|
|
555
|
+
await conn.close()
|
|
502
556
|
|
|
557
|
+
@task
|
|
558
|
+
async def my_task(
|
|
559
|
+
params: dict = Depends(build_query_params),
|
|
560
|
+
user: User = Depends(get_user),
|
|
561
|
+
db: Connection = Depends(get_db_connection),
|
|
562
|
+
) -> None:
|
|
563
|
+
await db.execute("UPDATE users SET ...", params)
|
|
503
564
|
```
|
|
504
565
|
"""
|
|
505
566
|
return cast(R, _Depends(dependency))
|
docket/docket.py
CHANGED
|
@@ -17,7 +17,6 @@ from typing import (
|
|
|
17
17
|
NoReturn,
|
|
18
18
|
ParamSpec,
|
|
19
19
|
Protocol,
|
|
20
|
-
Self,
|
|
21
20
|
Sequence,
|
|
22
21
|
TypedDict,
|
|
23
22
|
TypeVar,
|
|
@@ -25,6 +24,8 @@ from typing import (
|
|
|
25
24
|
overload,
|
|
26
25
|
)
|
|
27
26
|
|
|
27
|
+
from typing_extensions import Self
|
|
28
|
+
|
|
28
29
|
import redis.exceptions
|
|
29
30
|
from opentelemetry import propagate, trace
|
|
30
31
|
from redis.asyncio import ConnectionPool, Redis
|
|
@@ -156,12 +157,13 @@ class Docket:
|
|
|
156
157
|
"""
|
|
157
158
|
Args:
|
|
158
159
|
name: The name of the docket.
|
|
159
|
-
url: The URL of the Redis server. For example:
|
|
160
|
+
url: The URL of the Redis server or in-memory backend. For example:
|
|
160
161
|
- "redis://localhost:6379/0"
|
|
161
162
|
- "redis://user:password@localhost:6379/0"
|
|
162
163
|
- "redis://user:password@localhost:6379/0?ssl=true"
|
|
163
164
|
- "rediss://localhost:6379/0"
|
|
164
165
|
- "unix:///path/to/redis.sock"
|
|
166
|
+
- "memory://" (in-memory backend for testing)
|
|
165
167
|
heartbeat_interval: How often workers send heartbeat messages to the docket.
|
|
166
168
|
missed_heartbeats: How many heartbeats a worker can miss before it is
|
|
167
169
|
considered dead.
|
|
@@ -183,7 +185,30 @@ class Docket:
|
|
|
183
185
|
self.tasks = {fn.__name__: fn for fn in standard_tasks}
|
|
184
186
|
self.strike_list = StrikeList()
|
|
185
187
|
|
|
186
|
-
|
|
188
|
+
# Check if we should use in-memory backend (fakeredis)
|
|
189
|
+
# Support memory:// URLs for in-memory dockets
|
|
190
|
+
if self.url.startswith("memory://"):
|
|
191
|
+
try:
|
|
192
|
+
from fakeredis.aioredis import FakeConnection, FakeServer
|
|
193
|
+
|
|
194
|
+
# All memory:// URLs share a single FakeServer instance
|
|
195
|
+
# Multiple dockets with different names are isolated by Redis key prefixes
|
|
196
|
+
# (e.g., docket1:stream vs docket2:stream)
|
|
197
|
+
if not hasattr(Docket, "_memory_server"):
|
|
198
|
+
Docket._memory_server = FakeServer() # type: ignore
|
|
199
|
+
|
|
200
|
+
server = Docket._memory_server # type: ignore
|
|
201
|
+
self._connection_pool = ConnectionPool(
|
|
202
|
+
connection_class=FakeConnection, server=server
|
|
203
|
+
)
|
|
204
|
+
except ImportError as e:
|
|
205
|
+
raise ImportError(
|
|
206
|
+
"fakeredis is required for memory:// URLs. "
|
|
207
|
+
"Install with: pip install pydocket[memory]"
|
|
208
|
+
) from e
|
|
209
|
+
else:
|
|
210
|
+
self._connection_pool = ConnectionPool.from_url(self.url) # type: ignore
|
|
211
|
+
|
|
187
212
|
self._monitor_strikes_task = asyncio.create_task(self._monitor_strikes())
|
|
188
213
|
|
|
189
214
|
# Ensure that the stream and worker group exist
|
docket/execution.py
CHANGED
|
@@ -3,16 +3,9 @@ import enum
|
|
|
3
3
|
import inspect
|
|
4
4
|
import logging
|
|
5
5
|
from datetime import datetime
|
|
6
|
-
from typing import
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
Callable,
|
|
10
|
-
Hashable,
|
|
11
|
-
Literal,
|
|
12
|
-
Mapping,
|
|
13
|
-
Self,
|
|
14
|
-
cast,
|
|
15
|
-
)
|
|
6
|
+
from typing import Any, Awaitable, Callable, Hashable, Literal, Mapping, cast
|
|
7
|
+
|
|
8
|
+
from typing_extensions import Self
|
|
16
9
|
|
|
17
10
|
import cloudpickle # type: ignore[import]
|
|
18
11
|
import opentelemetry.context
|
|
@@ -35,6 +28,12 @@ def get_signature(function: Callable[..., Any]) -> inspect.Signature:
|
|
|
35
28
|
CACHE_SIZE.set(len(_signature_cache), {"cache": "signature"})
|
|
36
29
|
return _signature_cache[function]
|
|
37
30
|
|
|
31
|
+
signature_attr = getattr(function, "__signature__", None)
|
|
32
|
+
if isinstance(signature_attr, inspect.Signature):
|
|
33
|
+
_signature_cache[function] = signature_attr
|
|
34
|
+
CACHE_SIZE.set(len(_signature_cache), {"cache": "signature"})
|
|
35
|
+
return signature_attr
|
|
36
|
+
|
|
38
37
|
signature = inspect.signature(function)
|
|
39
38
|
_signature_cache[function] = signature
|
|
40
39
|
CACHE_SIZE.set(len(_signature_cache), {"cache": "signature"})
|
|
@@ -161,7 +160,7 @@ def compact_signature(signature: inspect.Signature) -> str:
|
|
|
161
160
|
return ", ".join(parameters)
|
|
162
161
|
|
|
163
162
|
|
|
164
|
-
class Operator(enum.
|
|
163
|
+
class Operator(str, enum.Enum):
|
|
165
164
|
EQUAL = "=="
|
|
166
165
|
NOT_EQUAL = "!="
|
|
167
166
|
GREATER_THAN = ">"
|
docket/instrumentation.py
CHANGED
|
@@ -193,7 +193,14 @@ def metrics_server(
|
|
|
193
193
|
yield
|
|
194
194
|
return
|
|
195
195
|
|
|
196
|
-
|
|
196
|
+
import sys
|
|
197
|
+
from typing import Any
|
|
198
|
+
|
|
199
|
+
# wsgiref.types was added in Python 3.11
|
|
200
|
+
if sys.version_info >= (3, 11): # pragma: no cover
|
|
201
|
+
from wsgiref.types import WSGIApplication
|
|
202
|
+
else: # pragma: no cover
|
|
203
|
+
WSGIApplication = Any # type: ignore[misc,assignment]
|
|
197
204
|
|
|
198
205
|
from prometheus_client import REGISTRY
|
|
199
206
|
from prometheus_client.exposition import (
|
docket/tasks.py
CHANGED
|
@@ -16,7 +16,7 @@ from .worker import Worker
|
|
|
16
16
|
|
|
17
17
|
async def trace(
|
|
18
18
|
message: str,
|
|
19
|
-
logger: logging.LoggerAdapter[logging.Logger] = TaskLogger(),
|
|
19
|
+
logger: "logging.LoggerAdapter[logging.Logger]" = TaskLogger(),
|
|
20
20
|
docket: Docket = CurrentDocket(),
|
|
21
21
|
worker: Worker = CurrentWorker(),
|
|
22
22
|
execution: Execution = CurrentExecution(),
|
|
@@ -46,7 +46,7 @@ async def fail(
|
|
|
46
46
|
|
|
47
47
|
|
|
48
48
|
async def sleep(
|
|
49
|
-
seconds: float, logger: logging.LoggerAdapter[logging.Logger] = TaskLogger()
|
|
49
|
+
seconds: float, logger: "logging.LoggerAdapter[logging.Logger]" = TaskLogger()
|
|
50
50
|
) -> None:
|
|
51
51
|
logger.info("Sleeping for %s seconds", seconds)
|
|
52
52
|
await asyncio.sleep(seconds)
|
docket/worker.py
CHANGED
|
@@ -6,13 +6,12 @@ import sys
|
|
|
6
6
|
import time
|
|
7
7
|
from datetime import datetime, timedelta, timezone
|
|
8
8
|
from types import TracebackType
|
|
9
|
-
from typing import
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
)
|
|
9
|
+
from typing import Coroutine, Mapping, Protocol, cast
|
|
10
|
+
|
|
11
|
+
if sys.version_info < (3, 11): # pragma: no cover
|
|
12
|
+
from exceptiongroup import ExceptionGroup
|
|
13
|
+
|
|
14
|
+
from typing_extensions import Self
|
|
16
15
|
|
|
17
16
|
from opentelemetry import trace
|
|
18
17
|
from opentelemetry.trace import Status, StatusCode, Tracer
|
|
@@ -167,16 +166,18 @@ class Worker:
|
|
|
167
166
|
for task_path in tasks:
|
|
168
167
|
docket.register_collection(task_path)
|
|
169
168
|
|
|
170
|
-
async with
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
169
|
+
async with (
|
|
170
|
+
Worker( # pragma: no branch - context manager exit varies across interpreters
|
|
171
|
+
docket=docket,
|
|
172
|
+
name=name,
|
|
173
|
+
concurrency=concurrency,
|
|
174
|
+
redelivery_timeout=redelivery_timeout,
|
|
175
|
+
reconnection_delay=reconnection_delay,
|
|
176
|
+
minimum_check_interval=minimum_check_interval,
|
|
177
|
+
scheduling_resolution=scheduling_resolution,
|
|
178
|
+
schedule_automatic_tasks=schedule_automatic_tasks,
|
|
179
|
+
) as worker
|
|
180
|
+
):
|
|
180
181
|
if until_finished:
|
|
181
182
|
await worker.run_until_finished()
|
|
182
183
|
else:
|
|
@@ -279,13 +280,22 @@ class Worker:
|
|
|
279
280
|
|
|
280
281
|
async def get_new_deliveries(redis: Redis) -> RedisReadGroupResponse:
|
|
281
282
|
logger.debug("Getting new deliveries", extra=log_context)
|
|
282
|
-
|
|
283
|
+
# Use non-blocking read with in-memory backend + manual sleep
|
|
284
|
+
# This is necessary because fakeredis's async blocking operations don't
|
|
285
|
+
# properly yield control to the asyncio event loop
|
|
286
|
+
is_memory = self.docket.url.startswith("memory://")
|
|
287
|
+
result = await redis.xreadgroup(
|
|
283
288
|
groupname=self.docket.worker_group_name,
|
|
284
289
|
consumername=self.name,
|
|
285
290
|
streams={self.docket.stream_key: ">"},
|
|
286
|
-
block=
|
|
291
|
+
block=0
|
|
292
|
+
if is_memory
|
|
293
|
+
else int(self.minimum_check_interval.total_seconds() * 1000),
|
|
287
294
|
count=available_slots,
|
|
288
295
|
)
|
|
296
|
+
if is_memory and not result:
|
|
297
|
+
await asyncio.sleep(self.minimum_check_interval.total_seconds())
|
|
298
|
+
return result
|
|
289
299
|
|
|
290
300
|
def start_task(
|
|
291
301
|
message_id: RedisMessageID,
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: pydocket
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.12.0
|
|
4
4
|
Summary: A distributed background task system for Python functions
|
|
5
5
|
Project-URL: Homepage, https://github.com/chrisguidry/docket
|
|
6
6
|
Project-URL: Bug Tracker, https://github.com/chrisguidry/docket/issues
|
|
@@ -19,11 +19,15 @@ Classifier: Development Status :: 4 - Beta
|
|
|
19
19
|
Classifier: License :: OSI Approved :: MIT License
|
|
20
20
|
Classifier: Operating System :: OS Independent
|
|
21
21
|
Classifier: Programming Language :: Python :: 3
|
|
22
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
23
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
22
24
|
Classifier: Programming Language :: Python :: 3.12
|
|
23
25
|
Classifier: Programming Language :: Python :: 3.13
|
|
26
|
+
Classifier: Programming Language :: Python :: 3.14
|
|
24
27
|
Classifier: Typing :: Typed
|
|
25
|
-
Requires-Python: >=3.
|
|
28
|
+
Requires-Python: >=3.10
|
|
26
29
|
Requires-Dist: cloudpickle>=3.1.1
|
|
30
|
+
Requires-Dist: exceptiongroup>=1.2.0; python_version < '3.11'
|
|
27
31
|
Requires-Dist: opentelemetry-api>=1.30.0
|
|
28
32
|
Requires-Dist: opentelemetry-exporter-prometheus>=0.51b0
|
|
29
33
|
Requires-Dist: prometheus-client>=0.21.1
|
|
@@ -31,6 +35,7 @@ Requires-Dist: python-json-logger>=3.2.1
|
|
|
31
35
|
Requires-Dist: redis>=4.6
|
|
32
36
|
Requires-Dist: rich>=13.9.4
|
|
33
37
|
Requires-Dist: typer>=0.15.1
|
|
38
|
+
Requires-Dist: typing-extensions>=4.12.0
|
|
34
39
|
Requires-Dist: uuid7>=0.1.0
|
|
35
40
|
Description-Content-Type: text/markdown
|
|
36
41
|
|
|
@@ -69,6 +74,7 @@ from docket import Docket, Worker
|
|
|
69
74
|
|
|
70
75
|
async with Docket() as docket:
|
|
71
76
|
async with Worker(docket) as worker:
|
|
77
|
+
worker.register(greet)
|
|
72
78
|
await worker.run_until_finished()
|
|
73
79
|
```
|
|
74
80
|
|
|
@@ -98,7 +104,7 @@ reference](https://chrisguidry.github.io/docket/api-reference/).
|
|
|
98
104
|
## Installing `docket`
|
|
99
105
|
|
|
100
106
|
Docket is [available on PyPI](https://pypi.org/project/pydocket/) under the package name
|
|
101
|
-
`pydocket`. It targets Python 3.
|
|
107
|
+
`pydocket`. It targets Python 3.10 or above.
|
|
102
108
|
|
|
103
109
|
With [`uv`](https://docs.astral.sh/uv/):
|
|
104
110
|
|
|
@@ -119,6 +125,18 @@ pip install pydocket
|
|
|
119
125
|
Docket requires a [Redis](http://redis.io/) server with Streams support (which was
|
|
120
126
|
introduced in Redis 5.0.0). Docket is tested with Redis 6 and 7.
|
|
121
127
|
|
|
128
|
+
For testing without Redis, Docket includes [fakeredis](https://github.com/cunla/fakeredis-py) for in-memory operation:
|
|
129
|
+
|
|
130
|
+
```python
|
|
131
|
+
from docket import Docket
|
|
132
|
+
|
|
133
|
+
async with Docket(name="my-docket", url="memory://my-docket") as docket:
|
|
134
|
+
# Use docket normally - all operations are in-memory
|
|
135
|
+
...
|
|
136
|
+
```
|
|
137
|
+
|
|
138
|
+
See [Testing with Docket](https://chrisguidry.github.io/docket/testing/#using-in-memory-backend-no-redis-required) for more details.
|
|
139
|
+
|
|
122
140
|
# Hacking on `docket`
|
|
123
141
|
|
|
124
142
|
We use [`uv`](https://docs.astral.sh/uv/) for project management, so getting set up
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
docket/__init__.py,sha256=ChJS2JRyruj22Vi504eXrmQNPQ97L_Sj52OJCuhjoeM,956
|
|
2
|
+
docket/__main__.py,sha256=wcCrL4PjG51r5wVKqJhcoJPTLfHW0wNbD31DrUN0MWI,28
|
|
3
|
+
docket/agenda.py,sha256=RqrVkCuWAvwn_q6graCU-lLRQltbJ0QQheJ34T-Gjck,6667
|
|
4
|
+
docket/annotations.py,sha256=enTC_b76hxEFbsas3236ihqA8C5Ijvff8PSb2WM8og0,2378
|
|
5
|
+
docket/cli.py,sha256=IyO5kfeQ2NSoBBzxKFKHuTCW-iO321mGVWePNR3tPFg,26800
|
|
6
|
+
docket/dependencies.py,sha256=aPL6dgncRim9Oogi8OfeBGjOabOpMtuaRGkGtdfbvRE,22692
|
|
7
|
+
docket/docket.py,sha256=vU858EENc6pT7cLWHSetLY0e2ObVO0Gr9j0dSyDEcpw,32408
|
|
8
|
+
docket/execution.py,sha256=9aUWNulNSdG2C54UFc-buKM06oiKPvBu4FmM_GVaZEA,15402
|
|
9
|
+
docket/instrumentation.py,sha256=k5yogK3GEGeU6zFWsOv3ENBS2li0jhVQ2BdON8Ov7uM,5908
|
|
10
|
+
docket/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
11
|
+
docket/tasks.py,sha256=FzgLNPS_Q370auMSdHmuQxGsHM_GbWuzT-Lja0zAJWs,1441
|
|
12
|
+
docket/worker.py,sha256=emQpdralGmXSNzWthFLjWzTLysAw-jDxWnnAffTyjOc,36127
|
|
13
|
+
pydocket-0.12.0.dist-info/METADATA,sha256=O-TtY_L93hX5m9ghYPoiBiiHpPLvWqXB2xaRfLb47vM,6140
|
|
14
|
+
pydocket-0.12.0.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
|
|
15
|
+
pydocket-0.12.0.dist-info/entry_points.txt,sha256=4WOk1nUlBsUT5O3RyMci2ImuC5XFswuopElYcLHtD5k,47
|
|
16
|
+
pydocket-0.12.0.dist-info/licenses/LICENSE,sha256=YuVWU_ZXO0K_k2FG8xWKe5RGxV24AhJKTvQmKfqXuyk,1087
|
|
17
|
+
pydocket-0.12.0.dist-info/RECORD,,
|
pydocket-0.11.1.dist-info/RECORD
DELETED
|
@@ -1,17 +0,0 @@
|
|
|
1
|
-
docket/__init__.py,sha256=ChJS2JRyruj22Vi504eXrmQNPQ97L_Sj52OJCuhjoeM,956
|
|
2
|
-
docket/__main__.py,sha256=wcCrL4PjG51r5wVKqJhcoJPTLfHW0wNbD31DrUN0MWI,28
|
|
3
|
-
docket/agenda.py,sha256=RqrVkCuWAvwn_q6graCU-lLRQltbJ0QQheJ34T-Gjck,6667
|
|
4
|
-
docket/annotations.py,sha256=wttix9UOeMFMAWXAIJUfUw5GjESJZsACb4YXJCozP7Q,2348
|
|
5
|
-
docket/cli.py,sha256=rTfri2--u4Q5PlXyh7Ub_F5uh3-TtZOWLUp9WY_TvAE,25750
|
|
6
|
-
docket/dependencies.py,sha256=BC0bnt10cr9_S1p5JAP_bnC9RwZkTr9ulPBrxC7eZnA,20247
|
|
7
|
-
docket/docket.py,sha256=NWyulaZYfcNeaqZSJMG54bHqTC5gVggzYFHjpTTY90A,31240
|
|
8
|
-
docket/execution.py,sha256=Lqzgj5EO3v5OD0w__5qBut7WnlEcHZfAYj-BYRdiJf8,15138
|
|
9
|
-
docket/instrumentation.py,sha256=zLYgtuXbNOcotcSlD9pgLVdNp2rPddyxj9JwM3K19Go,5667
|
|
10
|
-
docket/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
11
|
-
docket/tasks.py,sha256=RIlSM2omh-YDwVnCz6M5MtmK8T_m_s1w2OlRRxDUs6A,1437
|
|
12
|
-
docket/worker.py,sha256=P4j9uHXt5KcU5e9S4SmQ9v6OCRFMLjYwbMR9PeRvVXc,35390
|
|
13
|
-
pydocket-0.11.1.dist-info/METADATA,sha256=zl27q0Qf9js2bjyDHEjVWighIGWd4me36FLqY3yt5MI,5419
|
|
14
|
-
pydocket-0.11.1.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
|
|
15
|
-
pydocket-0.11.1.dist-info/entry_points.txt,sha256=4WOk1nUlBsUT5O3RyMci2ImuC5XFswuopElYcLHtD5k,47
|
|
16
|
-
pydocket-0.11.1.dist-info/licenses/LICENSE,sha256=YuVWU_ZXO0K_k2FG8xWKe5RGxV24AhJKTvQmKfqXuyk,1087
|
|
17
|
-
pydocket-0.11.1.dist-info/RECORD,,
|
|
File without changes
|
|
File without changes
|
|
File without changes
|