mageflow 0.3.2__tar.gz → 0.3.3__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (89) hide show
  1. {mageflow-0.3.2 → mageflow-0.3.3}/PKG-INFO +2 -2
  2. {mageflow-0.3.2 → mageflow-0.3.3}/mageflow/callbacks.py +31 -7
  3. {mageflow-0.3.2 → mageflow-0.3.3}/mageflow/chain/workflows.py +0 -1
  4. {mageflow-0.3.2 → mageflow-0.3.3}/mageflow/clients/hatchet/adapter.py +31 -16
  5. {mageflow-0.3.2 → mageflow-0.3.3}/mageflow/clients/hatchet/mageflow.py +18 -12
  6. {mageflow-0.3.2 → mageflow-0.3.3}/mageflow/clients/hatchet/workflow.py +0 -1
  7. {mageflow-0.3.2 → mageflow-0.3.3}/mageflow/config.py +10 -6
  8. {mageflow-0.3.2 → mageflow-0.3.3}/mageflow/lifecycle/signature.py +0 -1
  9. {mageflow-0.3.2 → mageflow-0.3.3}/mageflow/lifecycle/task.py +0 -1
  10. {mageflow-0.3.2 → mageflow-0.3.3}/mageflow/swarm/workflows.py +0 -1
  11. {mageflow-0.3.2 → mageflow-0.3.3}/pyproject.toml +6 -2
  12. {mageflow-0.3.2 → mageflow-0.3.3}/tests/integration/hatchet/models.py +19 -1
  13. mageflow-0.3.3/tests/integration/hatchet/test_retry_cache.py +65 -0
  14. {mageflow-0.3.2 → mageflow-0.3.3}/tests/integration/hatchet/test_ttl.py +1 -1
  15. {mageflow-0.3.2 → mageflow-0.3.3}/tests/integration/hatchet/worker.py +55 -4
  16. {mageflow-0.3.2 → mageflow-0.3.3}/tests/unit/assertions.py +0 -1
  17. {mageflow-0.3.2 → mageflow-0.3.3}/tests/unit/callbacks/conftest.py +6 -4
  18. {mageflow-0.3.2 → mageflow-0.3.3}/tests/unit/callbacks/test_handle_task_callback.py +10 -17
  19. mageflow-0.3.3/tests/unit/callbacks/test_retry_cache.py +789 -0
  20. {mageflow-0.3.2 → mageflow-0.3.3}/tests/unit/clients/test_hatchet_adapter.py +79 -3
  21. {mageflow-0.3.2 → mageflow-0.3.3}/tests/unit/test_remove_ttl.py +17 -3
  22. {mageflow-0.3.2 → mageflow-0.3.3}/tests/unit/test_ttl_config.py +13 -10
  23. mageflow-0.3.3/tests/unit/test_ttl_validation.py +48 -0
  24. {mageflow-0.3.2 → mageflow-0.3.3}/tests/unit/workflows/test_chain_end.py +1 -4
  25. {mageflow-0.3.2 → mageflow-0.3.3}/tests/unit/workflows/test_swarm_item_failed.py +5 -16
  26. {mageflow-0.3.2 → mageflow-0.3.3}/.gitignore +0 -0
  27. {mageflow-0.3.2 → mageflow-0.3.3}/mageflow/__init__.py +5 -5
  28. {mageflow-0.3.2 → mageflow-0.3.3}/mageflow/chain/__init__.py +0 -0
  29. {mageflow-0.3.2 → mageflow-0.3.3}/mageflow/chain/messages.py +0 -0
  30. {mageflow-0.3.2 → mageflow-0.3.3}/mageflow/client.py +1 -1
  31. {mageflow-0.3.2 → mageflow-0.3.3}/mageflow/clients/__init__.py +0 -0
  32. {mageflow-0.3.2 → mageflow-0.3.3}/mageflow/clients/hatchet/__init__.py +0 -0
  33. {mageflow-0.3.2 → mageflow-0.3.3}/mageflow/clients/inner_task_names.py +0 -0
  34. {mageflow-0.3.2 → mageflow-0.3.3}/mageflow/lifecycle/__init__.py +0 -0
  35. {mageflow-0.3.2 → mageflow-0.3.3}/mageflow/startup.py +1 -1
  36. {mageflow-0.3.2 → mageflow-0.3.3}/mageflow/swarm/__init__.py +0 -0
  37. {mageflow-0.3.2 → mageflow-0.3.3}/mageflow/swarm/consts.py +0 -0
  38. {mageflow-0.3.2 → mageflow-0.3.3}/mageflow/swarm/messages.py +0 -0
  39. {mageflow-0.3.2 → mageflow-0.3.3}/mageflow/utils/__init__.py +0 -0
  40. {mageflow-0.3.2 → mageflow-0.3.3}/mageflow/utils/mageflow.py +0 -0
  41. {mageflow-0.3.2 → mageflow-0.3.3}/mageflow/utils/pythonic.py +0 -0
  42. {mageflow-0.3.2 → mageflow-0.3.3}/tests/__init__.py +0 -0
  43. {mageflow-0.3.2 → mageflow-0.3.3}/tests/integration/__init__.py +0 -0
  44. {mageflow-0.3.2 → mageflow-0.3.3}/tests/integration/conftest.py +0 -0
  45. {mageflow-0.3.2 → mageflow-0.3.3}/tests/integration/hatchet/__init__.py +0 -0
  46. {mageflow-0.3.2 → mageflow-0.3.3}/tests/integration/hatchet/assertions.py +3 -3
  47. {mageflow-0.3.2 → mageflow-0.3.3}/tests/integration/hatchet/chain/__init__.py +0 -0
  48. {mageflow-0.3.2 → mageflow-0.3.3}/tests/integration/hatchet/chain/test__chain.py +1 -1
  49. {mageflow-0.3.2 → mageflow-0.3.3}/tests/integration/hatchet/chain/test_edge_cases.py +0 -0
  50. {mageflow-0.3.2 → mageflow-0.3.3}/tests/integration/hatchet/chain/test_stop_resume.py +1 -1
  51. {mageflow-0.3.2 → mageflow-0.3.3}/tests/integration/hatchet/conftest.py +1 -1
  52. {mageflow-0.3.2 → mageflow-0.3.3}/tests/integration/hatchet/docker-compose.hatchet.yml +0 -0
  53. {mageflow-0.3.2 → mageflow-0.3.3}/tests/integration/hatchet/signature/__init__.py +0 -0
  54. {mageflow-0.3.2 → mageflow-0.3.3}/tests/integration/hatchet/signature/test__signature.py +2 -2
  55. {mageflow-0.3.2 → mageflow-0.3.3}/tests/integration/hatchet/signature/test_edge_case.py +2 -2
  56. {mageflow-0.3.2 → mageflow-0.3.3}/tests/integration/hatchet/signature/test_stop_resume.py +1 -1
  57. {mageflow-0.3.2 → mageflow-0.3.3}/tests/integration/hatchet/swarm/__init__.py +0 -0
  58. {mageflow-0.3.2 → mageflow-0.3.3}/tests/integration/hatchet/swarm/test__swarm.py +2 -2
  59. {mageflow-0.3.2 → mageflow-0.3.3}/tests/integration/hatchet/swarm/test_edge_cases.py +2 -2
  60. {mageflow-0.3.2 → mageflow-0.3.3}/tests/integration/hatchet/swarm/test_stop_resume.py +2 -2
  61. {mageflow-0.3.2 → mageflow-0.3.3}/tests/integration/hatchet/swarm/test_workflow.py +0 -0
  62. {mageflow-0.3.2 → mageflow-0.3.3}/tests/integration/hatchet/test_complex_scenarios.py +3 -3
  63. {mageflow-0.3.2 → mageflow-0.3.3}/tests/integration/hatchet/test_task_models.py +1 -1
  64. {mageflow-0.3.2 → mageflow-0.3.3}/tests/integration/test_redis_ttl.py +1 -1
  65. {mageflow-0.3.2 → mageflow-0.3.3}/tests/unit/__init__.py +0 -0
  66. {mageflow-0.3.2 → mageflow-0.3.3}/tests/unit/callbacks/__init__.py +0 -0
  67. {mageflow-0.3.2 → mageflow-0.3.3}/tests/unit/clients/__init__.py +0 -0
  68. {mageflow-0.3.2 → mageflow-0.3.3}/tests/unit/conftest.py +6 -6
  69. {mageflow-0.3.2 → mageflow-0.3.3}/tests/unit/idempotency/__init__.py +0 -0
  70. {mageflow-0.3.2 → mageflow-0.3.3}/tests/unit/idempotency/conftest.py +4 -4
  71. {mageflow-0.3.2 → mageflow-0.3.3}/tests/unit/idempotency/test_chain_workflows_idempotent.py +3 -3
  72. {mageflow-0.3.2 → mageflow-0.3.3}/tests/unit/idempotency/test_fill_running_tasks_idempotent.py +3 -3
  73. {mageflow-0.3.2 → mageflow-0.3.3}/tests/unit/idempotency/test_fill_workflow_running_tasks_idempotent.py +3 -3
  74. {mageflow-0.3.2 → mageflow-0.3.3}/tests/unit/idempotency/test_swarm_item_done_idempotent.py +1 -1
  75. {mageflow-0.3.2 → mageflow-0.3.3}/tests/unit/idempotency/test_swarm_item_failed_idempotent.py +1 -1
  76. {mageflow-0.3.2 → mageflow-0.3.3}/tests/unit/race_condition/__init__.py +0 -0
  77. {mageflow-0.3.2 → mageflow-0.3.3}/tests/unit/race_condition/test_fill_running_tasks.py +1 -1
  78. {mageflow-0.3.2 → mageflow-0.3.3}/tests/unit/test_client.py +0 -0
  79. {mageflow-0.3.2 → mageflow-0.3.3}/tests/unit/test_client_signature_compatibility.py +0 -0
  80. {mageflow-0.3.2 → mageflow-0.3.3}/tests/unit/test_param_config.py +0 -0
  81. {mageflow-0.3.2 → mageflow-0.3.3}/tests/unit/utils.py +0 -0
  82. {mageflow-0.3.2 → mageflow-0.3.3}/tests/unit/workflows/__init__.py +0 -0
  83. {mageflow-0.3.2 → mageflow-0.3.3}/tests/unit/workflows/conftest.py +4 -4
  84. {mageflow-0.3.2 → mageflow-0.3.3}/tests/unit/workflows/test_chain_error.py +0 -0
  85. {mageflow-0.3.2 → mageflow-0.3.3}/tests/unit/workflows/test_fill_running_tasks.py +3 -3
  86. {mageflow-0.3.2 → mageflow-0.3.3}/tests/unit/workflows/test_fill_swarm_corrupted_callbacks.py +1 -1
  87. {mageflow-0.3.2 → mageflow-0.3.3}/tests/unit/workflows/test_fill_swarm_running_tasks.py +2 -2
  88. {mageflow-0.3.2 → mageflow-0.3.3}/tests/unit/workflows/test_swarm_item_done.py +1 -1
  89. {mageflow-0.3.2 → mageflow-0.3.3}/tox.ini +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: mageflow
3
- Version: 0.3.2
3
+ Version: 0.3.3
4
4
  Summary: Manage Graph Execution Flow - A unified interface for task orchestration across different task managers
5
5
  Project-URL: Homepage, https://imaginary-cherry.github.io/mageflow/
6
6
  Project-URL: Documentation, https://imaginary-cherry.github.io/mageflow/
@@ -26,7 +26,7 @@ Classifier: Topic :: Software Development :: Libraries :: Python Modules
26
26
  Classifier: Topic :: System :: Distributed Computing
27
27
  Classifier: Typing :: Typed
28
28
  Requires-Python: <3.14,>=3.10
29
- Requires-Dist: thirdmagic<0.1.0,>=0.0.3
29
+ Requires-Dist: thirdmagic<0.1.0,>=0.0.4
30
30
  Provides-Extra: dev
31
31
  Requires-Dist: black>=26.1.0; extra == 'dev'
32
32
  Requires-Dist: coverage[toml]<8.0.0,>=7.0.0; extra == 'dev'
@@ -7,11 +7,16 @@ from typing import Any
7
7
  from hatchet_sdk import Context
8
8
  from hatchet_sdk.runnables.types import EmptyModel
9
9
  from pydantic import BaseModel
10
-
11
- from mageflow.utils.pythonic import flexible_call
10
+ from thirdmagic.signature.retry_cache import (
11
+ retry_cache_ctx,
12
+ setup_retry_cache,
13
+ teardown_retry_cache,
14
+ )
12
15
  from thirdmagic.task.model import TaskSignature
13
16
  from thirdmagic.task_def import MageflowTaskDefinition
14
17
 
18
+ from mageflow.utils.pythonic import flexible_call
19
+
15
20
 
16
21
  class AcceptParams(Enum):
17
22
  JUST_MESSAGE = 1
@@ -27,6 +32,7 @@ def handle_task_callback(
27
32
  expected_params: AcceptParams = AcceptParams.NO_CTX,
28
33
  wrap_res: bool = True,
29
34
  send_signature: bool = False,
35
+ is_idempotent: bool = False,
30
36
  ):
31
37
  def task_decorator(func):
32
38
  @functools.wraps(func)
@@ -40,8 +46,18 @@ def handle_task_callback(
40
46
  # NOTE: This should not run, the task should cancel, but just in case
41
47
  return {"Error": "Task should have been canceled"}
42
48
  is_normal_run = lifecycle.is_vanilla_run()
49
+ is_task_finish = False
43
50
  signature = await lifecycle.start_task()
44
51
 
52
+ # Setup retry cache for signature idempotency on retries (durable tasks only)
53
+ cache_token = None
54
+ cache_state = None
55
+ if is_idempotent:
56
+ cache_state = await setup_retry_cache(
57
+ ctx.workflow_id, ctx.attempt_number
58
+ )
59
+ cache_token = retry_cache_ctx.set(cache_state)
60
+
45
61
  # Add params if user requires
46
62
  if send_signature:
47
63
  kwargs["signature"] = signature
@@ -54,19 +70,22 @@ def handle_task_callback(
54
70
  else:
55
71
  result = await flexible_call(func, message, ctx, *args, **kwargs)
56
72
  except asyncio.CancelledError as e:
73
+ is_task_finish = True
57
74
  if not is_normal_run:
58
75
  await lifecycle.task_failed(msg_data, e)
59
76
  raise
60
77
  except Exception as e:
61
- if is_normal_run:
62
- raise
63
- if not TaskSignature.ClientAdapter.should_task_retry(
78
+ will_retry = TaskSignature.ClientAdapter.should_task_retry(
64
79
  task_model, ctx.attempt_number, e
65
- ):
66
- await lifecycle.task_failed(msg_data, e)
80
+ )
81
+ if not will_retry:
82
+ is_task_finish = True
83
+ if not is_normal_run:
84
+ await lifecycle.task_failed(msg_data, e)
67
85
  raise
68
86
  else:
69
87
  # If this is a simple task, no signature, then we dont do any manipulation
88
+ is_task_finish = True
70
89
  if is_normal_run:
71
90
  return result
72
91
  task_results = HatchetResult(hatchet_results=result)
@@ -76,6 +95,11 @@ def handle_task_callback(
76
95
  return task_results
77
96
  else:
78
97
  return result
98
+ finally:
99
+ if cache_token is not None:
100
+ retry_cache_ctx.reset(cache_token)
101
+ if is_task_finish and cache_state:
102
+ await teardown_retry_cache(cache_state)
79
103
 
80
104
  wrapper.__signature__ = inspect.signature(func)
81
105
  return wrapper
@@ -2,7 +2,6 @@ from logging import Logger
2
2
  from typing import Any
3
3
 
4
4
  from rapyer.fields import RapyerKey
5
-
6
5
  from thirdmagic.clients.lifecycle import BaseLifecycle
7
6
 
8
7
 
@@ -8,6 +8,13 @@ from hatchet_sdk.runnables.types import EmptyModel
8
8
  from hatchet_sdk.runnables.workflow import BaseWorkflow
9
9
  from pydantic import BaseModel, TypeAdapter
10
10
  from rapyer.fields import RapyerKey
11
+ from thirdmagic.chain import ChainTaskSignature
12
+ from thirdmagic.clients.base import BaseClientAdapter
13
+ from thirdmagic.consts import TASK_ID_PARAM_NAME
14
+ from thirdmagic.signature import Signature
15
+ from thirdmagic.swarm import SwarmTaskSignature
16
+ from thirdmagic.task import TaskSignature
17
+ from thirdmagic.task_def import MageflowTaskDefinition
11
18
 
12
19
  from mageflow.chain.messages import ChainCallbackMessage, ChainErrorMessage
13
20
  from mageflow.clients.hatchet.workflow import MageflowWorkflow
@@ -25,13 +32,6 @@ from mageflow.swarm.messages import (
25
32
  SwarmErrorMessage,
26
33
  SwarmResultsMessage,
27
34
  )
28
- from thirdmagic.chain import ChainTaskSignature
29
- from thirdmagic.clients.base import BaseClientAdapter
30
- from thirdmagic.consts import TASK_ID_PARAM_NAME
31
- from thirdmagic.signature import Signature
32
- from thirdmagic.swarm import SwarmTaskSignature
33
- from thirdmagic.task import TaskSignature
34
- from thirdmagic.task_def import MageflowTaskDefinition
35
35
 
36
36
 
37
37
  class HatchetClientAdapter(BaseClientAdapter):
@@ -123,6 +123,15 @@ class HatchetClientAdapter(BaseClientAdapter):
123
123
  def extract_retries(self, client_task: BaseWorkflow) -> int:
124
124
  return client_task.tasks[0].retries
125
125
 
126
+ def _prepare_wf(self, signature: TaskSignature, set_return_field: bool, **kwargs):
127
+ total_kwargs = signature.kwargs | kwargs
128
+ workflow = self.hatchet.workflow(
129
+ name=signature.task_name, input_validator=signature.model_validators
130
+ )
131
+ return_field_name = signature.return_field_name if set_return_field else None
132
+ mageflow_wf = MageflowWorkflow(workflow, total_kwargs, return_field_name)
133
+ return mageflow_wf
134
+
126
135
  async def acall_signature(
127
136
  self,
128
137
  signature: TaskSignature,
@@ -134,17 +143,23 @@ class HatchetClientAdapter(BaseClientAdapter):
134
143
  if msg is None:
135
144
  msg = EmptyModel()
136
145
  options = self._update_options(signature, options)
137
- total_kwargs = signature.kwargs | kwargs
138
- workflow = self.hatchet.workflow(
139
- name=signature.task_name, input_validator=signature.model_validators
140
- )
141
- mageflow_wf = MageflowWorkflow(
142
- workflow,
143
- total_kwargs,
144
- signature.return_field_name if set_return_field else None,
145
- )
146
+ mageflow_wf = self._prepare_wf(signature, set_return_field, **kwargs)
146
147
  return await mageflow_wf.aio_run_no_wait(msg, options)
147
148
 
149
+ async def await_signature(
150
+ self,
151
+ signature: "TaskSignature",
152
+ msg: Any,
153
+ set_return_field: bool,
154
+ options: TriggerWorkflowOptions = None,
155
+ **kwargs,
156
+ ):
157
+ if msg is None:
158
+ msg = EmptyModel()
159
+ options = self._update_options(signature, options)
160
+ mageflow_wf = self._prepare_wf(signature, set_return_field, **kwargs)
161
+ return await mageflow_wf.aio_run(msg, options)
162
+
148
163
  def should_task_retry(
149
164
  self,
150
165
  task_definition: MageflowTaskDefinition,
@@ -17,6 +17,14 @@ from hatchet_sdk.runnables.types import (
17
17
  from hatchet_sdk.runnables.workflow import BaseWorkflow, Standalone
18
18
  from hatchet_sdk.worker.worker import LifespanFn
19
19
  from redis.asyncio import Redis
20
+ from thirdmagic import chain, sign
21
+ from thirdmagic.chain import ChainTaskSignature
22
+ from thirdmagic.signature import Signature
23
+ from thirdmagic.swarm import SwarmTaskSignature
24
+ from thirdmagic.swarm.creator import SignatureOptions, swarm
25
+ from thirdmagic.task import TaskInputType, TaskSignature, TaskSignatureConvertible
26
+ from thirdmagic.task_def import MageflowTaskDefinition
27
+ from thirdmagic.utils import HatchetTaskType
20
28
  from typing_extensions import override
21
29
 
22
30
  from mageflow.callbacks import AcceptParams, handle_task_callback
@@ -47,14 +55,6 @@ from mageflow.swarm.workflows import (
47
55
  swarm_item_failed,
48
56
  )
49
57
  from mageflow.utils.mageflow import does_task_wants_ctx
50
- from thirdmagic import chain, sign
51
- from thirdmagic.chain import ChainTaskSignature
52
- from thirdmagic.signature import Signature
53
- from thirdmagic.swarm import SwarmTaskSignature
54
- from thirdmagic.swarm.creator import SignatureOptions, swarm
55
- from thirdmagic.task import TaskInputType, TaskSignature, TaskSignatureConvertible
56
- from thirdmagic.task_def import MageflowTaskDefinition
57
- from thirdmagic.utils import HatchetTaskType
58
58
 
59
59
  Duration = timedelta | str
60
60
 
@@ -125,14 +125,16 @@ class HatchetMageflow(Hatchet):
125
125
  )
126
126
  )
127
127
 
128
- def task_decorator(self, func: Callable, hatchet_task):
128
+ def task_decorator(self, func: Callable, hatchet_task, is_idempotent: bool = False):
129
129
  param_config = (
130
130
  AcceptParams.ALL
131
131
  if does_task_wants_ctx(func)
132
132
  else self.mageflow_config.param_config
133
133
  )
134
134
  send_signature = getattr(func, "__send_signature__", False)
135
- handler_dec = handle_task_callback(param_config, send_signature=send_signature)
135
+ handler_dec = handle_task_callback(
136
+ param_config, send_signature=send_signature, is_idempotent=is_idempotent
137
+ )
136
138
  func = handler_dec(func)
137
139
  wf = hatchet_task(func)
138
140
  self._add_task_def(wf)
@@ -146,7 +148,9 @@ class HatchetMageflow(Hatchet):
146
148
  """
147
149
  hatchet_task = super().task(name=name, **kwargs)
148
150
 
149
- decorator = functools.partial(self.task_decorator, hatchet_task=hatchet_task)
151
+ decorator = functools.partial(
152
+ self.task_decorator, hatchet_task=hatchet_task, is_idempotent=False
153
+ )
150
154
  return decorator
151
155
 
152
156
  @override
@@ -156,7 +160,9 @@ class HatchetMageflow(Hatchet):
156
160
  """
157
161
  hatchet_task = super().durable_task(name=name, **kwargs)
158
162
 
159
- decorator = functools.partial(self.task_decorator, hatchet_task=hatchet_task)
163
+ decorator = functools.partial(
164
+ self.task_decorator, hatchet_task=hatchet_task, is_idempotent=True
165
+ )
160
166
 
161
167
  return decorator
162
168
 
@@ -3,7 +3,6 @@ from typing import Any
3
3
  from hatchet_sdk.runnables.workflow import Workflow
4
4
  from hatchet_sdk.utils.typing import JSONSerializableMapping
5
5
  from pydantic import BaseModel
6
-
7
6
  from thirdmagic.utils import deep_merge
8
7
 
9
8
 
@@ -1,26 +1,29 @@
1
1
  import dataclasses
2
- from dataclasses import dataclass, field
2
+ from dataclasses import field
3
3
  from typing import Optional
4
4
 
5
- from mageflow.callbacks import AcceptParams
5
+ from pydantic import Field
6
+ from pydantic.dataclasses import dataclass
6
7
  from thirdmagic.chain import ChainTaskSignature
7
8
  from thirdmagic.consts import REMOVED_TASK_TTL
8
- from thirdmagic.signature import SignatureConfig
9
+ from thirdmagic.signature import Signature, SignatureConfig
9
10
  from thirdmagic.swarm import SwarmTaskSignature
10
11
  from thirdmagic.swarm.state import PublishState
11
12
  from thirdmagic.task import TaskSignature
12
13
 
14
+ from mageflow.callbacks import AcceptParams
15
+
13
16
 
14
17
  @dataclass
15
18
  class SignatureTTLConfig:
16
19
  active_ttl: Optional[int] = None # seconds, None = use general
17
- ttl_when_sign_done: Optional[int] = None
20
+ ttl_when_sign_done: Optional[int] = Field(default=None, ge=REMOVED_TASK_TTL)
18
21
 
19
22
 
20
23
  @dataclass
21
24
  class TTLConfig:
22
25
  active_ttl: int = 24 * 60 * 60 # general active TTL (default 24h)
23
- ttl_when_sign_done: int = REMOVED_TASK_TTL # general done TTL (default 5min)
26
+ ttl_when_sign_done: int = Field(default=REMOVED_TASK_TTL, ge=REMOVED_TASK_TTL)
24
27
  task: SignatureTTLConfig = field(default_factory=SignatureTTLConfig)
25
28
  chain: SignatureTTLConfig = field(default_factory=SignatureTTLConfig)
26
29
  swarm: SignatureTTLConfig = field(default_factory=SignatureTTLConfig)
@@ -44,4 +47,5 @@ def apply_ttl_config(ttl_config: TTLConfig):
44
47
  done_ttl = sig_config.ttl_when_sign_done or ttl_config.ttl_when_sign_done
45
48
 
46
49
  sig_type.Meta = dataclasses.replace(sig_type.Meta, ttl=active_ttl)
47
- sig_type.SignatureSettings = SignatureConfig(ttl_when_sign_done=done_ttl)
50
+ if issubclass(sig_type, Signature):
51
+ sig_type.SignatureSettings = SignatureConfig(ttl_when_sign_done=done_ttl)
@@ -2,7 +2,6 @@ import asyncio
2
2
  from typing import Any, Optional, cast
3
3
 
4
4
  import rapyer
5
-
6
5
  from thirdmagic.clients.lifecycle import BaseLifecycle
7
6
  from thirdmagic.container import ContainerTaskSignature
8
7
  from thirdmagic.signature import Signature
@@ -1,7 +1,6 @@
1
1
  from typing import Any
2
2
 
3
3
  from pydantic import BaseModel
4
-
5
4
  from thirdmagic.clients.lifecycle import BaseLifecycle
6
5
 
7
6
 
@@ -3,7 +3,6 @@ from typing import Any, Optional, cast
3
3
 
4
4
  import rapyer
5
5
  from rapyer.fields import RapyerKey
6
-
7
6
  from thirdmagic.clients.lifecycle import BaseLifecycle
8
7
  from thirdmagic.signature import Signature
9
8
  from thirdmagic.swarm import PublishState
@@ -1,6 +1,6 @@
1
1
  [project]
2
2
  name = "mageflow"
3
- version = "0.3.2"
3
+ version = "0.3.3"
4
4
  description = "Manage Graph Execution Flow - A unified interface for task orchestration across different task managers"
5
5
  authors = [
6
6
  {name = "imaginary-cherry", email = "yedidyakfir@gmail.com"}
@@ -41,7 +41,7 @@ classifiers = [
41
41
  "Operating System :: OS Independent",
42
42
  ]
43
43
  dependencies = [
44
- "thirdmagic>=0.0.3, <0.1.0",
44
+ "thirdmagic>=0.0.4, <0.1.0",
45
45
  ]
46
46
 
47
47
  [project.optional-dependencies]
@@ -103,3 +103,7 @@ exclude_lines = [
103
103
 
104
104
  [tool.coverage.html]
105
105
  directory = "htmlcov"
106
+
107
+ [tool.ruff]
108
+ select = ["I", "F401"]
109
+
@@ -1,7 +1,6 @@
1
1
  from typing import Any
2
2
 
3
3
  from pydantic import BaseModel, Field
4
-
5
4
  from thirdmagic.message import ReturnValue
6
5
 
7
6
 
@@ -50,6 +49,25 @@ class SignatureKeysResult(BaseModel):
50
49
  swarm_sub_task_keys: list[str]
51
50
  publish_state_key: str
52
51
 
52
+ def is_key_in_keys(self, key: str) -> bool:
53
+ if key in self.task_keys:
54
+ return True
55
+ if key in self.chain_sub_task_keys:
56
+ return True
57
+ if key in self.swarm_sub_task_keys:
58
+ return True
59
+ if key == self.publish_state_key:
60
+ return True
61
+ if key == self.chain_key:
62
+ return True
63
+ if key == self.swarm_key:
64
+ return True
65
+ return False
66
+
67
+
68
+ class SignatureKeyWithWF(SignatureKeysResult):
69
+ workflow_id: str
70
+
53
71
 
54
72
  class MageflowTestError(Exception):
55
73
  pass
@@ -0,0 +1,65 @@
1
+ import pytest
2
+ from thirdmagic.task import TaskSignature
3
+
4
+ from tests.integration.hatchet.conftest import HatchetInitData
5
+ from tests.integration.hatchet.models import ContextMessage, SignatureKeyWithWF
6
+ from tests.integration.hatchet.worker import retry_cache_durable_task
7
+
8
+
9
+ async def _assert_retry_cache_idempotency(results: SignatureKeyWithWF):
10
+ workflow_id = results.workflow_id
11
+
12
+ # Assert - get the keys stored by each attempt from Redis
13
+ attempt_1_raw = await TaskSignature.Meta.redis.json().get( # type: ignore[misc]
14
+ f"retry-cache-test:{workflow_id}:attempt-1"
15
+ )
16
+ attempt_2_raw = await TaskSignature.Meta.redis.json().get( # type: ignore[misc]
17
+ f"retry-cache-test:{workflow_id}:attempt-2"
18
+ )
19
+
20
+ assert attempt_1_raw is not None, "Attempt 1 keys not found in Redis"
21
+ assert attempt_2_raw is not None, "Attempt 2 keys not found in Redis"
22
+ assert (
23
+ attempt_1_raw == attempt_2_raw
24
+ ), "Keys shouldn't be different between attempts"
25
+
26
+ last_sign = results.task_keys[-1]
27
+ results.task_keys = results.task_keys[:-1]
28
+ assert not results.is_key_in_keys(last_sign), "Last signature should not be in keys"
29
+ assert results.model_dump(mode="json") == attempt_1_raw
30
+
31
+
32
+ @pytest.mark.asyncio(loop_scope="session")
33
+ async def test__retry_cache__durable_task_retry__no_duplicate_signatures(
34
+ hatchet_client_init: HatchetInitData,
35
+ test_ctx,
36
+ ctx_metadata,
37
+ trigger_options,
38
+ ):
39
+ # Arrange
40
+ hatchet = hatchet_client_init.hatchet
41
+
42
+ signature = await hatchet.asign(retry_cache_durable_task)
43
+ msg = ContextMessage(base_data=test_ctx)
44
+
45
+ # Act - trigger the durable task which will fail on attempt 1 and succeed on attempt 2
46
+ wf_res = await signature.aio_run(msg, options=trigger_options)
47
+
48
+ results = SignatureKeyWithWF(
49
+ **wf_res["retry_cache_durable_task"]["hatchet_results"]
50
+ )
51
+
52
+ await _assert_retry_cache_idempotency(results)
53
+
54
+
55
+ @pytest.mark.asyncio(loop_scope="session")
56
+ async def test__retry_cache__task_retry__no_duplicate_signatures(
57
+ test_ctx, ctx_metadata, trigger_options
58
+ ):
59
+ # Arrange
60
+ msg = ContextMessage(base_data=test_ctx)
61
+
62
+ # Act - trigger directly via hatchet aio_run (not through a signature)
63
+ results = await retry_cache_durable_task.aio_run(msg, options=trigger_options)
64
+
65
+ await _assert_retry_cache_idempotency(results)
@@ -1,7 +1,7 @@
1
1
  import pytest
2
2
 
3
3
  from tests.integration.hatchet.conftest import HatchetInitData
4
- from tests.integration.hatchet.models import ContextMessage, SignatureKeysResult
4
+ from tests.integration.hatchet.models import ContextMessage
5
5
  from tests.integration.hatchet.worker import (
6
6
  CHAIN_ACTIVE_TTL,
7
7
  SWARM_ACTIVE_TTL,
@@ -4,7 +4,7 @@ import logging
4
4
  import os
5
5
  from datetime import datetime, timedelta
6
6
 
7
- from thirdmagic.consts import TASK_ID_PARAM_NAME
7
+ from thirdmagic.consts import REMOVED_TASK_TTL, TASK_ID_PARAM_NAME
8
8
  from thirdmagic.task import TaskSignature
9
9
 
10
10
  # Start coverage if COVERAGE_PROCESS_START is set
@@ -32,6 +32,7 @@ from tests.integration.hatchet.models import (
32
32
  MessageWithMsgResults,
33
33
  MessageWithResult,
34
34
  SignatureKeysResult,
35
+ SignatureKeyWithWF,
35
36
  SleepTaskMessage,
36
37
  )
37
38
 
@@ -53,11 +54,11 @@ hatchet = Hatchet(debug=True, config=config_obj)
53
54
 
54
55
  # Per-type TTL configuration for tests
55
56
  TASK_ACTIVE_TTL = 600 # 10 minutes
56
- TASK_DONE_TTL = 60 # 1 minute
57
+ TASK_DONE_TTL = REMOVED_TASK_TTL + 60 # 1 minute
57
58
  CHAIN_ACTIVE_TTL = 900 # 15 minutes
58
- CHAIN_DONE_TTL = 90 # 1.5 minutes
59
+ CHAIN_DONE_TTL = REMOVED_TASK_TTL + 90 # 1.5 minutes
59
60
  SWARM_ACTIVE_TTL = 1200 # 20 minutes
60
- SWARM_DONE_TTL = 120 # 2 minutes
61
+ SWARM_DONE_TTL = REMOVED_TASK_TTL + 120 # 2 minutes
61
62
  MAX_DONE_TTL = max(TASK_DONE_TTL, CHAIN_DONE_TTL, SWARM_DONE_TTL)
62
63
 
63
64
  TEST_MAGEFLOW_CONFIG = MageflowConfig(
@@ -225,6 +226,55 @@ async def create_signatures_for_ttl_test(msg: ContextMessage) -> SignatureKeysRe
225
226
  )
226
227
 
227
228
 
229
+ @hatchet.durable_task(
230
+ name="retry_cache_durable_task",
231
+ input_validator=ContextMessage,
232
+ retries=3,
233
+ execution_timeout=timedelta(seconds=60),
234
+ )
235
+ @hatchet.with_ctx
236
+ async def retry_cache_durable_task(
237
+ msg: ContextMessage, ctx: Context
238
+ ) -> SignatureKeyWithWF:
239
+ # Create standalone task signatures
240
+ sig1 = await hatchet.asign(task1)
241
+ sig2 = await hatchet.asign(task2)
242
+
243
+ # Create a chain
244
+ chain_sub1 = await hatchet.asign(task1)
245
+ chain_sub2 = await hatchet.asign(task2)
246
+ chain_sig = await hatchet.achain([chain_sub1, chain_sub2])
247
+
248
+ # Create a swarm
249
+ swarm_sub1 = await hatchet.asign(task1)
250
+ swarm_sub2 = await hatchet.asign(task2)
251
+ swarm_sig = await hatchet.aswarm([swarm_sub1, swarm_sub2], is_swarm_closed=True)
252
+
253
+ # Collect all created signature keys
254
+ results = SignatureKeyWithWF(
255
+ task_keys=[sig1.key, sig2.key],
256
+ chain_key=chain_sig.key,
257
+ chain_sub_task_keys=[chain_sub1.key, chain_sub2.key],
258
+ swarm_key=swarm_sig.key,
259
+ swarm_sub_task_keys=[swarm_sub1.key, swarm_sub2.key],
260
+ publish_state_key=swarm_sig.publishing_state_id,
261
+ workflow_id=ctx.workflow_id,
262
+ )
263
+ all_keys = results.model_dump(mode="json")
264
+
265
+ # Store keys in Redis for test verification, keyed by attempt number
266
+ attempt_key = f"retry-cache-test:{ctx.workflow_id}:attempt-{ctx.attempt_number}"
267
+ await TaskSignature.Meta.redis.json().set(attempt_key, "$", all_keys) # type: ignore[misc]
268
+
269
+ if ctx.attempt_number == 1:
270
+ raise ValueError("Intentional first attempt failure for retry cache test")
271
+
272
+ new_task = await hatchet.asign(task2)
273
+ results.task_keys.append(new_task.key)
274
+
275
+ return results
276
+
277
+
228
278
  workflows = [
229
279
  task1,
230
280
  task2,
@@ -245,6 +295,7 @@ workflows = [
245
295
  cancel_retry,
246
296
  accept_msg_results,
247
297
  create_signatures_for_ttl_test,
298
+ retry_cache_durable_task,
248
299
  ]
249
300
 
250
301
 
@@ -2,7 +2,6 @@ from typing import Literal, TypeVar
2
2
 
3
3
  from rapyer.fields import RapyerKey
4
4
  from redis.asyncio import Redis
5
-
6
5
  from thirdmagic.consts import REMOVED_TASK_TTL
7
6
  from thirdmagic.task import TaskSignature
8
7
 
@@ -5,15 +5,15 @@ from unittest.mock import AsyncMock, MagicMock
5
5
 
6
6
  import pytest_asyncio
7
7
  from hatchet_sdk import Context
8
-
9
- import mageflow
10
- from mageflow.callbacks import AcceptParams, handle_task_callback
11
- from tests.integration.hatchet.models import ContextMessage
12
8
  from thirdmagic.consts import TASK_ID_PARAM_NAME
13
9
  from thirdmagic.signature.status import SignatureStatus
14
10
  from thirdmagic.task import TaskSignature
15
11
  from thirdmagic.task_def import MageflowTaskDefinition
16
12
 
13
+ import mageflow
14
+ from mageflow.callbacks import AcceptParams, handle_task_callback
15
+ from tests.integration.hatchet.models import ContextMessage
16
+
17
17
 
18
18
  @dataclass
19
19
  class MockContextConfig:
@@ -86,6 +86,7 @@ def handler_factory(
86
86
  send_signature: bool = False,
87
87
  return_value: Any = "success_result",
88
88
  raises: BaseException | None = None,
89
+ durable: bool = False,
89
90
  ):
90
91
  tracked_calls: list[CallTracker] = []
91
92
 
@@ -93,6 +94,7 @@ def handler_factory(
93
94
  expected_params=expected_params,
94
95
  wrap_res=wrap_res,
95
96
  send_signature=send_signature,
97
+ is_idempotent=durable,
96
98
  )
97
99
  async def handler(*args, **kwargs):
98
100
  tracked_calls.append(CallTracker(args=args, kwargs=kwargs))