ttflow 0.5.8__py3-none-any.whl

ttflow/__init__.py

@@ -0,0 +1,15 @@
+from .core.context import Context
+from .core.run_context import RunContext
+from .core.workflow import WorkflowRunResult
+from .ttflow import Client, event_trigger, every_trigger, setup, state_trigger
+
+__all__ = [
+    "Client",
+    "Context",
+    "RunContext",
+    "WorkflowRunResult",
+    "event_trigger",
+    "every_trigger",
+    "setup",
+    "state_trigger",
+]

ttflow/constants.py

@@ -0,0 +1,8 @@
+# StateRepositoryで使用するシステムキー
+STATE_KEY_EVENTS = "_events"
+STATE_KEY_SYSTEM_LOCK = "_system_lock"
+STATE_KEY_WORKFLOWS_HASH = "workflows_hash"
+
+# ステートキーのプレフィックス
+STATE_PREFIX_LOGS = "_logs"
+STATE_PREFIX_RUN_STATE = "_run_state"

ttflow/core/__init__.py

@@ -0,0 +1 @@
+from .event import _enque_event  # noqa

ttflow/core/context.py

@@ -0,0 +1,28 @@
+import uuid
+
+
+class Context:
+    """
+    ワークフローの実行時に渡されるコンテキスト
+    """
+
+    def __init__(
+        self,
+        workflow_name: str,
+        run_id: str | None = None,
+        paused_info: dict | None = None,
+    ):
+        self.workflow_name = workflow_name
+        self.paused_info = (
+            paused_info  # このrunがpauseからの再開だった場合、そのpause情報が入る
+        )
+        if run_id is None:
+            run_id = uuid.uuid4().hex
+        self.run_id = run_id
+        self.used_count = 0
+
+    def _use(self):
+        self.used_count += 1
+
+    def get_run_state_token(self):
+        return f"{self.run_id}:{self.used_count}"

ttflow/core/event.py

@@ -0,0 +1,71 @@
+from collections.abc import Generator
+from dataclasses import asdict
+from typing import Any
+
+from dacite import from_dict
+
+from ..constants import STATE_KEY_EVENTS
+from .global_env import Event, Global
+
+# _global.eventsはオンメモリのイベントキューである
+# ttflowの一回の実行は、イベントキューが空になるまで続く
+
+
+def _enque_event(
+    g: Global,
+    event_name: str,
+    args: Any,
+    process_immediately: bool = True,
+) -> None:
+    """イベントをキューに追加する
+
+    Args:
+        process_immediately: Trueならこのrun中に処理される。Falseなら次回のrunで処理される。
+    """
+    e = Event(
+        event_name=event_name,
+        args=args,
+    )
+    if process_immediately:
+        g.events.append(e)
+    else:
+        g.events_for_next_run.append(e)
+
+
+# triggerは実態としては単にイベントである
+def _enque_trigger(g: Global, name: str, args: Any) -> None:
+    # triggerなので即時実行
+    _enque_event(g, f"_trigger_{name}", args, process_immediately=True)
+
+
+def load_events_from_state(g: Global) -> None:
+    es = _read_events_from_state(g)
+    g.events.extend(es)
+
+
+def _read_events_from_state(g: Global) -> list[Event]:
+    unprocessed_events = g.state.read_state(STATE_KEY_EVENTS, default=[])
+    return [from_dict(data_class=Event, data=e) for e in unprocessed_events]
+
+
+def flush_events_for_next_run_to_state(g: Global) -> None:
+    es = [asdict(e) for e in g.events_for_next_run]
+    g.state.save_state(STATE_KEY_EVENTS, es)
+
+
+def _pop_event(
+    g: Global,
+) -> Event | None:
+    if len(g.events) == 0:
+        return None
+    return g.events.pop(0)
+
+
+def iterate_events(
+    g: Global,
+) -> Generator[Event, None, None]:
+    while True:
+        e = _pop_event(g)
+        if e is None:
+            break
+        yield e

ttflow/core/global_env.py

@@ -0,0 +1,44 @@
+from collections.abc import Callable
+from dataclasses import dataclass
+from typing import Any
+
+from ..state_repository.buffer_cache_proxy import BufferCacheStateRepositoryProxy
+from .trigger import Trigger
+
+
+@dataclass
+class Event:
+    event_name: str
+    args: Any
+
+
+class Workflow:
+    def __init__(self, trigger: Trigger, f: Callable):
+        self.trigger = trigger
+        self.f = f
+
+    @property
+    def name(self) -> str:
+        return self.f.__name__
+
+    @property
+    def description(self) -> str | None:
+        return self.f.__doc__
+
+
+class Global:
+    def __init__(self, state: BufferCacheStateRepositoryProxy):
+        self.state = state
+
+        # 登録されたワークフロー
+        self.workflows: list[Workflow] = []
+
+        # イベントキュー
+        self.events: list[Event] = []
+
+        # 実行中にエンキューされたイベント。
+        self.events_for_next_run: list[Event] = []
+
+    def purge_events(self) -> None:
+        self.events = []
+        self.events_for_next_run = []

ttflow/core/pause.py

@@ -0,0 +1,46 @@
+from ..system_states.event_log import _get_event_logs
+from ..system_states.run_state import _is_already_executed, _mark_as_executed
+from .context import Context
+from .global_env import Global
+
+# ワークフローの中断
+# ワークフローは中断時にPauseExceptionを投げる。
+# その場合、stateに中断情報が保存される。
+# 中断ワークフローが再実行される場合、run_idが同じになるので
+# run_idに対して処理が冪等になっていれば、何度中断しても問題ない。
+
+
+class PauseException(Exception):
+    def __init__(self, pause_id: str):
+        super().__init__()
+        self.pause_id = pause_id
+
+
+def _wait_event(g: Global, c: Context, event_name: str) -> None:
+    c._use()
+    pause_id = f"{c.run_id}:{c.used_count}"
+
+    # 初回なので中断情報を保存する
+    if c.paused_info is None:
+        raise PauseException(pause_id)
+
+    # 既にpaused、先に進むか判断する
+    event_log = _get_event_logs(g)
+    target_events = [
+        a
+        for a in event_log
+        if a["event_name"] == event_name and a["timestamp"] > c.paused_info["timestamp"]
+    ]
+    if len(target_events) == 0:
+        raise PauseException(pause_id)
+
+
+def _pause_once(g: Global, c: Context) -> None:
+    """一度だけ中断します。次回無条件で再開します"""
+    cache = _is_already_executed(g, c)
+    if cache is not None:
+        return cache.value
+    # fを実行する前に計算しておく必要がある。変わってしまうので
+    token = c.get_run_state_token()
+    _mark_as_executed(g, c, token, None)
+    raise PauseException(c.get_run_state_token())

ttflow/core/run_context.py

@@ -0,0 +1,48 @@
+from typing import Any
+
+from ..system_states.logs import log
+from .context import Context
+from .event import _enque_event
+from .global_env import Global
+from .pause import _pause_once, _wait_event
+from .state import add_list_state, get_state, set_state
+
+
+class RunContext:
+    def __init__(self, _global: Global, _context: Context):
+        self._global = _global
+        self._context = _context
+
+    def get_context_data(self) -> Context:
+        return self._context
+
+    def get_state(self, state_name: str, default: Any = None) -> Any:
+        return get_state(self._global, self._context, state_name, default)
+
+    def set_state(self, state_name: str, value: Any) -> None:
+        return set_state(self._global, self._context, state_name, value)
+
+    def add_list_state(
+        self,
+        state_name: str,
+        value: Any,
+        max_length: int | None = None,
+    ) -> None:
+        return add_list_state(
+            self._global, self._context, state_name, value, max_length=max_length
+        )
+
+    def log(self, message: str) -> None:
+        return log(self._global, self._context, message)
+
+    def wait_event(self, event_name: str) -> None:
+        """指定したイベントが発行されるまで中断します"""
+        _wait_event(self._global, self._context, event_name)
+
+    def pause_once(self) -> None:
+        """一度だけ中断します。次回無条件で再開します"""
+        _pause_once(self._global, self._context)
+
+    def event(self, name: str, args: Any) -> None:
+        """eventの発生をstateにキューイングします。次回のrun()で実行されます。"""
+        _enque_event(self._global, name, args)

ttflow/core/state.py

@@ -0,0 +1,66 @@
+from typing import Any
+
+from ..errors import InvalidStateError
+from ..system_states.run_state import _execute_once
+from .context import Context
+from .event import _enque_event
+from .global_env import Global
+
+
+# ステートを書き込む。再実行時は何もしない
+def set_state(g: Global, c: Context, state_name: str, value: Any) -> None:
+    @_execute_once(g, c)
+    def a():
+        # ステートを書き込み、変更があったら差分イベントを発行する
+        current_state = g.state.read_state(state_name)
+        g.state.save_state(state_name, value)
+        if current_state != value:
+            _enque_event(
+                g,
+                f"state_changed_{state_name}",
+                {"old": current_state, "new": value},
+            )
+
+    return a()
+
+
+def get_state(g: Global, c: Context, state_name: str, default: Any = None) -> Any:
+    """ステートを取得する。再実行時はキャッシュする"""
+
+    @_execute_once(g, c)
+    def a():
+        return g.state.read_state(state_name, default=default)
+
+    return a()
+
+
+def add_list_state(
+    g: Global,
+    c: Context,
+    state_name: str,
+    value: Any,
+    max_length: int | None = None,
+) -> None:
+    values = get_state(g, c, state_name, default=[])
+    if not isinstance(values, list):
+        raise InvalidStateError(f"state {state_name} is not list")
+    values = [a for a in values]
+    values.append(value)
+    if max_length is not None:
+        values = values[-max_length:]
+    set_state(g, c, state_name, values)
+
+
+def _add_list_state_raw(
+    g: Global,
+    state_name: str,
+    value: Any,
+    max_length: int | None = None,
+) -> None:
+    values = g.state.read_state(state_name, [])
+    if not isinstance(values, list):
+        raise InvalidStateError(f"state {state_name} is not list")
+    values.append(value)
+    if max_length is not None:
+        values = values[-max_length:]
+    g.state.save_state(state_name, values)

ttflow/core/system_events/every.py

@@ -0,0 +1,29 @@
+from dataclasses import dataclass
+
+from ..event import _enque_event
+from ..global_env import Global
+
+# _every イベントは、毎回一度自動的に発生するイベントです。
+# 毎回実行したいような場合に使用します
+
+SYSTEM_EVENT__EVERY = "_every"
+
+
+@dataclass
+class EveryEvent:
+    pass
+
+
+def _enque_every_event(g: Global) -> None:
+    _enque_event(
+        g,
+        event_name=SYSTEM_EVENT__EVERY,
+        args=None,
+        process_immediately=True,
+    )
+
+
+def try_parse_event__every(event_raw) -> EveryEvent | None:
+    if event_raw.event_name != SYSTEM_EVENT__EVERY:
+        return None
+    return EveryEvent()

ttflow/core/system_events/pause.py

@@ -0,0 +1,51 @@
+import time
+from dataclasses import asdict, dataclass
+from typing import Any
+
+from dacite import from_dict
+
+from ..event import Event, _enque_event
+from ..global_env import Global
+
+# _pause イベントは、ワークフローの中断を表すイベントです。
+# 中断されたワークフローは、即時ではなく次回の実行まで待機します。
+
+SYSTEM_EVENT_PAUSE = "_pause"
+
+
+@dataclass
+class PauseEvent:
+    workflow_name: str  # 実行してるワークフロー名
+    run_id: str  # run_id
+    pause_id: str  # 中断のID
+    args: Any  # 実行時の引数
+    timestamp: float  # 中断時刻
+
+
+def _enque_pause_event(
+    g: Global,
+    workflow_name: str,
+    run_id: str,
+    pause_id: str,
+    args: Any,
+) -> None:
+    _enque_event(
+        g,
+        event_name=SYSTEM_EVENT_PAUSE,
+        args=asdict(
+            PauseEvent(
+                workflow_name=workflow_name,
+                run_id=run_id,
+                pause_id=pause_id,
+                args=args,
+                timestamp=time.time(),
+            )
+        ),
+        process_immediately=False,
+    )
+
+
+def try_parse_pause_event(event_raw: Event) -> PauseEvent | None:
+    if event_raw.event_name != SYSTEM_EVENT_PAUSE:
+        return None
+    return from_dict(data_class=PauseEvent, data=event_raw.args)

ttflow/core/trigger.py

@@ -0,0 +1,9 @@
+class Trigger:
+    def __init__(self, trigger_type: str):
+        self.trigger_type = trigger_type
+
+
+class EventTrigger(Trigger):
+    def __init__(self, event_name: str):
+        super().__init__("event")
+        self.event_name = event_name

ttflow/core/workflow.py

@@ -0,0 +1,183 @@
+import asyncio
+import functools
+import inspect
+import logging
+from collections.abc import Callable, Generator
+from dataclasses import dataclass
+from typing import Any
+
+from ..errors import SideeffectUsageError, WorkflowDirectCallError
+from ..system_states.completed import add_completed_runs_log, add_failed_runs_log
+from ..system_states.logs import _get_logs
+from ..system_states.run_state import (
+    _delete_run_state,
+    _execute_once,
+    _execute_once_async,
+)
+from .context import Context
+from .global_env import Global, Workflow
+from .pause import PauseException
+from .run_context import RunContext
+from .system_events.pause import _enque_pause_event
+from .trigger import EventTrigger, Trigger
+
+logger = logging.getLogger(__name__)
+
+
+def _find_workflow(g: Global, workflow_name: str) -> Workflow | None:
+    for wf in g.workflows:
+        if wf.f.__name__ == workflow_name:
+            return wf
+    return None
+
+
+def _find_event_triggered_workflows(
+    g: Global, event_name: str
+) -> Generator[Workflow, None, None]:
+    for wf in g.workflows:
+        if isinstance(wf.trigger, EventTrigger) and wf.trigger.event_name == event_name:
+            yield wf
+
+
+@dataclass
+class WorkflowRunResult:
+    workflow_name: str
+    run_id: str
+    status: str  # succeeded, failed, paused
+    error_message: str | None
+    logs: list[str]
+
+
+def _call_workflow_func(wf: Workflow, g: Global, c: Context, args: Any):
+    """ワークフロー関数を呼び出す。asyncの場合はasyncio.run()で実行する"""
+    params = inspect.signature(wf.f).parameters
+    if inspect.iscoroutinefunction(wf.f):
+        # asyncワークフロー
+        if len(params) >= 2:
+            asyncio.run(wf.f(RunContext(g, c), args))
+        else:
+            asyncio.run(wf.f(RunContext(g, c)))
+    else:
+        # syncワークフロー
+        if len(params) >= 2:
+            wf.f(RunContext(g, c), args)
+        else:
+            wf.f(RunContext(g, c))
+
+
+def exec_workflow(g: Global, c: Context, wf: Workflow, args: Any) -> WorkflowRunResult:
+    """workflowを実行する
+    中断する場合、paused_workflowsステートにその状態を保存する
+    完了したらcompleted_runs_logステートに記録する
+    """
+
+    try:
+        _call_workflow_func(wf, g, c, args)
+    except PauseException as e:
+        logger.info("ワークフローを中断します")
+        _enque_pause_event(
+            g,
+            workflow_name=wf.f.__name__,
+            run_id=c.run_id,
+            pause_id=e.pause_id,
+            args=args,
+        )
+        print(f"{wf.f.__name__}: 中断します")
+        return WorkflowRunResult(
+            workflow_name=wf.f.__name__,
+            run_id=c.run_id,
+            status="paused",
+            error_message=None,
+            logs=_get_logs(g, c.run_id),
+        )
+    except Exception as e:
+        logger.error(f"ワークフローが失敗しました: {e}")
+        print(f"{wf.f.__name__}: error: {e}")
+        add_failed_runs_log(g, c)
+        return WorkflowRunResult(
+            workflow_name=wf.f.__name__,
+            run_id=c.run_id,
+            status="failed",
+            error_message=repr(e),
+            logs=_get_logs(g, c.run_id),
+        )
+    add_completed_runs_log(g, c)
+    _delete_run_state(g, c.run_id)
+    print(f"{wf.f.__name__}: 正常終了しました")
+    return WorkflowRunResult(
+        workflow_name=wf.f.__name__,
+        run_id=c.run_id,
+        status="succeeded",
+        error_message=None,
+        logs=_get_logs(g, c.run_id),
+    )
+
+
+def workflow(g: Global, trigger: Trigger | str | None = None) -> Callable:
+    def _decorator(f: Callable) -> Callable:
+        if trigger is None:
+            t = EventTrigger(f"_trigger_{f.__name__}")
+        elif isinstance(trigger, str):
+            t = EventTrigger(f"_trigger_{trigger}")
+        else:
+            t = trigger
+        wf = Workflow(t, f)
+        g.workflows.append(wf)
+
+        @functools.wraps(f)
+        def _wrapper(*args, **kwargs):
+            raise WorkflowDirectCallError("workflow can not be called directly")
+
+        return _wrapper
+
+    return _decorator
+
+
+def sideeffect(g: Global) -> Callable:
+    def _decorator(f: Callable) -> Callable:
+        if inspect.iscoroutinefunction(f):
+            # async sideeffect: syncワークフローからの呼び出しを検出するため、
+            # 通常の関数でラップし、実行中イベントループの有無で判定する
+            @functools.wraps(f)
+            def _async_guard(*args, **kwargs):
+                try:
+                    asyncio.get_running_loop()
+                except RuntimeError:
+                    raise SideeffectUsageError(
+                        "async sideeffectはasyncワークフローからのみ呼び出せます"
+                    )
+                return _async_impl(*args, **kwargs)
+
+            async def _async_impl(*args, **kwargs):
+                if len(args) == 0 or not isinstance(args[0], RunContext):
+                    raise SideeffectUsageError(
+                        "sideeffectはRunContextを第1引数に取る必要があります"
+                    )
+                c = args[0]
+
+                @_execute_once_async(g, c.get_context_data())
+                async def a():
+                    return await f(*args, **kwargs)
+
+                return await a()
+
+            return _async_guard
+        else:
+            # sync sideeffect
+            @functools.wraps(f)
+            def _wrapper(*args, **kwargs):
+                if len(args) == 0 or not isinstance(args[0], RunContext):
+                    raise SideeffectUsageError(
+                        "sideeffectはRunContextを第1引数に取る必要があります"
+                    )
+                c = args[0]
+
+                @_execute_once(g, c.get_context_data())
+                def a():
+                    return f(*args, **kwargs)
+
+                return a()
+
+            return _wrapper
+
+    return _decorator

ttflow/errors.py

@@ -0,0 +1,22 @@
+class TtflowError(Exception):
+    """ttflowの基底例外"""
+
+
+class StateLockedError(TtflowError):
+    """状態がロックされている"""
+
+
+class UnknownRepositoryError(TtflowError):
+    """不明なStateRepository指定"""
+
+
+class InvalidStateError(TtflowError):
+    """ステートの値が不正"""
+
+
+class WorkflowDirectCallError(TtflowError):
+    """ワークフロー関数を直接呼び出した"""
+
+
+class SideeffectUsageError(TtflowError):
+    """sideeffect関数の使い方が不正"""

ttflow/powerup/run_by_cli.py

@@ -0,0 +1,43 @@
+from typing import Any, Optional
+
+import fire
+
+from ttflow import Client, WorkflowRunResult
+
+
+def run(client: Client):
+    def _internal(trigger_name: Optional[str] = None, args: Any = None):
+        results = client.run(trigger_name, args)
+        _print_workflow_results(results)
+
+    return _internal
+
+
+def _print_workflow_results(results: list[WorkflowRunResult]):
+    print()
+    print("---------RUN SUMMARY---------")
+    print(f"{len(results)}件のワークフローが実行されました")
+    for i, result in enumerate(results):
+        print(f"\t{i + 1}件目")
+        print(f"\t  ワークフロー名: {result.workflow_name}")
+        print(f"\t  run_id: {result.run_id}")
+        print(f"\t  状態: {result.status}")
+        print("\t  ログ:")
+        for log in result.logs:
+            print(f"\t    - {log}")
+
+
+def clear_state(client: Client):
+    def _internal():
+        client._global.state.clear_state()
+
+    return _internal
+
+
+def run_by_cli(client: Client, *, enabled_dangerous_clear_state_command=False):
+    opts: dict[str, Any] = {
+        "run": run(client),
+    }
+    if enabled_dangerous_clear_state_command:
+        opts["clear_state"] = clear_state(client)
+    fire.Fire(opts)