dotflow 0.15.0.dev2__py3-none-any.whl → 0.15.0.dev3__py3-none-any.whl

dotflow/__init__.py

@@ -1,6 +1,6 @@
 """Dotflow __init__ module."""
 
-__version__ = "0.15.0.dev2"
+__version__ = "0.15.0.dev3"
 __description__ = "🎲 Dotflow turns an idea into flow!"
 
 from .core.action import Action as action

dotflow/abc/flow.py

@@ -4,6 +4,7 @@ from abc import ABC, abstractmethod
 from uuid import UUID
 
 from dotflow.core.task import Task
+from dotflow.core.types import TypeStatus
 
 
 class Flow(ABC):
@@ -40,3 +41,21 @@ class Flow(ABC):
     @abstractmethod
     def run(self) -> None:
         return None
+
+    def _try_restore_checkpoint(self, task: Task):
+        """Attempts to restore a checkpoint. Returns the Context if found, None otherwise."""
+        if not self.resume:
+            return None
+
+        context = task.config.storage.get(
+            key=task.config.storage.key(task=task)
+        )
+
+        if context.storage is None:
+            return None
+
+        task.status = TypeStatus.COMPLETED
+        task.current_context = context
+        self._flow_callback(task=task)
+
+        return context

dotflow/cli/commands/cloud.py

@@ -20,6 +20,11 @@ def _get_template_dir() -> Path | None:
     cloud_dir = target / settings.TEMPLATE_CLOUD_DIR
 
     if cloud_dir.exists():
+        run(
+            ["git", "-C", str(target), "pull", "--ff-only"],
+            capture_output=True,
+            text=True,
+        )
         return cloud_dir
 
     try:
@@ -133,7 +138,9 @@ class CloudGenerateCommand(Command):
 
         for filename in files:
             filepath = (output / filename).resolve()
-            if not str(filepath).startswith(str(output)):
+            try:
+                filepath.relative_to(output)
+            except ValueError:
                 print(
                     settings.ERROR_ALERT,
                     f"  Skipped (unsafe filename): {filename}",

dotflow/cli/commands/init.py

@@ -6,12 +6,10 @@ from rich import print  # type: ignore
 from dotflow.cli.command import Command
 from dotflow.settings import Settings as settings
 
-TEMPLATE_REPO = "https://github.com/dotflow-io/template"
-
 
 class InitCommand(Command):
     def setup(self):
-        cookiecutter(TEMPLATE_REPO)
+        cookiecutter(settings.TEMPLATE_REPO)
 
         print(
             settings.INFO_ALERT,

dotflow/cli/commands/storage.py

@@ -0,0 +1,55 @@
+"""Storage resolver for CLI commands"""
+
+from dotflow.providers import (
+    StorageDefault,
+    StorageFile,
+    StorageGCS,
+    StorageS3,
+)
+from dotflow.settings import Settings as settings
+
+
+class StorageResolver:
+    """Resolves a storage provider instance from CLI params."""
+
+    def __init__(self, params):
+        self.params = params
+
+    def resolve(self):
+        """Returns a storage instance or None if no --storage was specified."""
+        if not self.params.storage:
+            return None
+
+        resolvers = {
+            "default": self._resolve_local,
+            "file": self._resolve_local,
+            "s3": self._resolve_s3,
+            "gcs": self._resolve_gcs,
+        }
+
+        resolver = resolvers.get(self.params.storage)
+        if resolver is None:
+            return None
+
+        return resolver()
+
+    def _resolve_local(self):
+        storage_cls = {"default": StorageDefault, "file": StorageFile}
+        return storage_cls[self.params.storage](path=self.params.path)
+
+    def _resolve_s3(self):
+        self._require("bucket")
+        return StorageS3(bucket=self.params.bucket, region=self.params.region)
+
+    def _resolve_gcs(self):
+        self._require("bucket")
+        return StorageGCS(
+            bucket=self.params.bucket, project=self.params.gcp_project
+        )
+
+    def _require(self, arg):
+        if not getattr(self.params, arg, None):
+            raise SystemExit(
+                f"{settings.ERROR_ALERT} --storage {self.params.storage}"
+                f" requires --{arg}"
+            )

dotflow/cloud/gcp/services/apis.py

@@ -2,8 +2,6 @@
 
 from __future__ import annotations
 
-import contextlib
-
 from dotflow.core.exception import ModuleNotFound
 
 
@@ -33,5 +31,9 @@ class APIs:
         print("  Enabling APIs...")
         for api in self.REQUIRED:
             name = f"projects/{self._project}/services/{api}"
-            with contextlib.suppress(Exception):
+            try:
                 self._client.enable_service(name=name).result()
+            except Exception as err:
+                if "already enabled" in str(err).lower():
+                    continue
+                print(f"  Warning: Failed to enable {api}: {err}")

dotflow/cloud/github/actions_deployer.py

@@ -71,19 +71,11 @@ class ActionsDeployer(Deployer):
         print("  Pushing files...")
 
         project_dir = Path.cwd()
+        tracked = self._get_tracked_files(project_dir)
         count = 0
 
-        for filepath in sorted(project_dir.rglob("*")):
-            if not filepath.is_file():
-                continue
-
-            relative = str(filepath.relative_to(project_dir))
-
-            if relative.startswith(".") and not relative.startswith(".github"):
-                continue
-
-            if "__pycache__" in relative or ".egg-info" in relative:
-                continue
+        for relative in tracked:
+            filepath = project_dir / relative
 
             try:
                 content = filepath.read_text()
@@ -95,6 +87,17 @@ class ActionsDeployer(Deployer):
 
         print(f"  Pushed {count} files")
 
+    def _get_tracked_files(self, project_dir: Path) -> list[str]:
+        """Get files that git would track, respecting .gitignore."""
+        from git import Repo
+
+        repo = Repo(project_dir)
+        tracked = set(repo.git.ls_files("--cached").splitlines())
+        untracked = set(
+            repo.git.ls_files("--others", "--exclude-standard").splitlines()
+        )
+        return sorted(tracked | untracked)
+
     def _create_or_update_file(self, repo, path: str, content: str):
         """Create or update a single file in the repo."""
         try:

dotflow/core/action.py

@@ -1,14 +1,13 @@
 """Action module"""
 
 import asyncio
+import inspect
 from collections.abc import Callable
 from concurrent.futures import ThreadPoolExecutor
-from time import sleep
 from types import FunctionType
 
 from dotflow.core.context import Context
-from dotflow.core.exception import ExecutionWithClassError, TaskError
-from dotflow.core.types.status import TypeStatus
+from dotflow.core.exception import ExecutionWithClassError
 
 
 def is_execution_with_class_internal_error(error: Exception) -> bool:
@@ -22,7 +21,11 @@ def is_execution_with_class_internal_error(error: Exception) -> bool:
 
 
 class Action:
-    """
+    """Decorator for creating task steps.
+
+    Accepts configuration for retry, timeout, retry_delay, and backoff,
+    which are stored as attributes and consumed by TaskEngine during execution.
+
     Import:
         You can import the **action** decorator directly from dotflow:
 
@@ -43,7 +46,6 @@ class Action:
             def my_task():
                 print("task")
 
-
         With Timeout
 
             @action(timeout=60)
@@ -62,6 +64,10 @@ class Action:
             def my_task():
                 print("task")
 
+    Note:
+        Retry, timeout, and backoff are enforced by TaskEngine.execute_with_retry(),
+        not by Action itself. Action stores the configuration as attributes.
+
     Args:
         func (Callable):
 
@@ -103,13 +109,13 @@ class Action:
 
             if contexts:
                 return Context(
-                    storage=self._run_action(*args, task=task, **contexts),
+                    storage=self._run_action(*args, **contexts),
                     task_id=task.task_id,
                     workflow_id=task.workflow_id,
                 )
 
             return Context(
-                storage=self._run_action(*args, task=task),
+                storage=self._run_action(*args),
                 task_id=task.task_id,
                 workflow_id=task.workflow_id,
             )
@@ -123,92 +129,47 @@ class Action:
 
             if contexts:
                 return Context(
-                    storage=self._run_action(*_args, task=task, **contexts),
+                    storage=self._run_action(*_args, **contexts),
                     task_id=task.task_id,
                     workflow_id=task.workflow_id,
                 )
 
             return Context(
-                storage=self._run_action(*_args, task=task),
+                storage=self._run_action(*_args),
                 task_id=task.task_id,
                 workflow_id=task.workflow_id,
             )
 
+        action.retry = self.retry
+        action.timeout = self.timeout
+        action.retry_delay = self.retry_delay
+        action.backoff = self.backoff
+
         return action
 
-    def _run_action(self, *args, task=None, **kwargs):
-        current_delay = self.retry_delay
-
-        is_async = asyncio.iscoroutinefunction(self.func)
-
-        for attempt in range(1, self.retry + 1):
-            try:
-                if self.timeout:
-                    executor = ThreadPoolExecutor(max_workers=1)
-                    try:
-                        future = executor.submit(
-                            self._call_func,
-                            is_async,
-                            *args,
-                            **kwargs,
-                        )
-                        result = future.result(timeout=self.timeout)
-                    except TimeoutError:
-                        future.cancel()
-                        executor.shutdown(wait=False, cancel_futures=True)
-                        raise
-                    except Exception:
-                        executor.shutdown(wait=False)
-                        raise
-                    else:
-                        executor.shutdown(wait=False)
-                else:
-                    result = self._call_func(is_async, *args, **kwargs)
-
-                return result
-
-            except TimeoutError:
-                raise
-
-            except Exception as error:
-                last_exception = error
-
-                if is_execution_with_class_internal_error(
-                    error=last_exception
-                ):
-                    raise ExecutionWithClassError() from None
-
-                if attempt == self.retry:
-                    raise last_exception from None
-
-                if task is not None:
-                    task.retry_count += 1
-                    task.errors = TaskError(
-                        error=error,
-                        attempt=attempt,
-                    )
-                    task.status = TypeStatus.RETRY
-
-                sleep(current_delay)
-                if self.backoff:
-                    current_delay *= 2
+    def _run_action(self, *args, **kwargs):
+        is_async = inspect.iscoroutinefunction(self.func)
+
+        try:
+            return self._call_func(is_async, *args, **kwargs)
+        except Exception as error:
+            if is_execution_with_class_internal_error(error=error):
+                raise ExecutionWithClassError() from None
+            raise
 
     def _call_func(self, is_async, *args, **kwargs):
-        if is_async:
-            try:
-                loop = asyncio.get_running_loop()
-            except RuntimeError:
-                loop = None
-
-            if loop and loop.is_running():
-                import concurrent.futures
-
-                with concurrent.futures.ThreadPoolExecutor() as pool:
-                    return pool.submit(
-                        asyncio.run, self.func(*args, **kwargs)
-                    ).result()
+        if not is_async:
+            return self.func(*args, **kwargs)
+
+        try:
+            asyncio.get_running_loop()
+        except RuntimeError:
             return asyncio.run(self.func(*args, **kwargs))
-        return self.func(*args, **kwargs)
+
+        with ThreadPoolExecutor(max_workers=1) as pool:
+            return pool.submit(
+                asyncio.run, self.func(*args, **kwargs)
+            ).result()
 
     def _set_params(self):
         if isinstance(self.func, FunctionType):

dotflow/core/context.py

@@ -68,8 +68,14 @@ class Context(ContextInstance):
 
     @task_id.setter
     def task_id(self, value: int):
-        if isinstance(value, int):
-            self._task_id = value
+        if value is None:
+            self._task_id = None
+            return
+        if not isinstance(value, int):
+            raise TypeError(
+                f"task_id must be an int, got {type(value).__name__}: {value!r}"
+            )
+        self._task_id = value
 
     @property
     def workflow_id(self):
@@ -77,6 +83,9 @@ class Context(ContextInstance):
 
     @workflow_id.setter
     def workflow_id(self, value: UUID):
+        if value is None:
+            self._workflow_id = None
+            return
         if isinstance(value, str):
             try:
                 value = UUID(value)
@@ -84,8 +93,12 @@ class Context(ContextInstance):
                 raise ValueError(
                     f"Invalid workflow_id: '{value}' is not a valid UUID format."
                 ) from err
-        if isinstance(value, UUID):
-            self._workflow_id = value
+        if not isinstance(value, UUID):
+            raise TypeError(
+                f"workflow_id must be a UUID or UUID string, "
+                f"got {type(value).__name__}: {value!r}"
+            )
+        self._workflow_id = value
 
     @property
     def storage(self):

dotflow/core/engine.py

@@ -0,0 +1,277 @@
+"""TaskEngine module"""
+
+import re
+from collections.abc import Callable
+from concurrent.futures import ThreadPoolExecutor
+from contextlib import contextmanager
+from datetime import datetime
+from inspect import getsourcelines
+from time import sleep
+from types import FunctionType
+from uuid import UUID
+
+try:
+    from types import NoneType
+except ImportError:
+    NoneType = type(None)
+
+from dotflow.core.action import Action
+from dotflow.core.context import Context
+from dotflow.core.exception import ExecutionWithClassError, TaskError
+from dotflow.core.task import Task
+from dotflow.core.types import TypeStatus
+from dotflow.logging import logger
+
+
+class TaskEngine:
+    """Manages the execution lifecycle of a single task.
+
+    Separates the task lifecycle (status, duration, error handling)
+    from how tasks are executed (sequential, parallel, background).
+    """
+
+    VALID_OBJECTS = [
+        str,
+        int,
+        float,
+        complex,
+        dict,
+        list,
+        tuple,
+        set,
+        frozenset,
+        range,
+        bool,
+        FunctionType,
+        NoneType,
+        bytes,
+        bytearray,
+        memoryview,
+    ]
+
+    def __init__(
+        self,
+        task: Task,
+        workflow_id: UUID,
+        previous_context: Context = None,
+    ) -> None:
+        self.task = task
+        self.workflow_id = workflow_id
+        self.previous_context = previous_context
+        self._start_time = None
+
+    @contextmanager
+    def start(self):
+        """Prepares the task for execution and manages its lifecycle."""
+        self.task.workflow_id = self.workflow_id
+        self.task.previous_context = self.previous_context
+        self.task.config.tracer.start_task(task=self.task)
+        self.task.status = TypeStatus.IN_PROGRESS
+        self._start_time = datetime.now()
+
+        try:
+            yield self
+        except AssertionError as err:
+            raise err
+        except Exception as err:
+            self.task.errors = err
+            self.task.current_context = None
+            self.task.status = TypeStatus.FAILED
+        else:
+            if self.task.status in (
+                TypeStatus.IN_PROGRESS,
+                TypeStatus.RETRY,
+            ):
+                self.task.status = TypeStatus.COMPLETED
+        finally:
+            self.task.duration = (
+                datetime.now() - self._start_time
+            ).total_seconds()
+            self.task.config.tracer.end_task(task=self.task)
+
+    def execute(self):
+        """Executes the task function and returns the context."""
+        current_context = self.task.step(
+            initial_context=self.task.initial_context,
+            previous_context=self.task.previous_context,
+            task=self.task,
+        )
+
+        if type(current_context.storage) not in self.VALID_OBJECTS:
+            current_context = self._execution_with_class(
+                class_instance=current_context.storage
+            )
+
+        self.task.current_context = current_context
+        return current_context
+
+    def execute_with_retry(self):
+        """Executes the task with retry, timeout, and backoff managed by the engine.
+
+        Reads retry, timeout, retry_delay, and backoff from the task's step
+        (the @action decorator) and manages the full retry loop.
+        """
+        step = self.task.step
+        max_attempts = max(1, step.retry)
+        timeout = step.timeout
+        retry_delay = step.retry_delay
+        backoff = step.backoff
+        current_delay = retry_delay
+
+        for attempt in range(1, max_attempts + 1):
+            try:
+                if timeout:
+                    result = self._execute_with_timeout(timeout)
+                else:
+                    result = self._execute_single()
+
+                self.task.current_context = result
+                return result
+
+            except TimeoutError:
+                raise
+
+            except Exception as error:
+                if self._is_class_internal_error(error):
+                    raise
+
+                if attempt == max_attempts:
+                    raise
+
+                self.task.retry_count += 1
+                self.task.errors = TaskError(
+                    error=error,
+                    attempt=attempt,
+                )
+                self.task.status = TypeStatus.RETRY
+
+                sleep(current_delay)
+                if backoff:
+                    current_delay *= 2
+
+    def _execute_single(self):
+        """Executes the task function once, handling class-based steps."""
+        current_context = self.task.step(
+            initial_context=self.task.initial_context,
+            previous_context=self.task.previous_context,
+            task=self.task,
+        )
+
+        if type(current_context.storage) not in self.VALID_OBJECTS:
+            current_context = self._execution_with_class(
+                class_instance=current_context.storage
+            )
+
+        return current_context
+
+    def _execute_with_timeout(self, seconds: int):
+        """Executes the task function with a real timeout using ThreadPoolExecutor."""
+        executor = ThreadPoolExecutor(max_workers=1)
+        try:
+            future = executor.submit(self._execute_single)
+            return future.result(timeout=seconds)
+        except TimeoutError:
+            future.cancel()
+            raise
+        finally:
+            executor.shutdown(wait=False, cancel_futures=True)
+
+    @staticmethod
+    def _is_class_internal_error(error: Exception) -> bool:
+        """Checks if an error is an internal class execution error."""
+        if isinstance(error, ExecutionWithClassError):
+            return True
+        message = str(error)
+        patterns = [
+            "initial_context",
+            "previous_context",
+            "missing 1 required positional argument: 'self'",
+        ]
+        return any(pattern in message for pattern in patterns)
+
+    def _is_action(self, class_instance: Callable, func: Callable):
+        try:
+            return (
+                callable(getattr(class_instance, func))
+                and getattr(class_instance, func).__module__
+                == Action.__module__
+                and not func.startswith("__")
+            )
+        except AttributeError:
+            return False
+
+    def _execution_orderer(
+        self, callable_list: list[str], class_instance: Callable
+    ) -> tuple[int, Callable]:
+        ordered_list = []
+
+        try:
+            inside_code = getsourcelines(class_instance.__class__)[0]
+
+            for callable_name in callable_list:
+                pattern = re.compile(
+                    rf"\bdef\s+{re.escape(callable_name)}\s*\("
+                )
+                for index, code in enumerate(inside_code):
+                    if pattern.search(code):
+                        ordered_list.append((index, callable_name))
+                        break
+
+            ordered_list.sort()
+            return ordered_list
+
+        except TypeError as err:
+            logger.error(
+                "Internal problem with ordering the class functions, "
+                "but don't worry, it was executed.: %s",
+                str(err),
+            )
+
+        for index, callable_name in enumerate(callable_list):
+            ordered_list.append((index, callable_name))
+
+        return ordered_list
+
+    def _execution_with_class(self, class_instance: Callable):
+        new_context = Context(storage=[])
+        previous_context = self.task.previous_context
+        callable_list = [
+            func
+            for func in dir(class_instance)
+            if self._is_action(class_instance, func)
+        ]
+
+        ordered_list = self._execution_orderer(
+            callable_list=callable_list, class_instance=class_instance
+        )
+
+        for index, new in enumerate(ordered_list):
+            new_object = getattr(class_instance, new[1])
+            try:
+                subcontext = new_object(
+                    initial_context=self.task.initial_context,
+                    previous_context=previous_context,
+                    task=self.task,
+                )
+                subcontext.task_id = index
+                new_context.storage.append(subcontext)
+                previous_context = subcontext
+
+            except Exception as error:
+                if not isinstance(error, ExecutionWithClassError):
+                    raise error
+
+                subcontext = new_object(
+                    class_instance,
+                    initial_context=self.task.initial_context,
+                    previous_context=previous_context,
+                    task=self.task,
+                )
+                subcontext.task_id = index
+                new_context.storage.append(subcontext)
+                previous_context = subcontext
+
+        if not new_context.storage:
+            return Context(storage=class_instance)
+
+        return new_context