dotflow 0.16.0.dev3__tar.gz → 1.0.0__tar.gz

{dotflow-0.16.0.dev3 → dotflow-1.0.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dotflow
-Version: 0.16.0.dev3
+Version: 1.0.0
 Summary: 🎲 Dotflow turns an idea into flow! Lightweight Python library for execution pipelines with retry, parallel, cron and async support.
 License: Apache License
                                     Version 2.0, January 2004
@@ -227,6 +227,7 @@ Requires-Dist: boto3 ; extra == "aws"
 Requires-Dist: boto3 ; extra == "deploy-aws"
 Requires-Dist: cookiecutter (>=2.0)
 Requires-Dist: croniter ; extra == "scheduler"
+Requires-Dist: eval-type-backport ; python_version < "3.10"
 Requires-Dist: google-auth ; extra == "deploy-gcp"
 Requires-Dist: google-cloud-artifact-registry ; extra == "deploy-gcp"
 Requires-Dist: google-cloud-build ; extra == "deploy-gcp"

{dotflow-0.16.0.dev3 → dotflow-1.0.0}/dotflow/__init__.py

@@ -1,6 +1,6 @@
 """Dotflow __init__ module."""
 
-__version__ = "0.16.0.dev3"
+__version__ = "1.0.0"
 __description__ = "🎲 Dotflow turns an idea into flow!"
 
 from .core.action import Action as action

{dotflow-0.16.0.dev3 → dotflow-1.0.0}/dotflow/abc/flow.py

@@ -1,5 +1,7 @@
 """Flow ABC"""
 
+from __future__ import annotations
+
 from abc import ABC, abstractmethod
 from uuid import UUID
 

{dotflow-0.16.0.dev3 → dotflow-1.0.0}/dotflow/abc/storage.py

@@ -23,3 +23,11 @@ class Storage(ABC):
     @abstractmethod
     def key(self, task: Callable):
         """Function that returns a key to get and post storage"""
+
+    @abstractmethod
+    def clear(self, workflow_id: str) -> None:
+        """Remove every persisted entry under ``workflow_id``.
+
+        Used by the input-fingerprint reset path when
+        ``on_input_change='reset'``.
+        """

{dotflow-0.16.0.dev3 → dotflow-1.0.0}/dotflow/cli/commands/__init__.py

@@ -4,6 +4,8 @@ from dotflow.cli.commands.cloud import CloudGenerateCommand, CloudListCommand
 from dotflow.cli.commands.deploy import DeployCommand
 from dotflow.cli.commands.init import InitCommand
 from dotflow.cli.commands.log import LogCommand
+from dotflow.cli.commands.login import LoginCommand
+from dotflow.cli.commands.logout import LogoutCommand
 from dotflow.cli.commands.schedule import ScheduleCommand
 from dotflow.cli.commands.start import StartCommand
 
@@ -13,6 +15,8 @@ __all__ = [
     "DeployCommand",
     "InitCommand",
     "LogCommand",
+    "LoginCommand",
+    "LogoutCommand",
     "ScheduleCommand",
     "StartCommand",
 ]

dotflow-1.0.0/dotflow/cli/commands/login.py

@@ -0,0 +1,115 @@
+"""Command login module."""
+
+from __future__ import annotations
+
+import os
+import time
+import webbrowser
+
+from requests import RequestException, post
+from rich import print  # type: ignore
+
+from dotflow.cli.command import Command
+from dotflow.core.config_file import save_cloud_config
+from dotflow.settings import Settings as settings
+
+DEFAULT_BASE_URL = "https://www.host.dotflow.io/api/v1"
+DEVICE_ENDPOINT = "/cli/device"
+TOKEN_ENDPOINT = "/cli/token"
+TIMEOUT = 15
+DEFAULT_INTERVAL = 5
+MAX_POLL_INTERVAL = 60
+
+
+class LoginCommand(Command):
+    def setup(self):
+        base_url = self._resolve_base_url()
+
+        handshake = self._start_device(base_url)
+        if handshake is None:
+            return
+
+        print(settings.INFO_ALERT, f"Opening {handshake['verification_uri']}")
+        print(settings.INFO_ALERT, f"Code: {handshake['user_code']}")
+
+        webbrowser.open(handshake["verification_uri"])
+
+        token = self._poll_token(base_url, handshake)
+
+        if not token:
+            return
+
+        save_cloud_config(token=token, base_url=base_url)
+        print(settings.INFO_ALERT, "Authenticated.")
+
+    def _resolve_base_url(self) -> str:
+        """``SERVER_BASE_URL`` env var wins over the default."""
+        explicit = os.environ.get("SERVER_BASE_URL")
+        return explicit.rstrip("/") if explicit else DEFAULT_BASE_URL
+
+    def _start_device(self, base_url: str) -> dict | None:
+        try:
+            response = post(
+                f"{base_url}{DEVICE_ENDPOINT}",
+                timeout=TIMEOUT,
+            )
+            response.raise_for_status()
+            return response.json()
+        except RequestException as error:
+            print(
+                settings.ERROR_ALERT,
+                f"Could not reach Dotflow Cloud: {error}",
+            )
+            return None
+
+    def _poll_token(self, base_url: str, handshake: dict) -> str | None:
+        interval = handshake.get("interval") or DEFAULT_INTERVAL
+        deadline = time.monotonic() + (handshake.get("expires_in") or 300)
+
+        while time.monotonic() < deadline:
+            try:
+                response = post(
+                    f"{base_url}{TOKEN_ENDPOINT}",
+                    json={"device_code": handshake["device_code"]},
+                    timeout=TIMEOUT,
+                )
+            except RequestException as error:
+                print(settings.ERROR_ALERT, f"Network error: {error}")
+                return None
+
+            if response.status_code == 200:
+                return response.json().get("api_token")
+
+            detail = self._detail(response)
+
+            if (
+                response.status_code == 400
+                and detail == "authorization_pending"
+            ):
+                time.sleep(interval)
+                continue
+
+            if response.status_code == 400 and detail in (
+                "slow_down",
+                "slow down",
+            ):
+                interval = min(interval + 5, MAX_POLL_INTERVAL)
+                time.sleep(interval)
+                continue
+
+            if response.status_code == 410:
+                print(settings.ERROR_ALERT, "Authorization expired.")
+                return None
+
+            print(settings.ERROR_ALERT, f"{response.status_code}: {detail}")
+            return None
+
+        print(settings.ERROR_ALERT, "Timed out waiting for authorization.")
+        return None
+
+    @staticmethod
+    def _detail(response) -> str:
+        try:
+            return str(response.json().get("detail", response.text))
+        except ValueError:
+            return response.text

dotflow-1.0.0/dotflow/cli/commands/logout.py

@@ -0,0 +1,15 @@
+"""Command logout module."""
+
+from rich import print  # type: ignore
+
+from dotflow.cli.command import Command
+from dotflow.core.config_file import clear_cloud_config
+from dotflow.settings import Settings as settings
+
+
+class LogoutCommand(Command):
+    def setup(self):
+        if clear_cloud_config():
+            print(settings.INFO_ALERT, "Signed out.")
+        else:
+            print(settings.WARNING_ALERT, "No saved credentials.")

{dotflow-0.16.0.dev3 → dotflow-1.0.0}/dotflow/cli/commands/start.py

@@ -1,6 +1,5 @@
 """Command start module"""
 
-import os
 from os import system
 
 from dotflow import Config, DotFlow
@@ -9,8 +8,6 @@ from dotflow.core.exception import InvalidWorkflowFactory, WorkflowFlagConflict
 from dotflow.core.module import Module
 from dotflow.core.types.execution import TypeExecution
 from dotflow.providers import (
-    ServerAPI,
-    ServerDefault,
     StorageDefault,
     StorageFile,
     StorageGCS,
@@ -38,7 +35,7 @@ class StartCommand(Command):
             initial_context=self.params.initial_context,
         )
 
-        workflow.start(mode=self.params.mode)
+        workflow.start(mode=self.params.mode, resume=self.params.resume)
 
     def _start_from_factory(self):
         step_only_flags = {
@@ -54,40 +51,18 @@ class StartCommand(Command):
         if not callable(factory):
             raise InvalidWorkflowFactory(factory=self.params.workflow)
 
-        result = factory()
+        workflow = factory()
 
-        if not isinstance(result, DotFlow):
+        if not isinstance(workflow, DotFlow):
             raise InvalidWorkflowFactory(factory=self.params.workflow)
 
-        result.start(mode=self.params.mode)
+        workflow.start(mode=self.params.mode, resume=self.params.resume)
 
     def _new_workflow(self):
-        config = self._build_config()
-        workflow_id = os.getenv("WORKFLOW_ID")
-
-        if config is None:
-            return DotFlow(workflow_id=workflow_id)
-        return DotFlow(config=config, workflow_id=workflow_id)
-
-    def _build_config(self):
         storage = self._build_storage()
-        server = self._build_server()
-
-        if storage is None and server is None:
-            return None
-
-        return Config(
-            storage=storage or StorageDefault(),
-            server=server or ServerDefault(),
-        )
-
-    def _build_server(self):
-        base_url = os.getenv("SERVER_BASE_URL")
-        user_token = os.getenv("SERVER_USER_TOKEN")
-
-        if not base_url or not user_token:
-            return None
-        return ServerAPI(base_url=base_url, user_token=user_token)
+        if storage is None:
+            return DotFlow()
+        return DotFlow(config=Config(storage=storage))
 
     def _build_storage(self):
         if not self.params.storage:

{dotflow-0.16.0.dev3 → dotflow-1.0.0}/dotflow/cli/setup.py

@@ -9,6 +9,8 @@ from dotflow.cli.commands import (
     DeployCommand,
     InitCommand,
     LogCommand,
+    LoginCommand,
+    LogoutCommand,
     ScheduleCommand,
     StartCommand,
 )
@@ -42,12 +44,26 @@ class Command:
 
         self.setup_init()
         self.setup_logs()
+        self.setup_login()
+        self.setup_logout()
         self.setup_start()
         self.setup_schedule()
         self.setup_cloud()
         self.setup_deploy()
         self.command()
 
+    def setup_login(self):
+        cmd = self.subparsers.add_parser(
+            "login", help="Authenticate the CLI via browser"
+        )
+        cmd.set_defaults(exec=LoginCommand)
+
+    def setup_logout(self):
+        cmd = self.subparsers.add_parser(
+            "logout", help="Remove saved CLI credentials"
+        )
+        cmd.set_defaults(exec=LogoutCommand)
+
     def setup_init(self):
         self.cmd_init = self.subparsers.add_parser(
             "init", help="Scaffold a new dotflow project"
@@ -93,6 +109,12 @@ class Command:
                 TypeExecution.PARALLEL,
             ],
         )
+        self.cmd_start.add_argument(
+            "--resume",
+            action="store_true",
+            default=False,
+            help="Enable checkpoint-based resume",
+        )
 
         cmd_start_parser.set_defaults(exec=StartCommand)
 

{dotflow-0.16.0.dev3 → dotflow-1.0.0}/dotflow/cloud/aws/services/s3.py

@@ -45,3 +45,26 @@ class S3(ObjectStorage):
             Body=dumps(data),
             ContentType="application/json",
         )
+
+    def delete_prefix(self, sub_prefix: str) -> None:
+        """Delete every object whose key starts with prefix + sub_prefix.
+
+        Empty ``sub_prefix`` is rejected to avoid accidentally wiping
+        the entire bucket prefix.
+        """
+        if not sub_prefix:
+            raise ValueError("delete_prefix requires a non-empty sub_prefix")
+
+        full_prefix = f"{self.prefix}{sub_prefix}"
+        paginator = self._s3.get_paginator("list_objects_v2")
+
+        for page in paginator.paginate(Bucket=self.bucket, Prefix=full_prefix):
+            objects = [
+                {"Key": item["Key"]} for item in page.get("Contents", [])
+            ]
+            if not objects:
+                continue
+            self._s3.delete_objects(
+                Bucket=self.bucket,
+                Delete={"Objects": objects},
+            )

{dotflow-0.16.0.dev3 → dotflow-1.0.0}/dotflow/cloud/gcp/services/gcs.py

@@ -43,3 +43,17 @@ class GCS(ObjectStorage):
             dumps(data),
             content_type="application/json",
         )
+
+    def delete_prefix(self, sub_prefix: str) -> None:
+        """Delete every blob whose name starts with prefix + sub_prefix.
+
+        Empty ``sub_prefix`` is rejected to avoid accidentally wiping
+        the entire bucket prefix.
+        """
+        if not sub_prefix:
+            raise ValueError("delete_prefix requires a non-empty sub_prefix")
+
+        full_prefix = f"{self.prefix}{sub_prefix}"
+
+        for blob in self._client.list_blobs(self._bucket, prefix=full_prefix):
+            blob.delete()

dotflow-1.0.0/dotflow/core/config_file.py

@@ -0,0 +1,95 @@
+"""Persistent CLI config stored at ``~/.dotflow/config.json``."""
+
+from __future__ import annotations
+
+import contextlib
+import json
+import os
+from pathlib import Path
+
+CONFIG_DIR_NAME = ".dotflow"
+CONFIG_FILE_NAME = "config.json"
+CLOUD_SECTION = "cloud"
+
+
+def config_path() -> Path:
+    """Return the absolute path of the CLI config file."""
+    return Path.home() / CONFIG_DIR_NAME / CONFIG_FILE_NAME
+
+
+def load_cloud_config() -> dict[str, str]:
+    """Read the ``cloud`` section from the config file. ``{}`` when absent."""
+    path = config_path()
+
+    if not path.exists():
+        return {}
+
+    try:
+        raw = path.read_text(encoding="utf-8")
+    except OSError:
+        return {}
+
+    try:
+        data = json.loads(raw)
+    except json.JSONDecodeError:
+        return {}
+
+    section = data.get(CLOUD_SECTION) if isinstance(data, dict) else None
+    if not isinstance(section, dict):
+        return {}
+
+    return {
+        key: str(value)
+        for key, value in section.items()
+        if isinstance(value, str)
+    }
+
+
+def save_cloud_config(token: str, base_url: str) -> Path:
+    """Persist ``cloud`` with the given token/base_url. Returns the file path."""
+    path = config_path()
+    path.parent.mkdir(parents=True, exist_ok=True)
+
+    payload = {
+        CLOUD_SECTION: {
+            "base_url": base_url,
+            "token": token,
+        }
+    }
+    content = json.dumps(payload, indent=2) + "\n"
+
+    fd = os.open(
+        str(path),
+        os.O_WRONLY | os.O_CREAT | os.O_TRUNC,
+        0o600,
+    )
+    with contextlib.suppress(OSError):
+        os.fchmod(fd, 0o600)
+    with os.fdopen(fd, "w", encoding="utf-8") as fh:
+        fh.write(content)
+
+    return path
+
+
+def clear_cloud_config() -> bool:
+    """Remove the config file. Returns True when something was deleted."""
+    path = config_path()
+
+    if not path.exists():
+        return False
+
+    try:
+        path.unlink()
+        return True
+    except OSError:
+        return False
+
+
+def resolve(key: str, env_var: str) -> str | None:
+    """Resolve a setting with env > file precedence. ``None`` when absent."""
+    env_value = os.environ.get(env_var)
+
+    if env_value:
+        return env_value
+
+    return load_cloud_config().get(key) or None

{dotflow-0.16.0.dev3 → dotflow-1.0.0}/dotflow/core/dotflow.py

@@ -2,12 +2,15 @@
 
 from __future__ import annotations
 
+import os
 from functools import partial
 from uuid import uuid4
 
 from dotflow.core.config import Config
 from dotflow.core.task import TaskBuilder
 from dotflow.core.workflow import Manager
+from dotflow.logging import logger
+from dotflow.utils import hostname
 
 
 class DotFlow:
@@ -33,9 +36,16 @@ class DotFlow:
         workflow_id (Optional[str]): Fixed workflow identifier for checkpoint
             resume. If not provided, a random UUID is generated.
 
+        name (Optional[str]): Human-readable label sent to the managed
+            server on registration. Defaults to the machine hostname so
+            runs from different hosts stay distinguishable in the
+            dashboard.
+
     Attributes:
         workflow_id (UUID):
 
+        name (str):
+
         task (List[Task]):
 
         start (Manager):
@@ -47,30 +57,48 @@ class DotFlow:
         self,
         config: Config | None = None,
         workflow_id: str | None = None,
+        name: str | None = None,
     ) -> None:
+        workflow_id = workflow_id or os.getenv("WORKFLOW_ID")
         self._externally_provided_id = workflow_id is not None
         self.workflow_id = workflow_id or uuid4()
+        self.name = name or hostname()
         self._config = config if config else Config()
-
-        if not self._externally_provided_id:
-            self._config.server.create_workflow(workflow=self.workflow_id)
+        self._manager: Manager | None = None
+        self._last_run_signature: tuple = ()
 
         self.task = TaskBuilder(
             config=self._config,
             workflow_id=self.workflow_id,
+            workflow_name=self.name,
         )
 
-        self.start = partial(
-            Manager,
-            tasks=self.task.queue,
-            workflow_id=self.workflow_id,
-            config=self._config,
-        )
+        if not self._externally_provided_id:
+            self._config.server.create_workflow(workflow=self)
 
         self.schedule = partial(
             self._config.scheduler.start, workflow=self.start
         )
 
+    def start(self, **kwargs) -> Manager:
+        """Run the workflow once; duplicate calls return the original Manager."""
+        signature = tuple(task.task_id for task in self.task.queue)
+        if self._manager is not None and signature == self._last_run_signature:
+            logger.warning(
+                "DotFlow.start() already ran for %s; ignoring duplicate call",
+                self.workflow_id,
+            )
+            return self._manager
+
+        self._last_run_signature = signature
+        self._manager = Manager(
+            tasks=self.task.queue,
+            workflow_id=self.workflow_id,
+            config=self._config,
+            **kwargs,
+        )
+        return self._manager
+
     def result_task(self):
         """
         Returns:

{dotflow-0.16.0.dev3 → dotflow-1.0.0}/dotflow/core/engine.py

@@ -1,5 +1,7 @@
 """TaskEngine module"""
 
+from __future__ import annotations
+
 import re
 from collections.abc import Callable
 from concurrent.futures import (

{dotflow-0.16.0.dev3 → dotflow-1.0.0}/dotflow/core/exception.py

@@ -21,6 +21,14 @@ MESSAGE_INVALID_WORKFLOW_FACTORY = (
 MESSAGE_WORKFLOW_FLAG_CONFLICT = (
     "{flag} is only valid with --step and cannot be used with --workflow."
 )
+MESSAGE_INPUT_CHANGED = (
+    "Workflow '{workflow_id}' was previously executed with a different "
+    "initial_context. Pass on_input_change='reset' to discard prior "
+    "checkpoints, or 'reuse' to ignore the new input."
+)
+MESSAGE_INVALID_ON_INPUT_CHANGE = (
+    "on_input_change must be one of 'reuse', 'reset', 'raise'; got '{value}'."
+)
 
 
 class MissingActionDecorator(Exception):
@@ -72,6 +80,16 @@ class ExecutionWithClassError(Exception):
         super().__init__("Unknown")
 
 
+class InputChangedError(Exception):
+    def __init__(self, workflow_id: str):
+        super().__init__(MESSAGE_INPUT_CHANGED.format(workflow_id=workflow_id))
+
+
+class InvalidOnInputChange(Exception):
+    def __init__(self, value: str):
+        super().__init__(MESSAGE_INVALID_ON_INPUT_CHANGE.format(value=value))
+
+
 class TaskError:
     def __init__(self, error: Exception = None, attempt: int = None) -> None:
         self.attempt = attempt

dotflow-1.0.0/dotflow/core/fingerprint.py

@@ -0,0 +1,45 @@
+"""Input fingerprint helpers for resume + on_input_change policy."""
+
+from __future__ import annotations
+
+from hashlib import sha256
+from json import dumps
+from typing import Any
+
+from dotflow.core.context import Context
+
+FP_KEY_SUFFIX = "-_input_fp"
+
+
+def fingerprint_of(values: list[Any]) -> str:
+    """SHA256 of JSON-serialized list of initial_context payloads.
+
+    Inputs must be JSON-serializable (primitives, lists, dicts).
+    Non-serializable objects fall back to ``repr``, which is **not**
+    guaranteed to be stable across processes — objects without a custom
+    ``__repr__`` will produce different fingerprints on every run because
+    the default repr embeds the memory address.
+    """
+    try:
+        encoded = dumps(values, sort_keys=True)
+    except (TypeError, ValueError):
+        encoded = repr(values)
+
+    return sha256(encoded.encode("utf-8")).hexdigest()
+
+
+def fp_key(workflow_id: str) -> str:
+    return f"{workflow_id}{FP_KEY_SUFFIX}"
+
+
+def read_fingerprint(storage, workflow_id: str) -> str | None:
+    context = storage.get(key=fp_key(workflow_id))
+
+    if isinstance(context, Context) and isinstance(context.storage, str):
+        return context.storage
+
+    return None
+
+
+def write_fingerprint(storage, workflow_id: str, value: str) -> None:
+    storage.post(key=fp_key(workflow_id), context=Context(storage=value))

{dotflow-0.16.0.dev3 → dotflow-1.0.0}/dotflow/core/module.py

@@ -14,7 +14,8 @@ class Module:
 
     @classmethod
     def import_module(cls, value: str):
-        module_path, _, attr_name = value.rpartition(".")
+        separator = ":" if ":" in value else "."
+        module_path, _, attr_name = value.rpartition(separator)
 
         if not module_path:
             raise ImportModuleError(module=value)

{dotflow-0.16.0.dev3 → dotflow-1.0.0}/dotflow/core/serializers/task.py

@@ -1,5 +1,7 @@
 """Task serializer module"""
 
+# mypy: disable-error-code="misc"
+
 from __future__ import annotations
 
 import json
@@ -49,6 +51,11 @@ class SerializerTask(BaseModel):
         exclude=True,
     )
 
+    @computed_field
+    @property
+    def id(self) -> Optional[str]:
+        return self.task_id
+
     @computed_field
     @property
     def error(self) -> Optional[SerializerTaskError]:

{dotflow-0.16.0.dev3 → dotflow-1.0.0}/dotflow/core/serializers/transport.py

@@ -1,5 +1,7 @@
 """Transport serializer module"""
 
+from __future__ import annotations
+
 from pydantic import BaseModel, Field, model_validator
 
 from dotflow.core.serializers.task import SerializerTask