dotflow 1.0.0.dev4__tar.gz → 1.0.2__tar.gz

{dotflow-1.0.0.dev4 → dotflow-1.0.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dotflow
-Version: 1.0.0.dev4
+Version: 1.0.2
 Summary: 🎲 Dotflow turns an idea into flow! Lightweight Python library for execution pipelines with retry, parallel, cron and async support.
 License: Apache License
                                     Version 2.0, January 2004
@@ -196,7 +196,7 @@ License-File: LICENSE
 Keywords: pipeline,workflow,etl,task-runner,orchestration,automation,parallel,async,retry,data-pipeline,scheduler,cron,cron-job
 Author: Fernando Celmer
 Author-email: email@fernandocelmer.com
-Requires-Python: >=3.9
+Requires-Python: >=3.10
 Classifier: Development Status :: 4 - Beta
 Classifier: Operating System :: OS Independent
 Classifier: Intended Audience :: Developers
@@ -205,7 +205,6 @@ Classifier: License :: OSI Approved :: Apache Software License
 Classifier: Topic :: Software Development :: Libraries
 Classifier: Topic :: Software Development :: Libraries :: Python Modules
 Classifier: Topic :: System :: Distributed Computing
-Classifier: Programming Language :: Python :: 3.9
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12

{dotflow-1.0.0.dev4 → dotflow-1.0.2}/dotflow/__init__.py

@@ -1,6 +1,6 @@
 """Dotflow __init__ module."""
 
-__version__ = "1.0.0.dev4"
+__version__ = "1.0.2"
 __description__ = "🎲 Dotflow turns an idea into flow!"
 
 from .core.action import Action as action

{dotflow-1.0.0.dev4 → dotflow-1.0.2}/dotflow/abc/flow.py

@@ -1,5 +1,7 @@
 """Flow ABC"""
 
+from __future__ import annotations
+
 from abc import ABC, abstractmethod
 from uuid import UUID
 

dotflow-1.0.2/dotflow/abc/storage.py

@@ -0,0 +1,77 @@
+"""Storage ABC"""
+
+from __future__ import annotations
+
+from abc import ABC, abstractmethod
+from collections.abc import Callable, Iterable
+from typing import Any
+
+from dotflow.core.context import Context
+
+
+class Storage(ABC):
+    """Storage"""
+
+    def __init__(self, *args, **kwargs):
+        pass
+
+    @abstractmethod
+    def post(
+        self,
+        key: str,
+        context: Context,
+        ttl: int | None = None,
+        fingerprint: str | None = None,
+    ) -> None:
+        """Persist context under key."""
+
+    @abstractmethod
+    def get(self, key: str) -> Context:
+        """Return stored context or empty Context()."""
+
+    @abstractmethod
+    def key(self, task: Callable) -> str:
+        """Storage key for task."""
+
+    def clear(self, workflow_id: str) -> None:
+        """Remove every entry under workflow_id.
+
+        Default implementation delegates to ``delete_prefix``. External
+        subclasses may override directly.
+        """
+        self.delete_prefix(f"{workflow_id}-")
+
+    def delete(self, key: str) -> bool:
+        """Remove key. Returns True when present.
+
+        Optional in 1.x. Becomes abstract in 2.0.
+        """
+        raise NotImplementedError
+
+    def delete_prefix(self, prefix: str) -> int:
+        """Remove keys starting with prefix. Returns count.
+
+        Optional in 1.x. Becomes abstract in 2.0.
+        """
+        raise NotImplementedError
+
+    def list_keys(self, prefix: str) -> Iterable[str]:
+        """Iterate keys starting with prefix.
+
+        Optional in 1.x. Becomes abstract in 2.0.
+        """
+        raise NotImplementedError
+
+    def atomic_swap(
+        self,
+        key: str,
+        expected: Any,
+        new: Any,
+        ttl: int | None = None,
+        fingerprint: str | None = None,
+    ) -> bool:
+        """Replace value when current equals expected.
+
+        Optional in 1.x. Becomes abstract in 2.0.
+        """
+        raise NotImplementedError

{dotflow-1.0.0.dev4 → dotflow-1.0.2}/dotflow/cli/commands/cloud.py

@@ -169,6 +169,7 @@ class CloudGenerateCommand(Command):
             content = content.replace("{{PROJECT_NAME}}", project_name)
             content = content.replace("{{MODULE_NAME}}", module_name)
 
+            filepath.parent.mkdir(parents=True, exist_ok=True)
             filepath.write_text(content)
             print(f"  Created: {filepath}")
 

{dotflow-1.0.0.dev4 → dotflow-1.0.2}/dotflow/cli/commands/login.py

@@ -23,14 +23,8 @@ MAX_POLL_INTERVAL = 60
 
 class LoginCommand(Command):
     def setup(self):
-        token = getattr(self.params, "token", None)
         base_url = self._resolve_base_url()
 
-        if token:
-            save_cloud_config(token=token, base_url=base_url)
-            print(settings.INFO_ALERT, "Token saved.")
-            return
-
         handshake = self._start_device(base_url)
         if handshake is None:
             return
@@ -49,10 +43,8 @@ class LoginCommand(Command):
         print(settings.INFO_ALERT, "Authenticated.")
 
     def _resolve_base_url(self) -> str:
-        """Flag or env var wins over the default."""
-        explicit = getattr(self.params, "base_url", None) or os.environ.get(
-            "SERVER_BASE_URL"
-        )
+        """``SERVER_BASE_URL`` env var wins over the default."""
+        explicit = os.environ.get("SERVER_BASE_URL")
         return explicit.rstrip("/") if explicit else DEFAULT_BASE_URL
 
     def _start_device(self, base_url: str) -> dict | None:

{dotflow-1.0.0.dev4 → dotflow-1.0.2}/dotflow/cli/commands/start.py

@@ -51,12 +51,12 @@ class StartCommand(Command):
         if not callable(factory):
             raise InvalidWorkflowFactory(factory=self.params.workflow)
 
-        result = factory()
+        workflow = factory()
 
-        if not isinstance(result, DotFlow):
+        if not isinstance(workflow, DotFlow):
             raise InvalidWorkflowFactory(factory=self.params.workflow)
 
-        result.start(mode=self.params.mode, resume=self.params.resume)
+        workflow.start(mode=self.params.mode, resume=self.params.resume)
 
     def _new_workflow(self):
         storage = self._build_storage()

{dotflow-1.0.0.dev4 → dotflow-1.0.2}/dotflow/cli/setup.py

@@ -56,17 +56,6 @@ class Command:
         cmd = self.subparsers.add_parser(
             "login", help="Authenticate the CLI via browser"
         )
-        cmd_group = cmd.add_argument_group("Usage: dotflow login [OPTIONS]")
-        cmd_group.add_argument(
-            "--base-url",
-            default=None,
-            help="Override the Dotflow Cloud API base URL",
-        )
-        cmd_group.add_argument(
-            "--token",
-            default=None,
-            help="Skip the browser flow and save a pre-obtained API token",
-        )
         cmd.set_defaults(exec=LoginCommand)
 
     def setup_logout(self):

dotflow-1.0.2/dotflow/cloud/aws/services/s3.py

@@ -0,0 +1,148 @@
+"""S3 operations — read and write objects."""
+
+from __future__ import annotations
+
+from json import dumps, loads
+
+from dotflow.cloud.core import ObjectStorage
+from dotflow.core.exception import ModuleNotFound
+
+_PRECONDITION_CODES = {
+    "PreconditionFailed",
+    "ConditionalRequestConflict",
+}
+
+
+class S3(ObjectStorage):
+    """Amazon S3 object storage."""
+
+    def __init__(self, bucket: str, prefix: str = "", region: str = None):
+        try:
+            import boto3
+        except ImportError:
+            raise ModuleNotFound(
+                module="boto3", library="dotflow[aws]"
+            ) from None
+
+        self._s3 = boto3.client("s3", region_name=region)
+        self.bucket = bucket
+        self.prefix = prefix
+
+        self._s3.head_bucket(Bucket=self.bucket)
+
+    def read(self, key: str) -> list:
+        """Read a JSON list from S3. Returns empty list if key not found."""
+        try:
+            response = self._s3.get_object(
+                Bucket=self.bucket,
+                Key=f"{self.prefix}{key}",
+            )
+            data = response["Body"].read().decode("utf-8")
+            return loads(data)
+        except self._s3.exceptions.NoSuchKey:
+            return []
+
+    def write(self, key: str, data: list) -> None:
+        """Write a JSON list to S3."""
+        self._s3.put_object(
+            Bucket=self.bucket,
+            Key=f"{self.prefix}{key}",
+            Body=dumps(data),
+            ContentType="application/json",
+        )
+
+    def delete(self, key: str) -> bool:
+        """Delete a single object."""
+        from botocore.exceptions import ClientError
+
+        full_key = f"{self.prefix}{key}"
+
+        try:
+            self._s3.head_object(Bucket=self.bucket, Key=full_key)
+        except ClientError as error:
+            code = error.response.get("Error", {}).get("Code")
+
+            if code in ("404", "NoSuchKey", "NotFound"):
+                return False
+
+            raise
+
+        self._s3.delete_object(Bucket=self.bucket, Key=full_key)
+
+        return True
+
+    def read_with_etag(self, key: str) -> tuple[list, str | None]:
+        """Return (data, etag). Etag is None when key is missing."""
+        try:
+            response = self._s3.get_object(
+                Bucket=self.bucket,
+                Key=f"{self.prefix}{key}",
+            )
+            data = response["Body"].read().decode("utf-8")
+
+            return loads(data), response.get("ETag")
+        except self._s3.exceptions.NoSuchKey:
+            return [], None
+
+    def write_if_match(self, key: str, data: list, etag: str | None) -> bool:
+        """Conditional PutObject. Returns False on precondition failure."""
+        from botocore.exceptions import ClientError
+
+        kwargs = {
+            "Bucket": self.bucket,
+            "Key": f"{self.prefix}{key}",
+            "Body": dumps(data),
+            "ContentType": "application/json",
+        }
+
+        if etag is None:
+            kwargs["IfNoneMatch"] = "*"
+        else:
+            kwargs["IfMatch"] = etag
+
+        try:
+            self._s3.put_object(**kwargs)
+            return True
+        except ClientError as error:
+            code = error.response.get("Error", {}).get("Code")
+
+            if code in _PRECONDITION_CODES:
+                return False
+
+            raise
+
+    def list_keys(self, sub_prefix: str) -> list[str]:
+        """Return keys starting with sub_prefix."""
+        full_prefix = f"{self.prefix}{sub_prefix}"
+        paginator = self._s3.get_paginator("list_objects_v2")
+        names = []
+        offset = len(self.prefix)
+
+        for page in paginator.paginate(Bucket=self.bucket, Prefix=full_prefix):
+            for item in page.get("Contents", []):
+                names.append(item["Key"][offset:])
+
+        return names
+
+    def delete_prefix(self, sub_prefix: str) -> None:
+        """Delete every object whose key starts with prefix + sub_prefix.
+
+        Empty ``sub_prefix`` is rejected to avoid accidentally wiping
+        the entire bucket prefix.
+        """
+        if not sub_prefix:
+            raise ValueError("delete_prefix requires a non-empty sub_prefix")
+
+        full_prefix = f"{self.prefix}{sub_prefix}"
+        paginator = self._s3.get_paginator("list_objects_v2")
+
+        for page in paginator.paginate(Bucket=self.bucket, Prefix=full_prefix):
+            objects = [
+                {"Key": item["Key"]} for item in page.get("Contents", [])
+            ]
+            if not objects:
+                continue
+            self._s3.delete_objects(
+                Bucket=self.bucket,
+                Delete={"Objects": objects},
+            )

dotflow-1.0.2/dotflow/cloud/gcp/services/gcs.py

@@ -0,0 +1,110 @@
+"""GCS operations — read and write objects."""
+
+from __future__ import annotations
+
+from json import dumps, loads
+
+from dotflow.cloud.core import ObjectStorage
+from dotflow.core.exception import ModuleNotFound
+
+
+class GCS(ObjectStorage):
+    """Google Cloud Storage object storage."""
+
+    def __init__(self, bucket: str, prefix: str = "", project: str = None):
+        try:
+            from google.api_core.exceptions import NotFound
+            from google.cloud import storage as gcs
+        except ImportError:
+            raise ModuleNotFound(
+                module="google-cloud-storage",
+                library="dotflow[gcp]",
+            ) from None
+
+        self._not_found = NotFound
+        self._client = gcs.Client(project=project)
+        self._bucket = self._client.bucket(bucket)
+        self._bucket.reload()
+        self.prefix = prefix
+
+    def read(self, key: str) -> list:
+        """Read a JSON list from GCS. Returns empty list if key not found."""
+        blob = self._bucket.blob(f"{self.prefix}{key}")
+        try:
+            data = blob.download_as_text()
+            return loads(data)
+        except self._not_found:
+            return []
+
+    def write(self, key: str, data: list) -> None:
+        """Write a JSON list to GCS."""
+        blob = self._bucket.blob(f"{self.prefix}{key}")
+        blob.upload_from_string(
+            dumps(data),
+            content_type="application/json",
+        )
+
+    def delete(self, key: str) -> bool:
+        """Delete a single blob."""
+        blob = self._bucket.blob(f"{self.prefix}{key}")
+
+        try:
+            blob.delete()
+            return True
+        except self._not_found:
+            return False
+
+    def read_with_generation(self, key: str) -> tuple[list, int | None]:
+        """Return (data, generation). Generation is None when missing."""
+        blob = self._bucket.blob(f"{self.prefix}{key}")
+
+        try:
+            data = blob.download_as_text()
+            return loads(data), blob.generation
+        except self._not_found:
+            return [], None
+
+    def write_if_generation_match(
+        self, key: str, data: list, generation: int | None
+    ) -> bool:
+        """Conditional upload. Returns False on precondition failure."""
+        from google.api_core.exceptions import PreconditionFailed
+
+        blob = self._bucket.blob(f"{self.prefix}{key}")
+        precondition = generation if generation is not None else 0
+
+        try:
+            blob.upload_from_string(
+                dumps(data),
+                content_type="application/json",
+                if_generation_match=precondition,
+            )
+            return True
+        except PreconditionFailed:
+            return False
+
+    def list_keys(self, sub_prefix: str) -> list[str]:
+        """Return blob names starting with sub_prefix."""
+        full_prefix = f"{self.prefix}{sub_prefix}"
+        offset = len(self.prefix)
+
+        return [
+            blob.name[offset:]
+            for blob in self._client.list_blobs(
+                self._bucket, prefix=full_prefix
+            )
+        ]
+
+    def delete_prefix(self, sub_prefix: str) -> None:
+        """Delete every blob whose name starts with prefix + sub_prefix.
+
+        Empty ``sub_prefix`` is rejected to avoid accidentally wiping
+        the entire bucket prefix.
+        """
+        if not sub_prefix:
+            raise ValueError("delete_prefix requires a non-empty sub_prefix")
+
+        full_prefix = f"{self.prefix}{sub_prefix}"
+
+        for blob in self._client.list_blobs(self._bucket, prefix=full_prefix):
+            blob.delete()

{dotflow-1.0.0.dev4 → dotflow-1.0.2}/dotflow/core/dotflow.py

@@ -9,6 +9,8 @@ from uuid import uuid4
 from dotflow.core.config import Config
 from dotflow.core.task import TaskBuilder
 from dotflow.core.workflow import Manager
+from dotflow.logging import logger
+from dotflow.utils import hostname
 
 
 class DotFlow:
@@ -34,9 +36,16 @@ class DotFlow:
         workflow_id (Optional[str]): Fixed workflow identifier for checkpoint
             resume. If not provided, a random UUID is generated.
 
+        name (Optional[str]): Human-readable label sent to the managed
+            server on registration. Defaults to the machine hostname so
+            runs from different hosts stay distinguishable in the
+            dashboard.
+
     Attributes:
         workflow_id (UUID):
 
+        name (str):
+
         task (List[Task]):
 
         start (Manager):
@@ -48,31 +57,48 @@ class DotFlow:
         self,
         config: Config | None = None,
         workflow_id: str | None = None,
+        name: str | None = None,
     ) -> None:
         workflow_id = workflow_id or os.getenv("WORKFLOW_ID")
         self._externally_provided_id = workflow_id is not None
         self.workflow_id = workflow_id or uuid4()
+        self.name = name or hostname()
         self._config = config if config else Config()
-
-        if not self._externally_provided_id:
-            self._config.server.create_workflow(workflow=self.workflow_id)
+        self._manager: Manager | None = None
+        self._last_run_signature: tuple = ()
 
         self.task = TaskBuilder(
             config=self._config,
             workflow_id=self.workflow_id,
+            workflow_name=self.name,
         )
 
-        self.start = partial(
-            Manager,
-            tasks=self.task.queue,
-            workflow_id=self.workflow_id,
-            config=self._config,
-        )
+        if not self._externally_provided_id:
+            self._config.server.create_workflow(workflow=self)
 
         self.schedule = partial(
             self._config.scheduler.start, workflow=self.start
         )
 
+    def start(self, **kwargs) -> Manager:
+        """Run the workflow once; duplicate calls return the original Manager."""
+        signature = tuple(task.task_id for task in self.task.queue)
+        if self._manager is not None and signature == self._last_run_signature:
+            logger.warning(
+                "DotFlow.start() already ran for %s; ignoring duplicate call",
+                self.workflow_id,
+            )
+            return self._manager
+
+        self._last_run_signature = signature
+        self._manager = Manager(
+            tasks=self.task.queue,
+            workflow_id=self.workflow_id,
+            config=self._config,
+            **kwargs,
+        )
+        return self._manager
+
     def result_task(self):
         """
         Returns:

{dotflow-1.0.0.dev4 → dotflow-1.0.2}/dotflow/core/engine.py

@@ -1,5 +1,7 @@
 """TaskEngine module"""
 
+from __future__ import annotations
+
 import re
 from collections.abc import Callable
 from concurrent.futures import (

{dotflow-1.0.0.dev4 → dotflow-1.0.2}/dotflow/core/exception.py

@@ -21,6 +21,14 @@ MESSAGE_INVALID_WORKFLOW_FACTORY = (
 MESSAGE_WORKFLOW_FLAG_CONFLICT = (
     "{flag} is only valid with --step and cannot be used with --workflow."
 )
+MESSAGE_INPUT_CHANGED = (
+    "Workflow '{workflow_id}' was previously executed with a different "
+    "initial_context. Pass on_input_change='reset' to discard prior "
+    "checkpoints, or 'reuse' to ignore the new input."
+)
+MESSAGE_INVALID_ON_INPUT_CHANGE = (
+    "on_input_change must be one of 'reuse', 'reset', 'raise'; got '{value}'."
+)
 
 
 class MissingActionDecorator(Exception):
@@ -72,6 +80,16 @@ class ExecutionWithClassError(Exception):
         super().__init__("Unknown")
 
 
+class InputChangedError(Exception):
+    def __init__(self, workflow_id: str):
+        super().__init__(MESSAGE_INPUT_CHANGED.format(workflow_id=workflow_id))
+
+
+class InvalidOnInputChange(Exception):
+    def __init__(self, value: str):
+        super().__init__(MESSAGE_INVALID_ON_INPUT_CHANGE.format(value=value))
+
+
 class TaskError:
     def __init__(self, error: Exception = None, attempt: int = None) -> None:
         self.attempt = attempt

dotflow-1.0.2/dotflow/core/fingerprint.py

@@ -0,0 +1,45 @@
+"""Input fingerprint helpers for resume + on_input_change policy."""
+
+from __future__ import annotations
+
+from hashlib import sha256
+from json import dumps
+from typing import Any
+
+from dotflow.core.context import Context
+
+FP_KEY_SUFFIX = "-_input_fp"
+
+
+def fingerprint_of(values: list[Any]) -> str:
+    """SHA256 of JSON-serialized list of initial_context payloads.
+
+    Inputs must be JSON-serializable (primitives, lists, dicts).
+    Non-serializable objects fall back to ``repr``, which is **not**
+    guaranteed to be stable across processes — objects without a custom
+    ``__repr__`` will produce different fingerprints on every run because
+    the default repr embeds the memory address.
+    """
+    try:
+        encoded = dumps(values, sort_keys=True)
+    except (TypeError, ValueError):
+        encoded = repr(values)
+
+    return sha256(encoded.encode("utf-8")).hexdigest()
+
+
+def fp_key(workflow_id: str) -> str:
+    return f"{workflow_id}{FP_KEY_SUFFIX}"
+
+
+def read_fingerprint(storage, workflow_id: str) -> str | None:
+    context = storage.get(key=fp_key(workflow_id))
+
+    if isinstance(context, Context) and isinstance(context.storage, str):
+        return context.storage
+
+    return None
+
+
+def write_fingerprint(storage, workflow_id: str, value: str) -> None:
+    storage.post(key=fp_key(workflow_id), context=Context(storage=value))

{dotflow-1.0.0.dev4 → dotflow-1.0.2}/dotflow/core/serializers/task.py

@@ -1,5 +1,7 @@
 """Task serializer module"""
 
+# mypy: disable-error-code="misc"
+
 from __future__ import annotations
 
 import json
@@ -49,6 +51,11 @@ class SerializerTask(BaseModel):
         exclude=True,
     )
 
+    @computed_field
+    @property
+    def id(self) -> Optional[str]:
+        return self.task_id
+
     @computed_field
     @property
     def error(self) -> Optional[SerializerTaskError]:

{dotflow-1.0.0.dev4 → dotflow-1.0.2}/dotflow/core/serializers/transport.py

@@ -1,5 +1,7 @@
 """Transport serializer module"""
 
+from __future__ import annotations
+
 from pydantic import BaseModel, Field, model_validator
 
 from dotflow.core.serializers.task import SerializerTask

dotflow-1.0.2/dotflow/core/serializers/workflow.py

@@ -0,0 +1,28 @@
+"""Workflow serializer module"""
+
+from __future__ import annotations
+
+# mypy: disable-error-code="misc"
+from uuid import UUID
+
+from pydantic import BaseModel, ConfigDict, Field, computed_field
+
+from dotflow.core.serializers.task import SerializerTask
+
+
+class SerializerWorkflow(BaseModel):
+    model_config = ConfigDict(title="workflow")
+
+    workflow_id: UUID = Field(default=None)
+    workflow_name: str | None = Field(default=None)
+    tasks: list[SerializerTask] = Field(default=[])
+
+    @computed_field
+    @property
+    def id(self) -> UUID | None:
+        return self.workflow_id
+
+    @computed_field
+    @property
+    def name(self) -> str | None:
+        return self.workflow_name