dotflow 0.14.1.dev2__tar.gz → 0.15.0.dev2__tar.gz

{dotflow-0.14.1.dev2 → dotflow-0.15.0.dev2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dotflow
-Version: 0.14.1.dev2
+Version: 0.15.0.dev2
 Summary: 🎲 Dotflow turns an idea into flow! Lightweight Python library for execution pipelines with retry, parallel, cron and async support.
 License: MIT License
          
@@ -42,15 +42,31 @@ Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
 Classifier: Programming Language :: Python :: 3.13
 Provides-Extra: aws
+Provides-Extra: deploy-aws
+Provides-Extra: deploy-gcp
+Provides-Extra: deploy-github
 Provides-Extra: gcp
+Provides-Extra: otel
 Provides-Extra: scheduler
+Provides-Extra: sentry
+Requires-Dist: PyGithub ; extra == "deploy-github"
 Requires-Dist: boto3 ; extra == "aws"
+Requires-Dist: boto3 ; extra == "deploy-aws"
 Requires-Dist: cookiecutter (>=2.0)
 Requires-Dist: croniter ; extra == "scheduler"
+Requires-Dist: google-auth ; extra == "deploy-gcp"
+Requires-Dist: google-cloud-artifact-registry ; extra == "deploy-gcp"
+Requires-Dist: google-cloud-build ; extra == "deploy-gcp"
+Requires-Dist: google-cloud-run ; extra == "deploy-gcp"
+Requires-Dist: google-cloud-service-usage ; extra == "deploy-gcp"
+Requires-Dist: google-cloud-storage ; extra == "deploy-gcp"
 Requires-Dist: google-cloud-storage ; extra == "gcp"
+Requires-Dist: opentelemetry-api ; extra == "otel"
+Requires-Dist: opentelemetry-sdk ; extra == "otel"
 Requires-Dist: pydantic
 Requires-Dist: requests
 Requires-Dist: rich
+Requires-Dist: sentry-sdk ; extra == "sentry"
 Requires-Dist: typing-extensions
 Project-URL: Changelog, https://dotflow-io.github.io/dotflow/nav/development/release-notes/
 Project-URL: Documentation, https://dotflow-io.github.io/dotflow/

{dotflow-0.14.1.dev2 → dotflow-0.15.0.dev2}/dotflow/__init__.py

@@ -1,6 +1,6 @@
 """Dotflow __init__ module."""
 
-__version__ = "0.14.1.dev2"
+__version__ = "0.15.0.dev2"
 __description__ = "🎲 Dotflow turns an idea into flow!"
 
 from .core.action import Action as action

dotflow-0.15.0.dev2/dotflow/abc/log.py

@@ -0,0 +1,122 @@
+"""Log ABC"""
+
+from __future__ import annotations
+
+import logging
+from abc import ABC
+from datetime import datetime, timezone
+from typing import Optional
+
+from pydantic import BaseModel, Field
+
+LEVELS = {
+    "DEBUG": logging.DEBUG,
+    "INFO": logging.INFO,
+    "WARNING": logging.WARNING,
+    "ERROR": logging.ERROR,
+}
+
+
+class LogRecordError(BaseModel):
+    exception: str = Field(default="")
+    message: str = Field(default="")
+
+
+class LogRecord(BaseModel):
+    timestamp: str = Field(default="")
+    level: str = Field(default="")
+    workflow_id: str = Field(default="")
+    task_id: str = Field(default="")
+    status: str = Field(default="")
+    duration: Optional[float] = Field(default=None)
+    retry_count: Optional[int] = Field(default=None)
+    error: Optional[LogRecordError] = Field(default=None)
+
+
+class Log(ABC):
+    """Log
+
+    Base class for all log providers. Subclasses must set
+    ``self._logger``, ``self._level`` and ``self._format``
+    in their ``__init__``.
+    """
+
+    _logger: logging.Logger
+    _level: int = logging.INFO
+    _format: str = "simple"
+
+    def info(self, **kwargs) -> None:
+        """Info"""
+        self._log(logging.INFO, **kwargs)
+
+    def error(self, **kwargs) -> None:
+        """Error"""
+        self._log(logging.ERROR, **kwargs)
+
+    def warning(self, **kwargs) -> None:
+        """Warning"""
+        self._log(logging.WARNING, **kwargs)
+
+    def debug(self, **kwargs) -> None:
+        """Debug"""
+        self._log(logging.DEBUG, **kwargs)
+
+    _FORMATTERS = {
+        "json": "_format_json",
+        "simple": "_format_text",
+    }
+
+    def _log(self, level: int, **kwargs) -> None:
+        if level < self._level:
+            return
+
+        formatter = getattr(
+            self, self._FORMATTERS.get(self._format, "_format_text")
+        )
+        task = kwargs.get("task")
+
+        if task:
+            message = formatter(level, task=task)
+        else:
+            message = formatter(level, **kwargs)
+
+        self._logger.log(level, message)
+
+    def _format_json(self, level: int, **kwargs) -> str:
+        task = kwargs.get("task")
+        if task:
+            record = LogRecord(
+                timestamp=datetime.now(timezone.utc).isoformat(),
+                level=logging.getLevelName(level),
+                workflow_id=str(task.workflow_id),
+                task_id=str(task.task_id),
+                status=str(task.status),
+            )
+
+            if task.duration is not None:
+                record.duration = task.duration
+
+            if task.retry_count:
+                record.retry_count = task.retry_count
+
+            if task.errors and level >= logging.ERROR:
+                last_error = task.errors[-1]
+                record.error = LogRecordError(
+                    exception=str(last_error.exception),
+                    message=str(last_error.message),
+                )
+
+            return record.model_dump_json(exclude_none=True)
+
+        record = LogRecord(**{k: str(v) for k, v in kwargs.items()})
+        return record.model_dump_json(exclude_none=True)
+
+    def _format_text(self, level: int, **kwargs) -> str:
+        task = kwargs.get("task")
+        if task:
+            message = f"ID {task.workflow_id} - {task.task_id} - {task.status}"
+            if task.errors and level >= logging.ERROR:
+                message += f" \n {task.errors[-1].traceback}"
+            return message
+
+        return str(kwargs)

dotflow-0.15.0.dev2/dotflow/abc/metrics.py

@@ -0,0 +1,32 @@
+"""Metrics ABC"""
+
+from abc import ABC, abstractmethod
+from typing import Any
+
+
+class Metrics(ABC):
+    """Metrics"""
+
+    @abstractmethod
+    def workflow_started(self, workflow_id: Any, **kwargs) -> None:
+        """Called when a workflow starts."""
+
+    @abstractmethod
+    def workflow_completed(self, workflow_id: Any, duration: float) -> None:
+        """Called when a workflow completes successfully."""
+
+    @abstractmethod
+    def workflow_failed(self, workflow_id: Any, duration: float) -> None:
+        """Called when a workflow fails."""
+
+    @abstractmethod
+    def task_completed(self, task: Any) -> None:
+        """Called when a task completes successfully."""
+
+    @abstractmethod
+    def task_failed(self, task: Any) -> None:
+        """Called when a task fails."""
+
+    @abstractmethod
+    def task_retried(self, task: Any) -> None:
+        """Called when a task is retried."""

dotflow-0.15.0.dev2/dotflow/abc/tracer.py

@@ -0,0 +1,24 @@
+"""Tracer ABC"""
+
+from abc import ABC, abstractmethod
+from typing import Any
+
+
+class Tracer(ABC):
+    """Tracer"""
+
+    @abstractmethod
+    def start_workflow(self, workflow_id: Any, **kwargs) -> None:
+        """Called when a workflow starts execution."""
+
+    @abstractmethod
+    def end_workflow(self, workflow_id: Any, **kwargs) -> None:
+        """Called when a workflow finishes execution."""
+
+    @abstractmethod
+    def start_task(self, task: Any) -> None:
+        """Called when a task starts execution."""
+
+    @abstractmethod
+    def end_task(self, task: Any) -> None:
+        """Called when a task finishes execution."""

dotflow-0.15.0.dev2/dotflow/cli/commands/__init__.py

@@ -0,0 +1,18 @@
+"""Commands __init__ module."""
+
+from dotflow.cli.commands.cloud import CloudGenerateCommand, CloudListCommand
+from dotflow.cli.commands.deploy import DeployCommand
+from dotflow.cli.commands.init import InitCommand
+from dotflow.cli.commands.log import LogCommand
+from dotflow.cli.commands.schedule import ScheduleCommand
+from dotflow.cli.commands.start import StartCommand
+
+__all__ = [
+    "CloudGenerateCommand",
+    "CloudListCommand",
+    "DeployCommand",
+    "InitCommand",
+    "LogCommand",
+    "ScheduleCommand",
+    "StartCommand",
+]

dotflow-0.15.0.dev2/dotflow/cli/commands/cloud.py

@@ -0,0 +1,194 @@
+"""Command cloud module"""
+
+import json
+import re
+from pathlib import Path
+from subprocess import run
+
+from rich import print  # type: ignore
+from rich.console import Console
+from rich.table import Table
+
+from dotflow.cli.command import Command
+from dotflow.settings import Settings as settings
+
+
+def _get_template_dir() -> Path | None:
+    import tempfile
+
+    target = Path(tempfile.gettempdir()) / "dotflow-template"
+    cloud_dir = target / settings.TEMPLATE_CLOUD_DIR
+
+    if cloud_dir.exists():
+        return cloud_dir
+
+    try:
+        result = run(
+            [
+                "git",
+                "clone",
+                "--depth",
+                "1",
+                "--branch",
+                settings.TEMPLATE_BRANCH,
+                settings.TEMPLATE_REPO,
+                str(target),
+            ],
+            capture_output=True,
+            text=True,
+        )
+        if result.returncode == 0:
+            return cloud_dir
+    except Exception as err:
+        print(
+            settings.ERROR_ALERT,
+            f"Failed to clone template repository: {err}",
+        )
+
+    return None
+
+
+def _load_registry(cloud_dir: Path) -> dict | None:
+    registry_path = cloud_dir / "registry.json"
+    if not registry_path.exists():
+        print(settings.ERROR_ALERT, "Registry not found in template.")
+        return None
+
+    try:
+        return json.loads(registry_path.read_text())
+    except Exception as err:
+        print(settings.ERROR_ALERT, f"Failed to read registry: {err}")
+        return None
+
+
+def _get_platforms(registry: dict) -> dict | None:
+    platforms = registry.get("platforms")
+    if not isinstance(platforms, dict):
+        print(settings.ERROR_ALERT, "Unexpected registry format.")
+        return None
+
+    return platforms
+
+
+def _read_project_name(pyproject: Path) -> str | None:
+    try:
+        in_project = False
+        for line in pyproject.read_text().splitlines():
+            stripped = line.strip()
+            if stripped.startswith("["):
+                in_project = stripped in ("[project]", "[tool.poetry]")
+            if in_project and re.match(r"^name\s*=", stripped):
+                return stripped.split("=", 1)[1].strip().strip('"').strip("'")
+    except Exception:
+        return None
+    return None
+
+
+class CloudGenerateCommand(Command):
+    def setup(self):
+        platform = self.params.platform
+
+        cloud_dir = _get_template_dir()
+        if cloud_dir is None:
+            return
+
+        registry = _load_registry(cloud_dir)
+        if registry is None:
+            return
+
+        platforms = _get_platforms(registry)
+        if platforms is None:
+            return
+
+        if platform not in platforms:
+            available = ", ".join(sorted(platforms.keys()))
+            print(
+                settings.ERROR_ALERT,
+                f"Unknown platform '{platform}'. Available: {available}",
+            )
+            return
+
+        platform_info = platforms[platform]
+        files = platform_info.get("files", [])
+        output = Path(self.params.output).resolve()
+        output.mkdir(parents=True, exist_ok=True)
+
+        project_name = self.params.project
+        if not project_name:
+            pyproject = Path.cwd() / "pyproject.toml"
+            if pyproject.exists():
+                project_name = _read_project_name(pyproject)
+            if not project_name:
+                project_name = Path.cwd().name
+
+        module_name = project_name.replace("-", "_")
+
+        print(
+            settings.INFO_ALERT,
+            f"Generating {platform_info.get('name', platform)} files"
+            f" for '{project_name}'...",
+        )
+
+        source_dir = cloud_dir / platform
+
+        for filename in files:
+            filepath = (output / filename).resolve()
+            if not str(filepath).startswith(str(output)):
+                print(
+                    settings.ERROR_ALERT,
+                    f"  Skipped (unsafe filename): {filename}",
+                )
+                continue
+
+            if filepath.exists():
+                print(
+                    settings.WARNING_ALERT,
+                    f"  Skipped (already exists): {filepath}",
+                )
+                continue
+
+            source = source_dir / filename
+            if not source.exists():
+                print(
+                    settings.ERROR_ALERT,
+                    f"  Skipped (template not found): {filename}",
+                )
+                continue
+
+            content = source.read_text()
+            content = content.replace("{{PROJECT_NAME}}", project_name)
+            content = content.replace("{{MODULE_NAME}}", module_name)
+
+            filepath.write_text(content)
+            print(f"  Created: {filepath}")
+
+        print(settings.INFO_ALERT, "Done.")
+
+
+class CloudListCommand(Command):
+    def setup(self):
+        cloud_dir = _get_template_dir()
+        if cloud_dir is None:
+            return
+
+        registry = _load_registry(cloud_dir)
+        if registry is None:
+            return
+
+        platforms = _get_platforms(registry)
+        if platforms is None:
+            return
+
+        table = Table(title="Available Platforms")
+        table.add_column("Platform", style="bold cyan")
+        table.add_column("Name", style="bold")
+        table.add_column("Description")
+
+        for key, info in platforms.items():
+            table.add_row(
+                key,
+                info.get("name", ""),
+                info.get("description", ""),
+            )
+
+        Console().print(table)

dotflow-0.15.0.dev2/dotflow/cli/commands/deploy.py

@@ -0,0 +1,110 @@
+"""Command deploy module."""
+
+from rich import print  # type: ignore
+
+from dotflow.cli.command import Command
+from dotflow.settings import Settings as settings
+
+SAM_PLATFORMS = {
+    "ecs-scheduled",
+}
+
+DEFAULT_REGIONS = {
+    "lambda": "us-east-1",
+    "lambda-scheduled": "us-east-1",
+    "lambda-s3-trigger": "us-east-1",
+    "lambda-sqs-trigger": "us-east-1",
+    "lambda-api-trigger": "us-east-1",
+    "ecs": "us-east-1",
+    "cloud-run": "us-central1",
+    "cloud-run-scheduled": "us-central1",
+}
+
+
+class DeployCommand(Command):
+    def setup(self):
+        platform = self.params.platform
+        name = self.params.project
+        region = self.params.region or DEFAULT_REGIONS.get(
+            platform, "us-east-1"
+        )
+        schedule = getattr(self.params, "schedule", None)
+
+        if platform in SAM_PLATFORMS:
+            print(
+                settings.INFO_ALERT,
+                f"'{platform}' requires SAM for full trigger setup.",
+            )
+            print("  Run: sam build && sam deploy")
+            return
+
+        deployers = {
+            "lambda": self._deploy_lambda,
+            "lambda-scheduled": self._deploy_lambda,
+            "lambda-s3-trigger": self._deploy_lambda_s3,
+            "lambda-sqs-trigger": self._deploy_lambda_sqs,
+            "lambda-api-trigger": self._deploy_lambda_api,
+            "ecs": self._deploy_ecs,
+            "cloud-run": self._deploy_cloud_run,
+            "cloud-run-scheduled": self._deploy_cloud_run,
+            "github-actions": self._deploy_github_actions,
+        }
+
+        handler = deployers.get(platform)
+        if not handler:
+            print(
+                settings.ERROR_ALERT,
+                f"Deploy not supported for '{platform}'.",
+            )
+            return
+
+        handler(name=name, region=region, schedule=schedule)
+
+    def _deploy_lambda(self, name: str, region: str, schedule: str = None):
+        """Deploy to AWS Lambda."""
+        from dotflow.cloud.aws import LambdaDeployer
+
+        deployer = LambdaDeployer(region=region)
+        deployer.deploy(name, schedule=schedule)
+
+    def _deploy_lambda_api(self, name: str, region: str, **kwargs):
+        """Deploy to AWS Lambda + API Gateway."""
+        from dotflow.cloud.aws import LambdaApiDeployer
+
+        deployer = LambdaApiDeployer(region=region)
+        deployer.deploy(name)
+
+    def _deploy_lambda_s3(self, name: str, region: str, **kwargs):
+        """Deploy to AWS Lambda + S3 Trigger."""
+        from dotflow.cloud.aws import LambdaS3Deployer
+
+        deployer = LambdaS3Deployer(region=region)
+        deployer.deploy(name)
+
+    def _deploy_lambda_sqs(self, name: str, region: str, **kwargs):
+        """Deploy to AWS Lambda + SQS Trigger."""
+        from dotflow.cloud.aws import LambdaSQSDeployer
+
+        deployer = LambdaSQSDeployer(region=region)
+        deployer.deploy(name)
+
+    def _deploy_ecs(self, name: str, region: str, **kwargs):
+        """Deploy to AWS ECS Fargate."""
+        from dotflow.cloud.aws import ECSDeployer
+
+        deployer = ECSDeployer(region=region)
+        deployer.deploy(name)
+
+    def _deploy_cloud_run(self, name: str, region: str, **kwargs):
+        """Deploy to Google Cloud Run."""
+        from dotflow.cloud.gcp import CloudRunDeployer
+
+        deployer = CloudRunDeployer(region=region)
+        deployer.deploy(name)
+
+    def _deploy_github_actions(self, name: str, **kwargs):
+        """Deploy to GitHub Actions."""
+        from dotflow.cloud.github import ActionsDeployer
+
+        deployer = ActionsDeployer()
+        deployer.deploy(name)

{dotflow-0.14.1.dev2 → dotflow-0.15.0.dev2}/dotflow/cli/setup.py

@@ -4,6 +4,9 @@ from rich import print  # type: ignore
 
 from dotflow import __description__, __version__
 from dotflow.cli.commands import (
+    CloudGenerateCommand,
+    CloudListCommand,
+    DeployCommand,
     InitCommand,
     LogCommand,
     ScheduleCommand,
@@ -39,6 +42,8 @@ class Command:
         self.setup_logs()
         self.setup_start()
         self.setup_schedule()
+        self.setup_cloud()
+        self.setup_deploy()
         self.command()
 
     def setup_init(self):
@@ -145,6 +150,66 @@ class Command:
 
         self.cmd_schedule.set_defaults(exec=ScheduleCommand)
 
+    def setup_cloud(self):
+        self.cmd_cloud = self.subparsers.add_parser(
+            "cloud",
+            help="Generate cloud infrastructure files for a target platform",
+        )
+        cloud_subparsers = self.cmd_cloud.add_subparsers()
+
+        cmd_generate = cloud_subparsers.add_parser(
+            "generate", help="Generate infrastructure files"
+        )
+        cmd_generate.add_argument(
+            "--platform",
+            required=True,
+            help="Target platform (e.g. docker, lambda, cloud-run, ecs, kubernetes)",
+        )
+        cmd_generate.add_argument(
+            "--project",
+            default=None,
+            help="Project name (defaults to current directory name)",
+        )
+        cmd_generate.add_argument(
+            "--output",
+            default=".",
+            help="Output directory (defaults to current directory)",
+        )
+        cmd_generate.set_defaults(exec=CloudGenerateCommand)
+
+        cmd_list = cloud_subparsers.add_parser(
+            "list", help="List available cloud platforms"
+        )
+        cmd_list.set_defaults(exec=CloudListCommand)
+
+    def setup_deploy(self):
+        self.cmd_deploy = self.subparsers.add_parser(
+            "deploy",
+            help="Deploy pipeline to a cloud platform",
+        )
+        self.cmd_deploy.add_argument(
+            "--platform",
+            required=True,
+            help="Target platform (e.g. lambda, ecs)",
+        )
+        self.cmd_deploy.add_argument(
+            "--project",
+            default=None,
+            required=True,
+            help="Project name",
+        )
+        self.cmd_deploy.add_argument(
+            "--region",
+            default=None,
+            help="Cloud region (e.g. us-east-1 for AWS, us-central1 for GCP)",
+        )
+        self.cmd_deploy.add_argument(
+            "--schedule",
+            default=None,
+            help="Schedule expression (e.g. 'rate(6 hours)')",
+        )
+        self.cmd_deploy.set_defaults(exec=DeployCommand)
+
     def setup_logs(self):
         self.cmd_logs = self.subparsers.add_parser("logs", help="Logs")
         self.cmd_logs = self.cmd_logs.add_argument_group(

dotflow-0.15.0.dev2/dotflow/cloud/__init__.py

@@ -0,0 +1 @@
+"""Deploy module."""

dotflow-0.15.0.dev2/dotflow/cloud/aws/__init__.py

@@ -0,0 +1,15 @@
+"""AWS deploy module."""
+
+from dotflow.cloud.aws.ecs_deployer import ECSDeployer
+from dotflow.cloud.aws.lambda_api_deployer import LambdaApiDeployer
+from dotflow.cloud.aws.lambda_deployer import LambdaDeployer
+from dotflow.cloud.aws.lambda_s3_deployer import LambdaS3Deployer
+from dotflow.cloud.aws.lambda_sqs_deployer import LambdaSQSDeployer
+
+__all__ = [
+    "LambdaDeployer",
+    "LambdaApiDeployer",
+    "LambdaS3Deployer",
+    "LambdaSQSDeployer",
+    "ECSDeployer",
+]