dotflow 0.15.0.dev1__tar.gz → 0.15.0.dev2__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {dotflow-0.15.0.dev1 → dotflow-0.15.0.dev2}/PKG-INFO +5 -1
- {dotflow-0.15.0.dev1 → dotflow-0.15.0.dev2}/dotflow/__init__.py +1 -1
- {dotflow-0.15.0.dev1 → dotflow-0.15.0.dev2}/dotflow/cli/commands/deploy.py +35 -3
- dotflow-0.15.0.dev2/dotflow/cloud/aws/__init__.py +15 -0
- dotflow-0.15.0.dev1/dotflow/cloud/aws/lambda_deployer.py → dotflow-0.15.0.dev2/dotflow/cloud/aws/base_lambda_deployer.py +30 -51
- dotflow-0.15.0.dev2/dotflow/cloud/aws/lambda_api_deployer.py +67 -0
- dotflow-0.15.0.dev2/dotflow/cloud/aws/lambda_deployer.py +49 -0
- dotflow-0.15.0.dev2/dotflow/cloud/aws/lambda_s3_deployer.py +56 -0
- dotflow-0.15.0.dev2/dotflow/cloud/aws/lambda_sqs_deployer.py +43 -0
- {dotflow-0.15.0.dev1 → dotflow-0.15.0.dev2}/dotflow/cloud/core.py +22 -0
- dotflow-0.15.0.dev2/dotflow/cloud/github/__init__.py +5 -0
- dotflow-0.15.0.dev2/dotflow/cloud/github/actions_deployer.py +113 -0
- dotflow-0.15.0.dev2/dotflow/cloud/github/constants.py +8 -0
- {dotflow-0.15.0.dev1 → dotflow-0.15.0.dev2}/dotflow/providers/__init__.py +12 -0
- dotflow-0.15.0.dev2/dotflow/providers/log_sentry.py +96 -0
- dotflow-0.15.0.dev2/dotflow/providers/tracer_sentry.py +126 -0
- {dotflow-0.15.0.dev1 → dotflow-0.15.0.dev2}/pyproject.toml +4 -2
- dotflow-0.15.0.dev1/dotflow/cloud/aws/__init__.py +0 -6
- {dotflow-0.15.0.dev1 → dotflow-0.15.0.dev2}/LICENSE +0 -0
- {dotflow-0.15.0.dev1 → dotflow-0.15.0.dev2}/README.md +0 -0
- {dotflow-0.15.0.dev1 → dotflow-0.15.0.dev2}/dotflow/abc/__init__.py +0 -0
- {dotflow-0.15.0.dev1 → dotflow-0.15.0.dev2}/dotflow/abc/api.py +0 -0
- {dotflow-0.15.0.dev1 → dotflow-0.15.0.dev2}/dotflow/abc/file.py +0 -0
- {dotflow-0.15.0.dev1 → dotflow-0.15.0.dev2}/dotflow/abc/flow.py +0 -0
- {dotflow-0.15.0.dev1 → dotflow-0.15.0.dev2}/dotflow/abc/http.py +0 -0
- {dotflow-0.15.0.dev1 → dotflow-0.15.0.dev2}/dotflow/abc/log.py +0 -0
- {dotflow-0.15.0.dev1 → dotflow-0.15.0.dev2}/dotflow/abc/metrics.py +0 -0
- {dotflow-0.15.0.dev1 → dotflow-0.15.0.dev2}/dotflow/abc/notify.py +0 -0
- {dotflow-0.15.0.dev1 → dotflow-0.15.0.dev2}/dotflow/abc/scheduler.py +0 -0
- {dotflow-0.15.0.dev1 → dotflow-0.15.0.dev2}/dotflow/abc/storage.py +0 -0
- {dotflow-0.15.0.dev1 → dotflow-0.15.0.dev2}/dotflow/abc/tcp.py +0 -0
- {dotflow-0.15.0.dev1 → dotflow-0.15.0.dev2}/dotflow/abc/tracer.py +0 -0
- {dotflow-0.15.0.dev1 → dotflow-0.15.0.dev2}/dotflow/cli/__init__.py +0 -0
- {dotflow-0.15.0.dev1 → dotflow-0.15.0.dev2}/dotflow/cli/command.py +0 -0
- {dotflow-0.15.0.dev1 → dotflow-0.15.0.dev2}/dotflow/cli/commands/__init__.py +0 -0
- {dotflow-0.15.0.dev1 → dotflow-0.15.0.dev2}/dotflow/cli/commands/cloud.py +0 -0
- {dotflow-0.15.0.dev1 → dotflow-0.15.0.dev2}/dotflow/cli/commands/init.py +0 -0
- {dotflow-0.15.0.dev1 → dotflow-0.15.0.dev2}/dotflow/cli/commands/log.py +0 -0
- {dotflow-0.15.0.dev1 → dotflow-0.15.0.dev2}/dotflow/cli/commands/schedule.py +0 -0
- {dotflow-0.15.0.dev1 → dotflow-0.15.0.dev2}/dotflow/cli/commands/start.py +0 -0
- {dotflow-0.15.0.dev1 → dotflow-0.15.0.dev2}/dotflow/cli/setup.py +0 -0
- {dotflow-0.15.0.dev1 → dotflow-0.15.0.dev2}/dotflow/cli/validators/__init__.py +0 -0
- {dotflow-0.15.0.dev1 → dotflow-0.15.0.dev2}/dotflow/cli/validators/start.py +0 -0
- {dotflow-0.15.0.dev1 → dotflow-0.15.0.dev2}/dotflow/cloud/__init__.py +0 -0
- {dotflow-0.15.0.dev1 → dotflow-0.15.0.dev2}/dotflow/cloud/aws/constants.py +0 -0
- {dotflow-0.15.0.dev1 → dotflow-0.15.0.dev2}/dotflow/cloud/aws/ecs_deployer.py +0 -0
- {dotflow-0.15.0.dev1 → dotflow-0.15.0.dev2}/dotflow/cloud/aws/services/__init__.py +0 -0
- {dotflow-0.15.0.dev1 → dotflow-0.15.0.dev2}/dotflow/cloud/aws/services/cloudwatch.py +0 -0
- {dotflow-0.15.0.dev1 → dotflow-0.15.0.dev2}/dotflow/cloud/aws/services/ecr.py +0 -0
- {dotflow-0.15.0.dev1 → dotflow-0.15.0.dev2}/dotflow/cloud/aws/services/iam.py +0 -0
- {dotflow-0.15.0.dev1 → dotflow-0.15.0.dev2}/dotflow/cloud/aws/services/s3.py +0 -0
- {dotflow-0.15.0.dev1 → dotflow-0.15.0.dev2}/dotflow/cloud/gcp/__init__.py +0 -0
- {dotflow-0.15.0.dev1 → dotflow-0.15.0.dev2}/dotflow/cloud/gcp/cloudrun_deployer.py +0 -0
- {dotflow-0.15.0.dev1 → dotflow-0.15.0.dev2}/dotflow/cloud/gcp/constants.py +0 -0
- {dotflow-0.15.0.dev1 → dotflow-0.15.0.dev2}/dotflow/cloud/gcp/services/__init__.py +0 -0
- {dotflow-0.15.0.dev1 → dotflow-0.15.0.dev2}/dotflow/cloud/gcp/services/apis.py +0 -0
- {dotflow-0.15.0.dev1 → dotflow-0.15.0.dev2}/dotflow/cloud/gcp/services/artifact_registry.py +0 -0
- {dotflow-0.15.0.dev1 → dotflow-0.15.0.dev2}/dotflow/cloud/gcp/services/gcs.py +0 -0
- {dotflow-0.15.0.dev1 → dotflow-0.15.0.dev2}/dotflow/core/__init__.py +0 -0
- {dotflow-0.15.0.dev1 → dotflow-0.15.0.dev2}/dotflow/core/action.py +0 -0
- {dotflow-0.15.0.dev1 → dotflow-0.15.0.dev2}/dotflow/core/config.py +0 -0
- {dotflow-0.15.0.dev1 → dotflow-0.15.0.dev2}/dotflow/core/context.py +0 -0
- {dotflow-0.15.0.dev1 → dotflow-0.15.0.dev2}/dotflow/core/decorators/__init__.py +0 -0
- {dotflow-0.15.0.dev1 → dotflow-0.15.0.dev2}/dotflow/core/decorators/time.py +0 -0
- {dotflow-0.15.0.dev1 → dotflow-0.15.0.dev2}/dotflow/core/dotflow.py +0 -0
- {dotflow-0.15.0.dev1 → dotflow-0.15.0.dev2}/dotflow/core/exception.py +0 -0
- {dotflow-0.15.0.dev1 → dotflow-0.15.0.dev2}/dotflow/core/execution.py +0 -0
- {dotflow-0.15.0.dev1 → dotflow-0.15.0.dev2}/dotflow/core/module.py +0 -0
- {dotflow-0.15.0.dev1 → dotflow-0.15.0.dev2}/dotflow/core/serializers/__init__.py +0 -0
- {dotflow-0.15.0.dev1 → dotflow-0.15.0.dev2}/dotflow/core/serializers/task.py +0 -0
- {dotflow-0.15.0.dev1 → dotflow-0.15.0.dev2}/dotflow/core/serializers/transport.py +0 -0
- {dotflow-0.15.0.dev1 → dotflow-0.15.0.dev2}/dotflow/core/serializers/workflow.py +0 -0
- {dotflow-0.15.0.dev1 → dotflow-0.15.0.dev2}/dotflow/core/task.py +0 -0
- {dotflow-0.15.0.dev1 → dotflow-0.15.0.dev2}/dotflow/core/types/__init__.py +0 -0
- {dotflow-0.15.0.dev1 → dotflow-0.15.0.dev2}/dotflow/core/types/execution.py +0 -0
- {dotflow-0.15.0.dev1 → dotflow-0.15.0.dev2}/dotflow/core/types/overlap.py +0 -0
- {dotflow-0.15.0.dev1 → dotflow-0.15.0.dev2}/dotflow/core/types/status.py +0 -0
- {dotflow-0.15.0.dev1 → dotflow-0.15.0.dev2}/dotflow/core/types/storage.py +0 -0
- {dotflow-0.15.0.dev1 → dotflow-0.15.0.dev2}/dotflow/core/types/workflow.py +0 -0
- {dotflow-0.15.0.dev1 → dotflow-0.15.0.dev2}/dotflow/core/workflow.py +0 -0
- {dotflow-0.15.0.dev1 → dotflow-0.15.0.dev2}/dotflow/logging.py +0 -0
- {dotflow-0.15.0.dev1 → dotflow-0.15.0.dev2}/dotflow/main.py +0 -0
- {dotflow-0.15.0.dev1 → dotflow-0.15.0.dev2}/dotflow/providers/api_default.py +0 -0
- {dotflow-0.15.0.dev1 → dotflow-0.15.0.dev2}/dotflow/providers/log_default.py +0 -0
- {dotflow-0.15.0.dev1 → dotflow-0.15.0.dev2}/dotflow/providers/log_opentelemetry.py +0 -0
- {dotflow-0.15.0.dev1 → dotflow-0.15.0.dev2}/dotflow/providers/metrics_default.py +0 -0
- {dotflow-0.15.0.dev1 → dotflow-0.15.0.dev2}/dotflow/providers/metrics_opentelemetry.py +0 -0
- {dotflow-0.15.0.dev1 → dotflow-0.15.0.dev2}/dotflow/providers/notify_default.py +0 -0
- {dotflow-0.15.0.dev1 → dotflow-0.15.0.dev2}/dotflow/providers/notify_discord.py +0 -0
- {dotflow-0.15.0.dev1 → dotflow-0.15.0.dev2}/dotflow/providers/notify_telegram.py +0 -0
- {dotflow-0.15.0.dev1 → dotflow-0.15.0.dev2}/dotflow/providers/scheduler_cron.py +0 -0
- {dotflow-0.15.0.dev1 → dotflow-0.15.0.dev2}/dotflow/providers/scheduler_default.py +0 -0
- {dotflow-0.15.0.dev1 → dotflow-0.15.0.dev2}/dotflow/providers/storage_default.py +0 -0
- {dotflow-0.15.0.dev1 → dotflow-0.15.0.dev2}/dotflow/providers/storage_file.py +0 -0
- {dotflow-0.15.0.dev1 → dotflow-0.15.0.dev2}/dotflow/providers/storage_gcs.py +0 -0
- {dotflow-0.15.0.dev1 → dotflow-0.15.0.dev2}/dotflow/providers/storage_s3.py +0 -0
- {dotflow-0.15.0.dev1 → dotflow-0.15.0.dev2}/dotflow/providers/tracer_default.py +0 -0
- {dotflow-0.15.0.dev1 → dotflow-0.15.0.dev2}/dotflow/providers/tracer_opentelemetry.py +0 -0
- {dotflow-0.15.0.dev1 → dotflow-0.15.0.dev2}/dotflow/settings.py +0 -0
- {dotflow-0.15.0.dev1 → dotflow-0.15.0.dev2}/dotflow/storage.py +0 -0
- {dotflow-0.15.0.dev1 → dotflow-0.15.0.dev2}/dotflow/types.py +0 -0
- {dotflow-0.15.0.dev1 → dotflow-0.15.0.dev2}/dotflow/utils/__init__.py +0 -0
- {dotflow-0.15.0.dev1 → dotflow-0.15.0.dev2}/dotflow/utils/basic_functions.py +0 -0
- {dotflow-0.15.0.dev1 → dotflow-0.15.0.dev2}/dotflow/utils/error_handler.py +0 -0
- {dotflow-0.15.0.dev1 → dotflow-0.15.0.dev2}/dotflow/utils/tools.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: dotflow
|
|
3
|
-
Version: 0.15.0.
|
|
3
|
+
Version: 0.15.0.dev2
|
|
4
4
|
Summary: 🎲 Dotflow turns an idea into flow! Lightweight Python library for execution pipelines with retry, parallel, cron and async support.
|
|
5
5
|
License: MIT License
|
|
6
6
|
|
|
@@ -44,9 +44,12 @@ Classifier: Programming Language :: Python :: 3.13
|
|
|
44
44
|
Provides-Extra: aws
|
|
45
45
|
Provides-Extra: deploy-aws
|
|
46
46
|
Provides-Extra: deploy-gcp
|
|
47
|
+
Provides-Extra: deploy-github
|
|
47
48
|
Provides-Extra: gcp
|
|
48
49
|
Provides-Extra: otel
|
|
49
50
|
Provides-Extra: scheduler
|
|
51
|
+
Provides-Extra: sentry
|
|
52
|
+
Requires-Dist: PyGithub ; extra == "deploy-github"
|
|
50
53
|
Requires-Dist: boto3 ; extra == "aws"
|
|
51
54
|
Requires-Dist: boto3 ; extra == "deploy-aws"
|
|
52
55
|
Requires-Dist: cookiecutter (>=2.0)
|
|
@@ -63,6 +66,7 @@ Requires-Dist: opentelemetry-sdk ; extra == "otel"
|
|
|
63
66
|
Requires-Dist: pydantic
|
|
64
67
|
Requires-Dist: requests
|
|
65
68
|
Requires-Dist: rich
|
|
69
|
+
Requires-Dist: sentry-sdk ; extra == "sentry"
|
|
66
70
|
Requires-Dist: typing-extensions
|
|
67
71
|
Project-URL: Changelog, https://dotflow-io.github.io/dotflow/nav/development/release-notes/
|
|
68
72
|
Project-URL: Documentation, https://dotflow-io.github.io/dotflow/
|
|
@@ -6,15 +6,15 @@ from dotflow.cli.command import Command
|
|
|
6
6
|
from dotflow.settings import Settings as settings
|
|
7
7
|
|
|
8
8
|
SAM_PLATFORMS = {
|
|
9
|
-
"lambda-s3-trigger",
|
|
10
|
-
"lambda-sqs-trigger",
|
|
11
|
-
"lambda-api-trigger",
|
|
12
9
|
"ecs-scheduled",
|
|
13
10
|
}
|
|
14
11
|
|
|
15
12
|
DEFAULT_REGIONS = {
|
|
16
13
|
"lambda": "us-east-1",
|
|
17
14
|
"lambda-scheduled": "us-east-1",
|
|
15
|
+
"lambda-s3-trigger": "us-east-1",
|
|
16
|
+
"lambda-sqs-trigger": "us-east-1",
|
|
17
|
+
"lambda-api-trigger": "us-east-1",
|
|
18
18
|
"ecs": "us-east-1",
|
|
19
19
|
"cloud-run": "us-central1",
|
|
20
20
|
"cloud-run-scheduled": "us-central1",
|
|
@@ -41,9 +41,13 @@ class DeployCommand(Command):
|
|
|
41
41
|
deployers = {
|
|
42
42
|
"lambda": self._deploy_lambda,
|
|
43
43
|
"lambda-scheduled": self._deploy_lambda,
|
|
44
|
+
"lambda-s3-trigger": self._deploy_lambda_s3,
|
|
45
|
+
"lambda-sqs-trigger": self._deploy_lambda_sqs,
|
|
46
|
+
"lambda-api-trigger": self._deploy_lambda_api,
|
|
44
47
|
"ecs": self._deploy_ecs,
|
|
45
48
|
"cloud-run": self._deploy_cloud_run,
|
|
46
49
|
"cloud-run-scheduled": self._deploy_cloud_run,
|
|
50
|
+
"github-actions": self._deploy_github_actions,
|
|
47
51
|
}
|
|
48
52
|
|
|
49
53
|
handler = deployers.get(platform)
|
|
@@ -63,6 +67,27 @@ class DeployCommand(Command):
|
|
|
63
67
|
deployer = LambdaDeployer(region=region)
|
|
64
68
|
deployer.deploy(name, schedule=schedule)
|
|
65
69
|
|
|
70
|
+
def _deploy_lambda_api(self, name: str, region: str, **kwargs):
|
|
71
|
+
"""Deploy to AWS Lambda + API Gateway."""
|
|
72
|
+
from dotflow.cloud.aws import LambdaApiDeployer
|
|
73
|
+
|
|
74
|
+
deployer = LambdaApiDeployer(region=region)
|
|
75
|
+
deployer.deploy(name)
|
|
76
|
+
|
|
77
|
+
def _deploy_lambda_s3(self, name: str, region: str, **kwargs):
|
|
78
|
+
"""Deploy to AWS Lambda + S3 Trigger."""
|
|
79
|
+
from dotflow.cloud.aws import LambdaS3Deployer
|
|
80
|
+
|
|
81
|
+
deployer = LambdaS3Deployer(region=region)
|
|
82
|
+
deployer.deploy(name)
|
|
83
|
+
|
|
84
|
+
def _deploy_lambda_sqs(self, name: str, region: str, **kwargs):
|
|
85
|
+
"""Deploy to AWS Lambda + SQS Trigger."""
|
|
86
|
+
from dotflow.cloud.aws import LambdaSQSDeployer
|
|
87
|
+
|
|
88
|
+
deployer = LambdaSQSDeployer(region=region)
|
|
89
|
+
deployer.deploy(name)
|
|
90
|
+
|
|
66
91
|
def _deploy_ecs(self, name: str, region: str, **kwargs):
|
|
67
92
|
"""Deploy to AWS ECS Fargate."""
|
|
68
93
|
from dotflow.cloud.aws import ECSDeployer
|
|
@@ -76,3 +101,10 @@ class DeployCommand(Command):
|
|
|
76
101
|
|
|
77
102
|
deployer = CloudRunDeployer(region=region)
|
|
78
103
|
deployer.deploy(name)
|
|
104
|
+
|
|
105
|
+
def _deploy_github_actions(self, name: str, **kwargs):
|
|
106
|
+
"""Deploy to GitHub Actions."""
|
|
107
|
+
from dotflow.cloud.github import ActionsDeployer
|
|
108
|
+
|
|
109
|
+
deployer = ActionsDeployer()
|
|
110
|
+
deployer.deploy(name)
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
"""AWS deploy module."""
|
|
2
|
+
|
|
3
|
+
from dotflow.cloud.aws.ecs_deployer import ECSDeployer
|
|
4
|
+
from dotflow.cloud.aws.lambda_api_deployer import LambdaApiDeployer
|
|
5
|
+
from dotflow.cloud.aws.lambda_deployer import LambdaDeployer
|
|
6
|
+
from dotflow.cloud.aws.lambda_s3_deployer import LambdaS3Deployer
|
|
7
|
+
from dotflow.cloud.aws.lambda_sqs_deployer import LambdaSQSDeployer
|
|
8
|
+
|
|
9
|
+
__all__ = [
|
|
10
|
+
"LambdaDeployer",
|
|
11
|
+
"LambdaApiDeployer",
|
|
12
|
+
"LambdaS3Deployer",
|
|
13
|
+
"LambdaSQSDeployer",
|
|
14
|
+
"ECSDeployer",
|
|
15
|
+
]
|
|
@@ -1,9 +1,7 @@
|
|
|
1
|
-
"""Lambda deployment."""
|
|
1
|
+
"""Base Lambda deployment — shared logic for all Lambda variants."""
|
|
2
2
|
|
|
3
3
|
from __future__ import annotations
|
|
4
4
|
|
|
5
|
-
import contextlib
|
|
6
|
-
|
|
7
5
|
from rich import print # type: ignore
|
|
8
6
|
|
|
9
7
|
from dotflow.cloud.aws.constants import BOTO3_NOT_FOUND, CREDENTIALS_NOT_FOUND
|
|
@@ -14,8 +12,8 @@ from dotflow.cloud.core import Deployer
|
|
|
14
12
|
from dotflow.settings import Settings as settings
|
|
15
13
|
|
|
16
14
|
|
|
17
|
-
class
|
|
18
|
-
"""
|
|
15
|
+
class BaseLambdaDeployer(Deployer):
|
|
16
|
+
"""Shared logic for all Lambda deployers."""
|
|
19
17
|
|
|
20
18
|
def __init__(self, region: str = "us-east-1"):
|
|
21
19
|
try:
|
|
@@ -30,8 +28,8 @@ class LambdaDeployer(Deployer):
|
|
|
30
28
|
raise SystemExit(CREDENTIALS_NOT_FOUND) from err
|
|
31
29
|
|
|
32
30
|
self._region = region
|
|
31
|
+
self._boto3 = boto3
|
|
33
32
|
self._lambda = boto3.client("lambda", region_name=region)
|
|
34
|
-
self._events = boto3.client("events", region_name=region)
|
|
35
33
|
|
|
36
34
|
self._ecr = ECR(
|
|
37
35
|
boto3.client("ecr", region_name=region),
|
|
@@ -41,34 +39,47 @@ class LambdaDeployer(Deployer):
|
|
|
41
39
|
self._iam = IAM(boto3.client("iam", region_name=region))
|
|
42
40
|
self._logs = CloudWatch(boto3.client("logs", region_name=region))
|
|
43
41
|
|
|
42
|
+
@property
|
|
43
|
+
def _function_arn(self) -> str:
|
|
44
|
+
return (
|
|
45
|
+
f"arn:aws:lambda:{self._region}:{self._account_id}"
|
|
46
|
+
f":function:{self._name}"
|
|
47
|
+
)
|
|
48
|
+
|
|
44
49
|
def setup(self, name: str) -> None:
|
|
45
50
|
"""Create IAM role and CloudWatch log group."""
|
|
51
|
+
self._name = name
|
|
46
52
|
self._role_arn = self._iam.ensure_lambda_role(name)
|
|
47
53
|
self._logs.ensure_log_group(f"/aws/lambda/{name}")
|
|
48
54
|
|
|
49
|
-
def
|
|
50
|
-
"""
|
|
51
|
-
|
|
55
|
+
def ensure_roles(self, name: str) -> str:
|
|
56
|
+
"""Create or get Lambda execution role."""
|
|
57
|
+
return self._iam.ensure_lambda_role(name)
|
|
52
58
|
|
|
53
|
-
|
|
59
|
+
def ensure_logs(self, name: str) -> None:
|
|
60
|
+
"""Create CloudWatch log group."""
|
|
61
|
+
self._logs.ensure_log_group(f"/aws/lambda/{name}")
|
|
62
|
+
|
|
63
|
+
def deploy(self, name: str, **kwargs) -> None:
|
|
64
|
+
"""Deploy Lambda and configure trigger."""
|
|
65
|
+
print(settings.INFO_ALERT, f"Deploying '{name}'...")
|
|
54
66
|
|
|
55
67
|
self.setup(name)
|
|
56
68
|
image_uri = self._ecr.push(name)
|
|
57
|
-
self.
|
|
58
|
-
|
|
59
|
-
if schedule:
|
|
60
|
-
self._create_schedule(name, schedule)
|
|
69
|
+
self._create_or_update_lambda(name, image_uri, self._role_arn)
|
|
70
|
+
self._configure_trigger(name, **kwargs)
|
|
61
71
|
|
|
62
72
|
print(settings.INFO_ALERT, "Done.")
|
|
63
73
|
|
|
64
|
-
def
|
|
74
|
+
def _create_or_update_lambda(
|
|
75
|
+
self, name: str, image_uri: str, role_arn: str
|
|
76
|
+
):
|
|
65
77
|
"""Create or update Lambda function."""
|
|
66
78
|
try:
|
|
67
79
|
self._lambda.get_function(FunctionName=name)
|
|
68
80
|
print(" Updating Lambda function...")
|
|
69
81
|
self._lambda.update_function_code(
|
|
70
|
-
FunctionName=name,
|
|
71
|
-
ImageUri=image_uri,
|
|
82
|
+
FunctionName=name, ImageUri=image_uri
|
|
72
83
|
)
|
|
73
84
|
except self._lambda.exceptions.ResourceNotFoundException:
|
|
74
85
|
print(" Creating Lambda function...")
|
|
@@ -81,37 +92,5 @@ class LambdaDeployer(Deployer):
|
|
|
81
92
|
MemorySize=512,
|
|
82
93
|
)
|
|
83
94
|
|
|
84
|
-
def
|
|
85
|
-
"""
|
|
86
|
-
rule_name = f"{name}-schedule"
|
|
87
|
-
function_arn = (
|
|
88
|
-
f"arn:aws:lambda:{self._region}:{self._account_id}:function:{name}"
|
|
89
|
-
)
|
|
90
|
-
|
|
91
|
-
print(f" Creating EventBridge rule '{rule_name}'...")
|
|
92
|
-
self._events.put_rule(
|
|
93
|
-
Name=rule_name,
|
|
94
|
-
ScheduleExpression=schedule,
|
|
95
|
-
State="ENABLED",
|
|
96
|
-
)
|
|
97
|
-
|
|
98
|
-
with contextlib.suppress(
|
|
99
|
-
self._lambda.exceptions.ResourceConflictException
|
|
100
|
-
):
|
|
101
|
-
self._lambda.add_permission(
|
|
102
|
-
FunctionName=name,
|
|
103
|
-
StatementId=rule_name,
|
|
104
|
-
Action="lambda:InvokeFunction",
|
|
105
|
-
Principal="events.amazonaws.com",
|
|
106
|
-
SourceArn=(
|
|
107
|
-
f"arn:aws:events:{self._region}:{self._account_id}"
|
|
108
|
-
f":rule/{rule_name}"
|
|
109
|
-
),
|
|
110
|
-
)
|
|
111
|
-
|
|
112
|
-
self._events.put_targets(
|
|
113
|
-
Rule=rule_name,
|
|
114
|
-
Targets=[{"Id": "1", "Arn": function_arn}],
|
|
115
|
-
)
|
|
116
|
-
|
|
117
|
-
print(f" Schedule: {schedule}")
|
|
95
|
+
def _configure_trigger(self, name: str, **kwargs) -> None:
|
|
96
|
+
"""Override in subclasses to configure the specific trigger."""
|
|
@@ -0,0 +1,67 @@
|
|
|
1
|
+
"""Lambda + API Gateway deployment."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import contextlib
|
|
6
|
+
|
|
7
|
+
from dotflow.cloud.aws.base_lambda_deployer import BaseLambdaDeployer
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
class LambdaApiDeployer(BaseLambdaDeployer):
|
|
11
|
+
"""Deploy dotflow pipeline as Lambda + API Gateway."""
|
|
12
|
+
|
|
13
|
+
def _configure_trigger(self, name: str, **kwargs) -> None:
|
|
14
|
+
"""Create HTTP API with Lambda integration."""
|
|
15
|
+
apigw = self._boto3.client("apigatewayv2", region_name=self._region)
|
|
16
|
+
api_name = f"{name}-api"
|
|
17
|
+
|
|
18
|
+
api_id = self._find_api(apigw, api_name)
|
|
19
|
+
|
|
20
|
+
if not api_id:
|
|
21
|
+
print(" Creating API Gateway...")
|
|
22
|
+
response = apigw.create_api(Name=api_name, ProtocolType="HTTP")
|
|
23
|
+
api_id = response["ApiId"]
|
|
24
|
+
|
|
25
|
+
integration = apigw.create_integration(
|
|
26
|
+
ApiId=api_id,
|
|
27
|
+
IntegrationType="AWS_PROXY",
|
|
28
|
+
IntegrationUri=self._function_arn,
|
|
29
|
+
PayloadFormatVersion="2.0",
|
|
30
|
+
)
|
|
31
|
+
|
|
32
|
+
apigw.create_route(
|
|
33
|
+
ApiId=api_id,
|
|
34
|
+
RouteKey="POST /workflow",
|
|
35
|
+
Target=f"integrations/{integration['IntegrationId']}",
|
|
36
|
+
)
|
|
37
|
+
|
|
38
|
+
apigw.create_stage(
|
|
39
|
+
ApiId=api_id, StageName="$default", AutoDeploy=True
|
|
40
|
+
)
|
|
41
|
+
|
|
42
|
+
with contextlib.suppress(
|
|
43
|
+
self._lambda.exceptions.ResourceConflictException
|
|
44
|
+
):
|
|
45
|
+
self._lambda.add_permission(
|
|
46
|
+
FunctionName=name,
|
|
47
|
+
StatementId=f"{api_name}-invoke",
|
|
48
|
+
Action="lambda:InvokeFunction",
|
|
49
|
+
Principal="apigateway.amazonaws.com",
|
|
50
|
+
SourceArn=(
|
|
51
|
+
f"arn:aws:execute-api:{self._region}:"
|
|
52
|
+
f"{self._account_id}:{api_id}/*"
|
|
53
|
+
),
|
|
54
|
+
)
|
|
55
|
+
|
|
56
|
+
url = (
|
|
57
|
+
f"https://{api_id}.execute-api."
|
|
58
|
+
f"{self._region}.amazonaws.com/workflow"
|
|
59
|
+
)
|
|
60
|
+
print(f" Endpoint: {url}")
|
|
61
|
+
|
|
62
|
+
def _find_api(self, apigw, name: str) -> str | None:
|
|
63
|
+
"""Find existing API by name."""
|
|
64
|
+
for api in apigw.get_apis().get("Items", []):
|
|
65
|
+
if api["Name"] == name:
|
|
66
|
+
return api["ApiId"]
|
|
67
|
+
return None
|
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
"""Lambda deployment."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import contextlib
|
|
6
|
+
|
|
7
|
+
from dotflow.cloud.aws.base_lambda_deployer import BaseLambdaDeployer
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
class LambdaDeployer(BaseLambdaDeployer):
|
|
11
|
+
"""Deploy dotflow pipelines to AWS Lambda."""
|
|
12
|
+
|
|
13
|
+
def _configure_trigger(self, name: str, **kwargs) -> None:
|
|
14
|
+
"""Create EventBridge schedule if provided."""
|
|
15
|
+
schedule = kwargs.get("schedule")
|
|
16
|
+
if not schedule:
|
|
17
|
+
return
|
|
18
|
+
|
|
19
|
+
rule_name = f"{name}-schedule"
|
|
20
|
+
|
|
21
|
+
print(f" Creating EventBridge rule '{rule_name}'...")
|
|
22
|
+
events = self._boto3.client("events", region_name=self._region)
|
|
23
|
+
|
|
24
|
+
events.put_rule(
|
|
25
|
+
Name=rule_name,
|
|
26
|
+
ScheduleExpression=schedule,
|
|
27
|
+
State="ENABLED",
|
|
28
|
+
)
|
|
29
|
+
|
|
30
|
+
with contextlib.suppress(
|
|
31
|
+
self._lambda.exceptions.ResourceConflictException
|
|
32
|
+
):
|
|
33
|
+
self._lambda.add_permission(
|
|
34
|
+
FunctionName=name,
|
|
35
|
+
StatementId=rule_name,
|
|
36
|
+
Action="lambda:InvokeFunction",
|
|
37
|
+
Principal="events.amazonaws.com",
|
|
38
|
+
SourceArn=(
|
|
39
|
+
f"arn:aws:events:{self._region}:{self._account_id}"
|
|
40
|
+
f":rule/{rule_name}"
|
|
41
|
+
),
|
|
42
|
+
)
|
|
43
|
+
|
|
44
|
+
events.put_targets(
|
|
45
|
+
Rule=rule_name,
|
|
46
|
+
Targets=[{"Id": "1", "Arn": self._function_arn}],
|
|
47
|
+
)
|
|
48
|
+
|
|
49
|
+
print(f" Schedule: {schedule}")
|
|
@@ -0,0 +1,56 @@
|
|
|
1
|
+
"""Lambda + S3 Trigger deployment."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import contextlib
|
|
6
|
+
|
|
7
|
+
from dotflow.cloud.aws.base_lambda_deployer import BaseLambdaDeployer
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
class LambdaS3Deployer(BaseLambdaDeployer):
|
|
11
|
+
"""Deploy dotflow pipeline as Lambda + S3 Trigger."""
|
|
12
|
+
|
|
13
|
+
def _configure_trigger(self, name: str, **kwargs) -> None:
|
|
14
|
+
"""Create S3 bucket with Lambda notification."""
|
|
15
|
+
s3 = self._boto3.client("s3", region_name=self._region)
|
|
16
|
+
bucket_name = f"{name}-source"
|
|
17
|
+
|
|
18
|
+
print(f" Creating S3 bucket '{bucket_name}'...")
|
|
19
|
+
try:
|
|
20
|
+
if self._region == "us-east-1":
|
|
21
|
+
s3.create_bucket(Bucket=bucket_name)
|
|
22
|
+
else:
|
|
23
|
+
s3.create_bucket(
|
|
24
|
+
Bucket=bucket_name,
|
|
25
|
+
CreateBucketConfiguration={
|
|
26
|
+
"LocationConstraint": self._region
|
|
27
|
+
},
|
|
28
|
+
)
|
|
29
|
+
except s3.exceptions.BucketAlreadyOwnedByYou:
|
|
30
|
+
pass
|
|
31
|
+
|
|
32
|
+
with contextlib.suppress(
|
|
33
|
+
self._lambda.exceptions.ResourceConflictException
|
|
34
|
+
):
|
|
35
|
+
self._lambda.add_permission(
|
|
36
|
+
FunctionName=name,
|
|
37
|
+
StatementId=f"{name}-s3-invoke",
|
|
38
|
+
Action="lambda:InvokeFunction",
|
|
39
|
+
Principal="s3.amazonaws.com",
|
|
40
|
+
SourceArn=f"arn:aws:s3:::{bucket_name}",
|
|
41
|
+
)
|
|
42
|
+
|
|
43
|
+
print(" Configuring S3 notification...")
|
|
44
|
+
s3.put_bucket_notification_configuration(
|
|
45
|
+
Bucket=bucket_name,
|
|
46
|
+
NotificationConfiguration={
|
|
47
|
+
"LambdaFunctionConfigurations": [
|
|
48
|
+
{
|
|
49
|
+
"LambdaFunctionArn": self._function_arn,
|
|
50
|
+
"Events": ["s3:ObjectCreated:*"],
|
|
51
|
+
}
|
|
52
|
+
]
|
|
53
|
+
},
|
|
54
|
+
)
|
|
55
|
+
|
|
56
|
+
print(f" Trigger: s3://{bucket_name}/*")
|
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
"""Lambda + SQS Trigger deployment."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from dotflow.cloud.aws.base_lambda_deployer import BaseLambdaDeployer
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
class LambdaSQSDeployer(BaseLambdaDeployer):
|
|
9
|
+
"""Deploy dotflow pipeline as Lambda + SQS Trigger."""
|
|
10
|
+
|
|
11
|
+
def _configure_trigger(self, name: str, **kwargs) -> None:
|
|
12
|
+
"""Create SQS queue and event source mapping."""
|
|
13
|
+
sqs = self._boto3.client("sqs", region_name=self._region)
|
|
14
|
+
queue_name = f"{name}-queue"
|
|
15
|
+
|
|
16
|
+
print(f" Creating SQS queue '{queue_name}'...")
|
|
17
|
+
try:
|
|
18
|
+
response = sqs.create_queue(
|
|
19
|
+
QueueName=queue_name,
|
|
20
|
+
Attributes={"VisibilityTimeout": "960"},
|
|
21
|
+
)
|
|
22
|
+
queue_url = response["QueueUrl"]
|
|
23
|
+
except sqs.exceptions.QueueNameAlreadyExists:
|
|
24
|
+
queue_url = sqs.get_queue_url(QueueName=queue_name)["QueueUrl"]
|
|
25
|
+
|
|
26
|
+
queue_arn = sqs.get_queue_attributes(
|
|
27
|
+
QueueUrl=queue_url, AttributeNames=["QueueArn"]
|
|
28
|
+
)["Attributes"]["QueueArn"]
|
|
29
|
+
|
|
30
|
+
existing = self._lambda.list_event_source_mappings(
|
|
31
|
+
FunctionName=name, EventSourceArn=queue_arn
|
|
32
|
+
)["EventSourceMappings"]
|
|
33
|
+
|
|
34
|
+
if not existing:
|
|
35
|
+
print(" Creating event source mapping...")
|
|
36
|
+
self._lambda.create_event_source_mapping(
|
|
37
|
+
FunctionName=name,
|
|
38
|
+
EventSourceArn=queue_arn,
|
|
39
|
+
BatchSize=1,
|
|
40
|
+
Enabled=True,
|
|
41
|
+
)
|
|
42
|
+
|
|
43
|
+
print(f" Queue: {queue_url}")
|
|
@@ -45,6 +45,28 @@ class ObjectStorage(ABC):
|
|
|
45
45
|
"""Write data to a key."""
|
|
46
46
|
|
|
47
47
|
|
|
48
|
+
class Queue(ABC):
|
|
49
|
+
"""Base class for message queues (SQS, Pub/Sub, Azure Queue, etc.)."""
|
|
50
|
+
|
|
51
|
+
@abstractmethod
|
|
52
|
+
def create(self, name: str) -> str:
|
|
53
|
+
"""Create queue. Returns queue URL/ID."""
|
|
54
|
+
|
|
55
|
+
@abstractmethod
|
|
56
|
+
def get_arn(self, name: str) -> str:
|
|
57
|
+
"""Get queue ARN/ID."""
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
class EventScheduler(ABC):
|
|
61
|
+
"""Base class for event schedulers (EventBridge, Cloud Scheduler, etc.)."""
|
|
62
|
+
|
|
63
|
+
@abstractmethod
|
|
64
|
+
def create_schedule(
|
|
65
|
+
self, name: str, expression: str, target_arn: str
|
|
66
|
+
) -> None:
|
|
67
|
+
"""Create a scheduled rule targeting a resource."""
|
|
68
|
+
|
|
69
|
+
|
|
48
70
|
class Deployer(ABC):
|
|
49
71
|
"""Base class for all cloud deployers."""
|
|
50
72
|
|
|
@@ -0,0 +1,113 @@
|
|
|
1
|
+
"""GitHub Actions deployment."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import os
|
|
6
|
+
from pathlib import Path
|
|
7
|
+
|
|
8
|
+
from rich import print # type: ignore
|
|
9
|
+
|
|
10
|
+
from dotflow.cloud.core import Deployer
|
|
11
|
+
from dotflow.cloud.github.constants import PYGITHUB_NOT_FOUND, TOKEN_NOT_FOUND
|
|
12
|
+
from dotflow.settings import Settings as settings
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
class ActionsDeployer(Deployer):
|
|
16
|
+
"""Deploy dotflow pipeline to GitHub Actions."""
|
|
17
|
+
|
|
18
|
+
def __init__(self, token: str = None):
|
|
19
|
+
try:
|
|
20
|
+
from github import Github, GithubException
|
|
21
|
+
except ImportError as err:
|
|
22
|
+
raise SystemExit(PYGITHUB_NOT_FOUND) from err
|
|
23
|
+
|
|
24
|
+
self._token = token or os.environ.get("GITHUB_TOKEN")
|
|
25
|
+
if not self._token:
|
|
26
|
+
raise SystemExit(TOKEN_NOT_FOUND)
|
|
27
|
+
|
|
28
|
+
self._github = Github(self._token)
|
|
29
|
+
self._GithubException = GithubException
|
|
30
|
+
self._user = self._github.get_user()
|
|
31
|
+
|
|
32
|
+
def setup(self, name: str) -> None:
|
|
33
|
+
"""No-op — GitHub manages everything."""
|
|
34
|
+
|
|
35
|
+
def ensure_roles(self, name: str) -> str:
|
|
36
|
+
"""No-op."""
|
|
37
|
+
return ""
|
|
38
|
+
|
|
39
|
+
def ensure_logs(self, name: str) -> None:
|
|
40
|
+
"""No-op."""
|
|
41
|
+
|
|
42
|
+
def deploy(self, name: str, **kwargs) -> None:
|
|
43
|
+
"""Create repo and push all project files."""
|
|
44
|
+
print(settings.INFO_ALERT, f"Deploying to GitHub Actions '{name}'...")
|
|
45
|
+
|
|
46
|
+
repo = self._create_or_get_repo(name)
|
|
47
|
+
self._push_files(repo)
|
|
48
|
+
|
|
49
|
+
print(f" Repository: {repo.html_url}")
|
|
50
|
+
print(f" Actions: {repo.html_url}/actions")
|
|
51
|
+
print(settings.INFO_ALERT, "Done.")
|
|
52
|
+
|
|
53
|
+
def _create_or_get_repo(self, name: str):
|
|
54
|
+
"""Create repo or get existing."""
|
|
55
|
+
try:
|
|
56
|
+
repo = self._user.get_repo(name)
|
|
57
|
+
print(f" Using existing repo '{name}'")
|
|
58
|
+
return repo
|
|
59
|
+
except self._GithubException:
|
|
60
|
+
pass
|
|
61
|
+
|
|
62
|
+
print(f" Creating repo '{name}'...")
|
|
63
|
+
return self._user.create_repo(
|
|
64
|
+
name=name,
|
|
65
|
+
private=True,
|
|
66
|
+
auto_init=False,
|
|
67
|
+
)
|
|
68
|
+
|
|
69
|
+
def _push_files(self, repo):
|
|
70
|
+
"""Push all project files to the repo."""
|
|
71
|
+
print(" Pushing files...")
|
|
72
|
+
|
|
73
|
+
project_dir = Path.cwd()
|
|
74
|
+
count = 0
|
|
75
|
+
|
|
76
|
+
for filepath in sorted(project_dir.rglob("*")):
|
|
77
|
+
if not filepath.is_file():
|
|
78
|
+
continue
|
|
79
|
+
|
|
80
|
+
relative = str(filepath.relative_to(project_dir))
|
|
81
|
+
|
|
82
|
+
if relative.startswith(".") and not relative.startswith(".github"):
|
|
83
|
+
continue
|
|
84
|
+
|
|
85
|
+
if "__pycache__" in relative or ".egg-info" in relative:
|
|
86
|
+
continue
|
|
87
|
+
|
|
88
|
+
try:
|
|
89
|
+
content = filepath.read_text()
|
|
90
|
+
except UnicodeDecodeError:
|
|
91
|
+
continue
|
|
92
|
+
|
|
93
|
+
self._create_or_update_file(repo, relative, content)
|
|
94
|
+
count += 1
|
|
95
|
+
|
|
96
|
+
print(f" Pushed {count} files")
|
|
97
|
+
|
|
98
|
+
def _create_or_update_file(self, repo, path: str, content: str):
|
|
99
|
+
"""Create or update a single file in the repo."""
|
|
100
|
+
try:
|
|
101
|
+
existing = repo.get_contents(path)
|
|
102
|
+
repo.update_file(
|
|
103
|
+
path=path,
|
|
104
|
+
message=f"update {path}",
|
|
105
|
+
content=content,
|
|
106
|
+
sha=existing.sha,
|
|
107
|
+
)
|
|
108
|
+
except self._GithubException:
|
|
109
|
+
repo.create_file(
|
|
110
|
+
path=path,
|
|
111
|
+
message=f"add {path}",
|
|
112
|
+
content=content,
|
|
113
|
+
)
|
|
@@ -13,6 +13,7 @@ from dotflow.providers.tracer_default import TracerDefault
|
|
|
13
13
|
__all__ = [
|
|
14
14
|
"LogDefault",
|
|
15
15
|
"LogOpenTelemetry",
|
|
16
|
+
"LogSentry",
|
|
16
17
|
"NotifyDefault",
|
|
17
18
|
"NotifyDiscord",
|
|
18
19
|
"NotifyTelegram",
|
|
@@ -26,6 +27,7 @@ __all__ = [
|
|
|
26
27
|
"MetricsOpenTelemetry",
|
|
27
28
|
"TracerDefault",
|
|
28
29
|
"TracerOpenTelemetry",
|
|
30
|
+
"TracerSentry",
|
|
29
31
|
]
|
|
30
32
|
|
|
31
33
|
|
|
@@ -62,4 +64,14 @@ def __getattr__(name):
|
|
|
62
64
|
|
|
63
65
|
return LogOpenTelemetry
|
|
64
66
|
|
|
67
|
+
if name == "LogSentry":
|
|
68
|
+
from dotflow.providers.log_sentry import LogSentry
|
|
69
|
+
|
|
70
|
+
return LogSentry
|
|
71
|
+
|
|
72
|
+
if name == "TracerSentry":
|
|
73
|
+
from dotflow.providers.tracer_sentry import TracerSentry
|
|
74
|
+
|
|
75
|
+
return TracerSentry
|
|
76
|
+
|
|
65
77
|
raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
|
|
@@ -0,0 +1,96 @@
|
|
|
1
|
+
"""Log Sentry"""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from dotflow.abc.log import Log
|
|
6
|
+
from dotflow.core.exception import ModuleNotFound
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
class LogSentry(Log):
|
|
10
|
+
"""
|
|
11
|
+
Import:
|
|
12
|
+
You can import the **LogSentry** class with:
|
|
13
|
+
|
|
14
|
+
from dotflow.providers import LogSentry
|
|
15
|
+
|
|
16
|
+
Example:
|
|
17
|
+
`class` dotflow.providers.log_sentry.LogSentry
|
|
18
|
+
|
|
19
|
+
from dotflow import Config, DotFlow
|
|
20
|
+
from dotflow.providers import LogSentry
|
|
21
|
+
|
|
22
|
+
config = Config(
|
|
23
|
+
log=LogSentry(
|
|
24
|
+
dsn="https://xxx@sentry.io/123",
|
|
25
|
+
environment="production",
|
|
26
|
+
),
|
|
27
|
+
)
|
|
28
|
+
|
|
29
|
+
workflow = DotFlow(config=config)
|
|
30
|
+
|
|
31
|
+
Args:
|
|
32
|
+
dsn (str): Sentry DSN for the project.
|
|
33
|
+
environment (str): Environment tag sent to Sentry.
|
|
34
|
+
Defaults to None.
|
|
35
|
+
traces_sample_rate (float): Sample rate for performance traces (0.0 to 1.0).
|
|
36
|
+
Defaults to 0.0.
|
|
37
|
+
"""
|
|
38
|
+
|
|
39
|
+
def __init__(
|
|
40
|
+
self,
|
|
41
|
+
dsn: str,
|
|
42
|
+
environment: str | None = None,
|
|
43
|
+
traces_sample_rate: float = 0.0,
|
|
44
|
+
) -> None:
|
|
45
|
+
try:
|
|
46
|
+
import sentry_sdk
|
|
47
|
+
except ImportError as err:
|
|
48
|
+
raise ModuleNotFound(
|
|
49
|
+
module="sentry-sdk", library="dotflow[sentry]"
|
|
50
|
+
) from err
|
|
51
|
+
|
|
52
|
+
sentry_sdk.init(
|
|
53
|
+
dsn=dsn,
|
|
54
|
+
environment=environment,
|
|
55
|
+
traces_sample_rate=traces_sample_rate,
|
|
56
|
+
)
|
|
57
|
+
self._sentry = sentry_sdk
|
|
58
|
+
|
|
59
|
+
def info(self, **kwargs) -> None:
|
|
60
|
+
task = kwargs.get("task")
|
|
61
|
+
if task:
|
|
62
|
+
self._sentry.add_breadcrumb(
|
|
63
|
+
category="dotflow",
|
|
64
|
+
message=f"task:{task.task_id} {task.status}",
|
|
65
|
+
level="info",
|
|
66
|
+
)
|
|
67
|
+
|
|
68
|
+
def error(self, **kwargs) -> None:
|
|
69
|
+
task = kwargs.get("task")
|
|
70
|
+
if not task or not task.errors:
|
|
71
|
+
return
|
|
72
|
+
|
|
73
|
+
last_error = task.errors[-1]
|
|
74
|
+
self._sentry.capture_message(
|
|
75
|
+
f"Task {task.task_id} failed: {last_error.message}",
|
|
76
|
+
level="error",
|
|
77
|
+
extras={
|
|
78
|
+
"workflow_id": str(task.workflow_id),
|
|
79
|
+
"task_id": str(task.task_id),
|
|
80
|
+
"exception": str(last_error.exception),
|
|
81
|
+
"attempt": last_error.attempt,
|
|
82
|
+
"traceback": last_error.traceback,
|
|
83
|
+
},
|
|
84
|
+
)
|
|
85
|
+
|
|
86
|
+
def warning(self, **kwargs) -> None:
|
|
87
|
+
task = kwargs.get("task")
|
|
88
|
+
if task:
|
|
89
|
+
self._sentry.add_breadcrumb(
|
|
90
|
+
category="dotflow",
|
|
91
|
+
message=f"task:{task.task_id} {task.status}",
|
|
92
|
+
level="warning",
|
|
93
|
+
)
|
|
94
|
+
|
|
95
|
+
def debug(self, **kwargs) -> None:
|
|
96
|
+
pass
|
|
@@ -0,0 +1,126 @@
|
|
|
1
|
+
"""Tracer Sentry"""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from typing import Any
|
|
6
|
+
|
|
7
|
+
from dotflow.abc.tracer import Tracer
|
|
8
|
+
from dotflow.core.exception import ModuleNotFound
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
class TracerSentry(Tracer):
|
|
12
|
+
"""
|
|
13
|
+
Import:
|
|
14
|
+
You can import the **TracerSentry** class with:
|
|
15
|
+
|
|
16
|
+
from dotflow.providers import TracerSentry
|
|
17
|
+
|
|
18
|
+
Example:
|
|
19
|
+
`class` dotflow.providers.tracer_sentry.TracerSentry
|
|
20
|
+
|
|
21
|
+
from dotflow import Config, DotFlow
|
|
22
|
+
from dotflow.providers import TracerSentry
|
|
23
|
+
|
|
24
|
+
config = Config(
|
|
25
|
+
tracer=TracerSentry(),
|
|
26
|
+
)
|
|
27
|
+
|
|
28
|
+
workflow = DotFlow(config=config)
|
|
29
|
+
|
|
30
|
+
Args:
|
|
31
|
+
dsn (str): Sentry DSN for the project. If None, uses the SDK
|
|
32
|
+
already initialized by LogSentry or manually.
|
|
33
|
+
environment (str): Environment tag sent to Sentry.
|
|
34
|
+
Defaults to None.
|
|
35
|
+
traces_sample_rate (float): Sample rate for performance traces (0.0 to 1.0).
|
|
36
|
+
Defaults to 1.0.
|
|
37
|
+
"""
|
|
38
|
+
|
|
39
|
+
def __init__(
|
|
40
|
+
self,
|
|
41
|
+
dsn: str | None = None,
|
|
42
|
+
environment: str | None = None,
|
|
43
|
+
traces_sample_rate: float = 1.0,
|
|
44
|
+
) -> None:
|
|
45
|
+
try:
|
|
46
|
+
import sentry_sdk
|
|
47
|
+
except ImportError as err:
|
|
48
|
+
raise ModuleNotFound(
|
|
49
|
+
module="sentry-sdk", library="dotflow[sentry]"
|
|
50
|
+
) from err
|
|
51
|
+
|
|
52
|
+
if dsn:
|
|
53
|
+
sentry_sdk.init(
|
|
54
|
+
dsn=dsn,
|
|
55
|
+
environment=environment,
|
|
56
|
+
traces_sample_rate=traces_sample_rate,
|
|
57
|
+
)
|
|
58
|
+
|
|
59
|
+
self._sentry = sentry_sdk
|
|
60
|
+
self._transactions: dict[str, Any] = {}
|
|
61
|
+
self._spans: dict[str, Any] = {}
|
|
62
|
+
|
|
63
|
+
def start_workflow(self, workflow_id: Any, **kwargs) -> None:
|
|
64
|
+
key = str(workflow_id)
|
|
65
|
+
transaction = self._sentry.start_transaction(
|
|
66
|
+
op="workflow",
|
|
67
|
+
name=f"workflow:{key}",
|
|
68
|
+
)
|
|
69
|
+
transaction.set_tag("dotflow.workflow_id", key)
|
|
70
|
+
|
|
71
|
+
for k, v in kwargs.items():
|
|
72
|
+
transaction.set_tag(f"dotflow.{k}", str(v))
|
|
73
|
+
|
|
74
|
+
self._transactions[key] = transaction
|
|
75
|
+
|
|
76
|
+
def end_workflow(self, workflow_id: Any, **kwargs) -> None:
|
|
77
|
+
key = str(workflow_id)
|
|
78
|
+
transaction = self._transactions.pop(key, None)
|
|
79
|
+
if not transaction:
|
|
80
|
+
return
|
|
81
|
+
|
|
82
|
+
failed = kwargs.get("failed", False)
|
|
83
|
+
transaction.set_status("internal_error" if failed else "ok")
|
|
84
|
+
|
|
85
|
+
if "duration" in kwargs:
|
|
86
|
+
transaction.set_data("dotflow.duration", kwargs["duration"])
|
|
87
|
+
|
|
88
|
+
transaction.finish()
|
|
89
|
+
|
|
90
|
+
def start_task(self, task: Any) -> None:
|
|
91
|
+
workflow_key = str(task.workflow_id)
|
|
92
|
+
task_key = f"{workflow_key}:{task.task_id}"
|
|
93
|
+
|
|
94
|
+
transaction = self._transactions.get(workflow_key)
|
|
95
|
+
if not transaction:
|
|
96
|
+
return
|
|
97
|
+
|
|
98
|
+
span = transaction.start_child(
|
|
99
|
+
op="task",
|
|
100
|
+
name=f"task:{task.task_id}",
|
|
101
|
+
)
|
|
102
|
+
span.set_tag("dotflow.workflow_id", workflow_key)
|
|
103
|
+
span.set_tag("dotflow.task_id", str(task.task_id))
|
|
104
|
+
self._spans[task_key] = span
|
|
105
|
+
|
|
106
|
+
def end_task(self, task: Any) -> None:
|
|
107
|
+
task_key = f"{task.workflow_id}:{task.task_id}"
|
|
108
|
+
span = self._spans.pop(task_key, None)
|
|
109
|
+
if not span:
|
|
110
|
+
return
|
|
111
|
+
|
|
112
|
+
span.set_tag("dotflow.task.status", str(task.status))
|
|
113
|
+
|
|
114
|
+
if task.duration is not None:
|
|
115
|
+
span.set_data("dotflow.task.duration", task.duration)
|
|
116
|
+
if task.retry_count:
|
|
117
|
+
span.set_data("dotflow.task.retry_count", task.retry_count)
|
|
118
|
+
|
|
119
|
+
if task.errors:
|
|
120
|
+
last_error = task.errors[-1]
|
|
121
|
+
span.set_data("dotflow.task.exception", str(last_error.exception))
|
|
122
|
+
span.set_status("internal_error")
|
|
123
|
+
else:
|
|
124
|
+
span.set_status("ok")
|
|
125
|
+
|
|
126
|
+
span.finish()
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
[project]
|
|
2
2
|
name = "dotflow"
|
|
3
|
-
version = "0.15.0.
|
|
3
|
+
version = "0.15.0.dev2"
|
|
4
4
|
authors = [
|
|
5
5
|
{ name="Fernando Celmer", email="email@fernandocelmer.com" },
|
|
6
6
|
]
|
|
@@ -58,12 +58,14 @@ aws = ["boto3"]
|
|
|
58
58
|
gcp = ["google-cloud-storage"]
|
|
59
59
|
deploy-aws = ["boto3"]
|
|
60
60
|
deploy-gcp = ["google-cloud-run", "google-cloud-build", "google-cloud-artifact-registry", "google-cloud-service-usage", "google-cloud-storage", "google-auth"]
|
|
61
|
+
deploy-github = ["PyGithub"]
|
|
61
62
|
scheduler = ["croniter"]
|
|
62
63
|
otel = ["opentelemetry-api", "opentelemetry-sdk"]
|
|
64
|
+
sentry = ["sentry-sdk"]
|
|
63
65
|
|
|
64
66
|
[tool.poetry]
|
|
65
67
|
name = "dotflow"
|
|
66
|
-
version = "0.15.0.
|
|
68
|
+
version = "0.15.0.dev2"
|
|
67
69
|
description = "🎲 Dotflow turns an idea into flow! Lightweight Python library for execution pipelines with retry, parallel, cron and async support."
|
|
68
70
|
authors = ["Fernando Celmer <email@fernandocelmer.com>"]
|
|
69
71
|
readme = "README.md"
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|