dotflow 0.15.0.dev3__py3-none-any.whl → 0.15.0.dev4__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- dotflow/__init__.py +1 -1
- dotflow/abc/server.py +30 -0
- dotflow/cli/commands/cloud.py +1 -0
- dotflow/cli/commands/deploy.py +128 -67
- dotflow/cli/commands/log.py +2 -1
- dotflow/cli/setup.py +5 -2
- dotflow/cloud/alibaba/__init__.py +11 -0
- dotflow/cloud/alibaba/constants.py +22 -0
- dotflow/cloud/alibaba/deployers/__init__.py +13 -0
- dotflow/cloud/alibaba/deployers/fc.py +134 -0
- dotflow/cloud/alibaba/deployers/fc_scheduled.py +78 -0
- dotflow/cloud/alibaba/services/__init__.py +1 -0
- dotflow/cloud/alibaba/services/cr.py +99 -0
- dotflow/cloud/aws/__init__.py +11 -7
- dotflow/cloud/aws/constants.py +15 -1
- dotflow/cloud/aws/deployers/__init__.py +19 -0
- dotflow/cloud/aws/{base_lambda_deployer.py → deployers/base_lambda.py} +6 -9
- dotflow/cloud/aws/{ecs_deployer.py → deployers/ecs.py} +11 -3
- dotflow/cloud/aws/deployers/ecs_scheduled.py +154 -0
- dotflow/cloud/aws/{lambda_deployer.py → deployers/lambda_.py} +9 -3
- dotflow/cloud/aws/{lambda_api_deployer.py → deployers/lambda_api.py} +6 -3
- dotflow/cloud/aws/{lambda_s3_deployer.py → deployers/lambda_s3.py} +10 -4
- dotflow/cloud/aws/{lambda_sqs_deployer.py → deployers/lambda_sqs.py} +19 -4
- dotflow/cloud/aws/schedule.py +66 -0
- dotflow/cloud/aws/services/cloudwatch.py +4 -1
- dotflow/cloud/aws/services/ecr.py +8 -5
- dotflow/cloud/aws/services/iam.py +4 -1
- dotflow/cloud/core.py +8 -2
- dotflow/cloud/gcp/__init__.py +1 -1
- dotflow/cloud/gcp/constants.py +9 -0
- dotflow/cloud/gcp/deployers/__init__.py +5 -0
- dotflow/cloud/gcp/{cloudrun_deployer.py → deployers/cloudrun.py} +12 -10
- dotflow/cloud/gcp/services/apis.py +8 -2
- dotflow/cloud/gcp/services/artifact_registry.py +16 -5
- dotflow/cloud/github/__init__.py +1 -1
- dotflow/cloud/github/deployers/__init__.py +5 -0
- dotflow/cloud/github/{actions_deployer.py → deployers/actions.py} +10 -15
- dotflow/core/config.py +5 -5
- dotflow/core/dotflow.py +1 -1
- dotflow/core/engine.py +5 -2
- dotflow/core/serializers/task.py +52 -34
- dotflow/core/serializers/transport.py +2 -2
- dotflow/core/serializers/workflow.py +1 -10
- dotflow/core/task.py +10 -2
- dotflow/core/types/workflow.py +1 -0
- dotflow/core/workflow.py +23 -5
- dotflow/providers/__init__.py +2 -0
- dotflow/providers/server_default.py +112 -0
- dotflow/settings.py +2 -0
- {dotflow-0.15.0.dev3.dist-info → dotflow-0.15.0.dev4.dist-info}/METADATA +77 -22
- {dotflow-0.15.0.dev3.dist-info → dotflow-0.15.0.dev4.dist-info}/RECORD +54 -42
- dotflow/abc/api.py +0 -24
- dotflow/providers/api_default.py +0 -158
- {dotflow-0.15.0.dev3.dist-info → dotflow-0.15.0.dev4.dist-info}/WHEEL +0 -0
- {dotflow-0.15.0.dev3.dist-info → dotflow-0.15.0.dev4.dist-info}/entry_points.txt +0 -0
- {dotflow-0.15.0.dev3.dist-info → dotflow-0.15.0.dev4.dist-info}/licenses/LICENSE +0 -0
dotflow/__init__.py
CHANGED
dotflow/abc/server.py
ADDED
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
"""Abstract base for remote server communication."""
|
|
2
|
+
|
|
3
|
+
from abc import ABC, abstractmethod
|
|
4
|
+
from typing import Any
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
class Server(ABC):
|
|
8
|
+
"""Server ABC provider for sending workflow and task
|
|
9
|
+
data to a remote API.
|
|
10
|
+
|
|
11
|
+
Implementations should handle HTTP communication with
|
|
12
|
+
a server like dotflow-api, sending execution data
|
|
13
|
+
(status, duration, errors, context) in real time.
|
|
14
|
+
"""
|
|
15
|
+
|
|
16
|
+
@abstractmethod
|
|
17
|
+
def create_workflow(self, workflow: Any) -> None:
|
|
18
|
+
"""Register a new workflow on the remote server."""
|
|
19
|
+
|
|
20
|
+
@abstractmethod
|
|
21
|
+
def update_workflow(self, workflow: Any, status: str = "") -> None:
|
|
22
|
+
"""Update workflow status on the remote server."""
|
|
23
|
+
|
|
24
|
+
@abstractmethod
|
|
25
|
+
def create_task(self, task: Any) -> None:
|
|
26
|
+
"""Register a new task on the remote server."""
|
|
27
|
+
|
|
28
|
+
@abstractmethod
|
|
29
|
+
def update_task(self, task: Any) -> None:
|
|
30
|
+
"""Update task data on the remote server."""
|
dotflow/cli/commands/cloud.py
CHANGED
dotflow/cli/commands/deploy.py
CHANGED
|
@@ -1,110 +1,171 @@
|
|
|
1
1
|
"""Command deploy module."""
|
|
2
2
|
|
|
3
3
|
from rich import print # type: ignore
|
|
4
|
+
from rich.prompt import Prompt
|
|
4
5
|
|
|
5
6
|
from dotflow.cli.command import Command
|
|
7
|
+
from dotflow.cloud.alibaba.constants import (
|
|
8
|
+
DEFAULT_REGION as ALI_DEFAULT_REGION,
|
|
9
|
+
)
|
|
10
|
+
from dotflow.cloud.alibaba.constants import PLATFORMS as ALI_PLATFORMS
|
|
11
|
+
from dotflow.cloud.alibaba.constants import (
|
|
12
|
+
SCHEDULED_PLATFORMS as ALI_SCHEDULED,
|
|
13
|
+
)
|
|
14
|
+
from dotflow.cloud.aws.constants import DEFAULT_REGION as AWS_DEFAULT_REGION
|
|
15
|
+
from dotflow.cloud.aws.constants import PLATFORMS as AWS_PLATFORMS
|
|
16
|
+
from dotflow.cloud.aws.constants import SCHEDULED_PLATFORMS as AWS_SCHEDULED
|
|
17
|
+
from dotflow.cloud.gcp.constants import DEFAULT_REGION as GCP_DEFAULT_REGION
|
|
18
|
+
from dotflow.cloud.gcp.constants import PLATFORMS as GCP_PLATFORMS
|
|
19
|
+
from dotflow.cloud.gcp.constants import SCHEDULED_PLATFORMS as GCP_SCHEDULED
|
|
6
20
|
from dotflow.settings import Settings as settings
|
|
7
21
|
|
|
8
|
-
SAM_PLATFORMS = {
|
|
9
|
-
"ecs-scheduled",
|
|
10
|
-
}
|
|
11
|
-
|
|
12
22
|
DEFAULT_REGIONS = {
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
"lambda-sqs-trigger": "us-east-1",
|
|
17
|
-
"lambda-api-trigger": "us-east-1",
|
|
18
|
-
"ecs": "us-east-1",
|
|
19
|
-
"cloud-run": "us-central1",
|
|
20
|
-
"cloud-run-scheduled": "us-central1",
|
|
23
|
+
**dict.fromkeys(AWS_PLATFORMS, AWS_DEFAULT_REGION),
|
|
24
|
+
**dict.fromkeys(GCP_PLATFORMS, GCP_DEFAULT_REGION),
|
|
25
|
+
**dict.fromkeys(ALI_PLATFORMS, ALI_DEFAULT_REGION),
|
|
21
26
|
}
|
|
22
27
|
|
|
28
|
+
SCHEDULED_PLATFORMS = AWS_SCHEDULED | GCP_SCHEDULED | ALI_SCHEDULED
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
class ScheduleResolver:
|
|
32
|
+
"""Resolves a schedule expression.
|
|
33
|
+
|
|
34
|
+
From CLI args, template, or user input.
|
|
35
|
+
"""
|
|
36
|
+
|
|
37
|
+
@classmethod
|
|
38
|
+
def _get_provider(cls, platform: str) -> type | None:
|
|
39
|
+
if platform in AWS_PLATFORMS:
|
|
40
|
+
from dotflow.cloud.aws.schedule import AWSSchedule
|
|
41
|
+
|
|
42
|
+
return AWSSchedule
|
|
43
|
+
|
|
44
|
+
return None
|
|
45
|
+
|
|
46
|
+
@classmethod
|
|
47
|
+
def resolve(cls, schedule: str | None, platform: str) -> str | None:
|
|
48
|
+
if platform not in SCHEDULED_PLATFORMS:
|
|
49
|
+
return schedule
|
|
50
|
+
|
|
51
|
+
provider = cls._get_provider(platform)
|
|
52
|
+
|
|
53
|
+
if schedule:
|
|
54
|
+
return provider.convert(schedule) if provider else schedule
|
|
55
|
+
|
|
56
|
+
raw = cls._ask_schedule(provider)
|
|
57
|
+
|
|
58
|
+
if not raw:
|
|
59
|
+
raise SystemExit(
|
|
60
|
+
f"{settings.ERROR_ALERT} --schedule (cron) is required for {platform}"
|
|
61
|
+
)
|
|
62
|
+
|
|
63
|
+
return provider.convert(raw) if provider else raw
|
|
64
|
+
|
|
65
|
+
@classmethod
|
|
66
|
+
def _ask_schedule(cls, provider: type | None) -> str | None:
|
|
67
|
+
template_schedule = None
|
|
68
|
+
if provider:
|
|
69
|
+
template_schedule = provider.read_from_template()
|
|
70
|
+
|
|
71
|
+
choices = {}
|
|
72
|
+
if template_schedule:
|
|
73
|
+
choices["1"] = f"Use from template.yaml: {template_schedule}"
|
|
74
|
+
choices["2"] = "Enter cron expression"
|
|
75
|
+
|
|
76
|
+
print(settings.QUESTION_ALERT, "Cron schedule is required:")
|
|
77
|
+
for key, label in choices.items():
|
|
78
|
+
print(f" [bold cyan]{key}[/bold cyan] - {label}")
|
|
79
|
+
|
|
80
|
+
choice = Prompt.ask(" Select", choices=list(choices.keys()))
|
|
81
|
+
|
|
82
|
+
if choice == "1" and template_schedule:
|
|
83
|
+
print(settings.INFO_ALERT, f"Using cron: {template_schedule}")
|
|
84
|
+
return template_schedule
|
|
85
|
+
|
|
86
|
+
print(" Format: [bold]min hour day month weekday[/bold]")
|
|
87
|
+
return Prompt.ask(" Cron (e.g. */5 * * * *)") or None
|
|
88
|
+
|
|
23
89
|
|
|
24
90
|
class DeployCommand(Command):
|
|
25
91
|
def setup(self):
|
|
26
92
|
platform = self.params.platform
|
|
27
93
|
name = self.params.project
|
|
28
|
-
region = self.params.region or DEFAULT_REGIONS.get(
|
|
29
|
-
|
|
94
|
+
region = self.params.region or DEFAULT_REGIONS.get(platform)
|
|
95
|
+
schedule = ScheduleResolver.resolve(
|
|
96
|
+
getattr(self.params, "schedule", None), platform
|
|
30
97
|
)
|
|
31
|
-
schedule = getattr(self.params, "schedule", None)
|
|
32
|
-
|
|
33
|
-
if platform in SAM_PLATFORMS:
|
|
34
|
-
print(
|
|
35
|
-
settings.INFO_ALERT,
|
|
36
|
-
f"'{platform}' requires SAM for full trigger setup.",
|
|
37
|
-
)
|
|
38
|
-
print(" Run: sam build && sam deploy")
|
|
39
|
-
return
|
|
40
98
|
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
"lambda-scheduled": self._deploy_lambda,
|
|
44
|
-
"lambda-s3-trigger": self._deploy_lambda_s3,
|
|
45
|
-
"lambda-sqs-trigger": self._deploy_lambda_sqs,
|
|
46
|
-
"lambda-api-trigger": self._deploy_lambda_api,
|
|
47
|
-
"ecs": self._deploy_ecs,
|
|
48
|
-
"cloud-run": self._deploy_cloud_run,
|
|
49
|
-
"cloud-run-scheduled": self._deploy_cloud_run,
|
|
50
|
-
"github-actions": self._deploy_github_actions,
|
|
51
|
-
}
|
|
52
|
-
|
|
53
|
-
handler = deployers.get(platform)
|
|
99
|
+
method_name = f"_deploy_{platform.replace('-', '_')}"
|
|
100
|
+
handler = getattr(self, method_name, None)
|
|
54
101
|
if not handler:
|
|
55
102
|
print(
|
|
56
|
-
settings.ERROR_ALERT,
|
|
57
|
-
f"Deploy not supported for '{platform}'.",
|
|
103
|
+
settings.ERROR_ALERT, f"Deploy not supported for '{platform}'."
|
|
58
104
|
)
|
|
59
105
|
return
|
|
60
106
|
|
|
61
107
|
handler(name=name, region=region, schedule=schedule)
|
|
62
108
|
|
|
63
|
-
def _deploy_lambda(self, name
|
|
64
|
-
"""Deploy to AWS Lambda."""
|
|
109
|
+
def _deploy_lambda(self, name, region, schedule=None):
|
|
65
110
|
from dotflow.cloud.aws import LambdaDeployer
|
|
66
111
|
|
|
67
|
-
|
|
68
|
-
deployer.deploy(name, schedule=schedule)
|
|
112
|
+
LambdaDeployer(region=region).deploy(name, schedule=schedule)
|
|
69
113
|
|
|
70
|
-
def
|
|
71
|
-
|
|
72
|
-
from dotflow.cloud.aws import LambdaApiDeployer
|
|
114
|
+
def _deploy_lambda_scheduled(self, name, region, schedule=None):
|
|
115
|
+
from dotflow.cloud.aws import LambdaDeployer
|
|
73
116
|
|
|
74
|
-
|
|
75
|
-
deployer.deploy(name)
|
|
117
|
+
LambdaDeployer(region=region).deploy(name, schedule=schedule)
|
|
76
118
|
|
|
77
|
-
def
|
|
78
|
-
"""Deploy to AWS Lambda + S3 Trigger."""
|
|
119
|
+
def _deploy_lambda_s3_trigger(self, name, region, **kwargs):
|
|
79
120
|
from dotflow.cloud.aws import LambdaS3Deployer
|
|
80
121
|
|
|
81
|
-
|
|
82
|
-
deployer.deploy(name)
|
|
122
|
+
LambdaS3Deployer(region=region).deploy(name)
|
|
83
123
|
|
|
84
|
-
def
|
|
85
|
-
"""Deploy to AWS Lambda + SQS Trigger."""
|
|
124
|
+
def _deploy_lambda_sqs_trigger(self, name, region, **kwargs):
|
|
86
125
|
from dotflow.cloud.aws import LambdaSQSDeployer
|
|
87
126
|
|
|
88
|
-
|
|
89
|
-
|
|
127
|
+
LambdaSQSDeployer(region=region).deploy(name)
|
|
128
|
+
|
|
129
|
+
def _deploy_lambda_api_trigger(self, name, region, **kwargs):
|
|
130
|
+
from dotflow.cloud.aws import LambdaApiDeployer
|
|
90
131
|
|
|
91
|
-
|
|
92
|
-
|
|
132
|
+
LambdaApiDeployer(region=region).deploy(name)
|
|
133
|
+
|
|
134
|
+
def _deploy_ecs(self, name, region, **kwargs):
|
|
93
135
|
from dotflow.cloud.aws import ECSDeployer
|
|
94
136
|
|
|
95
|
-
|
|
96
|
-
|
|
137
|
+
ECSDeployer(region=region).deploy(name)
|
|
138
|
+
|
|
139
|
+
def _deploy_ecs_scheduled(self, name, region, schedule=None):
|
|
140
|
+
from dotflow.cloud.aws import ECSScheduledDeployer
|
|
97
141
|
|
|
98
|
-
|
|
99
|
-
|
|
142
|
+
ECSScheduledDeployer(region=region).deploy(name, schedule=schedule)
|
|
143
|
+
|
|
144
|
+
def _deploy_cloud_run(self, name, region, **kwargs):
|
|
100
145
|
from dotflow.cloud.gcp import CloudRunDeployer
|
|
101
146
|
|
|
102
|
-
|
|
103
|
-
|
|
147
|
+
CloudRunDeployer(region=region).deploy(name)
|
|
148
|
+
|
|
149
|
+
def _deploy_cloud_run_scheduled(self, name, region, **kwargs):
|
|
150
|
+
from dotflow.cloud.gcp import CloudRunDeployer
|
|
104
151
|
|
|
105
|
-
|
|
106
|
-
|
|
152
|
+
CloudRunDeployer(region=region).deploy(name)
|
|
153
|
+
|
|
154
|
+
def _deploy_github_actions(self, name, **kwargs):
|
|
107
155
|
from dotflow.cloud.github import ActionsDeployer
|
|
108
156
|
|
|
109
|
-
|
|
110
|
-
|
|
157
|
+
ActionsDeployer().deploy(name)
|
|
158
|
+
|
|
159
|
+
def _deploy_alibaba_fc(self, name, region, **kwargs):
|
|
160
|
+
from dotflow.cloud.alibaba import AliyunFCDeployer
|
|
161
|
+
|
|
162
|
+
AliyunFCDeployer(region=region).deploy(name)
|
|
163
|
+
|
|
164
|
+
def _deploy_alibaba_fc_scheduled(self, name, region, schedule=None):
|
|
165
|
+
from dotflow.cloud.alibaba import (
|
|
166
|
+
AliyunFCScheduledDeployer,
|
|
167
|
+
)
|
|
168
|
+
|
|
169
|
+
AliyunFCScheduledDeployer(region=region).deploy(
|
|
170
|
+
name, schedule=schedule
|
|
171
|
+
)
|
dotflow/cli/commands/log.py
CHANGED
|
@@ -14,7 +14,8 @@ class LogCommand(Command):
|
|
|
14
14
|
|
|
15
15
|
print(
|
|
16
16
|
settings.INFO_ALERT,
|
|
17
|
-
|
|
17
|
+
"To access all logs, open the file"
|
|
18
|
+
f" ({settings.LOG_PATH.resolve()}).",
|
|
18
19
|
)
|
|
19
20
|
else:
|
|
20
21
|
print(settings.WARNING_ALERT, "Log file not found.")
|
dotflow/cli/setup.py
CHANGED
|
@@ -163,7 +163,7 @@ class Command:
|
|
|
163
163
|
cmd_generate.add_argument(
|
|
164
164
|
"--platform",
|
|
165
165
|
required=True,
|
|
166
|
-
help="Target platform (e.g. docker, lambda, cloud-run
|
|
166
|
+
help=("Target platform (e.g. docker, lambda, cloud-run)"),
|
|
167
167
|
)
|
|
168
168
|
cmd_generate.add_argument(
|
|
169
169
|
"--project",
|
|
@@ -206,7 +206,7 @@ class Command:
|
|
|
206
206
|
self.cmd_deploy.add_argument(
|
|
207
207
|
"--schedule",
|
|
208
208
|
default=None,
|
|
209
|
-
help="
|
|
209
|
+
help="Cron expression (e.g. '*/5 * * * *')",
|
|
210
210
|
)
|
|
211
211
|
self.cmd_deploy.set_defaults(exec=DeployCommand)
|
|
212
212
|
|
|
@@ -233,6 +233,9 @@ class Command:
|
|
|
233
233
|
except ImportModuleError as err:
|
|
234
234
|
print(settings.WARNING_ALERT, err)
|
|
235
235
|
|
|
236
|
+
except KeyboardInterrupt:
|
|
237
|
+
print("\n", settings.INFO_ALERT, "Aborted.")
|
|
238
|
+
|
|
236
239
|
except Exception as err:
|
|
237
240
|
logger.error(f"Internal problem: {str(err)}")
|
|
238
241
|
print(settings.ERROR_ALERT, MESSAGE_UNKNOWN_ERROR)
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
"""Alibaba Cloud constants."""
|
|
2
|
+
|
|
3
|
+
SDK_NOT_FOUND = (
|
|
4
|
+
"alibabacloud-fc-open20210406 is required: "
|
|
5
|
+
"pip install dotflow[deploy-alibaba]"
|
|
6
|
+
)
|
|
7
|
+
|
|
8
|
+
CREDENTIALS_NOT_FOUND = (
|
|
9
|
+
"Alibaba Cloud credentials not found. "
|
|
10
|
+
"Run 'aliyun configure' or set "
|
|
11
|
+
"ALIBABA_CLOUD_ACCESS_KEY_ID and "
|
|
12
|
+
"ALIBABA_CLOUD_ACCESS_KEY_SECRET."
|
|
13
|
+
)
|
|
14
|
+
|
|
15
|
+
PLATFORMS = {
|
|
16
|
+
"alibaba-fc",
|
|
17
|
+
"alibaba-fc-scheduled",
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
SCHEDULED_PLATFORMS = {"alibaba-fc-scheduled"}
|
|
21
|
+
|
|
22
|
+
DEFAULT_REGION = "cn-hangzhou"
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
"""Alibaba Cloud deployers."""
|
|
2
|
+
|
|
3
|
+
from dotflow.cloud.alibaba.deployers.fc import (
|
|
4
|
+
AliyunFCDeployer,
|
|
5
|
+
)
|
|
6
|
+
from dotflow.cloud.alibaba.deployers.fc_scheduled import (
|
|
7
|
+
AliyunFCScheduledDeployer,
|
|
8
|
+
)
|
|
9
|
+
|
|
10
|
+
__all__ = [
|
|
11
|
+
"AliyunFCDeployer",
|
|
12
|
+
"AliyunFCScheduledDeployer",
|
|
13
|
+
]
|
|
@@ -0,0 +1,134 @@
|
|
|
1
|
+
"""Alibaba Function Compute deployment."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import os
|
|
6
|
+
|
|
7
|
+
from rich import print # type: ignore
|
|
8
|
+
|
|
9
|
+
from dotflow.cloud.alibaba.constants import (
|
|
10
|
+
CREDENTIALS_NOT_FOUND,
|
|
11
|
+
SDK_NOT_FOUND,
|
|
12
|
+
)
|
|
13
|
+
from dotflow.cloud.alibaba.services.cr import ContainerRegistry
|
|
14
|
+
from dotflow.cloud.core import Deployer
|
|
15
|
+
from dotflow.settings import Settings as settings
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
class AliyunFCDeployer(Deployer):
|
|
19
|
+
"""Deploy dotflow pipeline to Alibaba Cloud FC."""
|
|
20
|
+
|
|
21
|
+
def __init__(
|
|
22
|
+
self,
|
|
23
|
+
region: str = "cn-hangzhou",
|
|
24
|
+
namespace: str = "dotflow",
|
|
25
|
+
):
|
|
26
|
+
try:
|
|
27
|
+
from alibabacloud_fc_open20210406.client import (
|
|
28
|
+
Client,
|
|
29
|
+
)
|
|
30
|
+
from alibabacloud_tea_openapi.models import (
|
|
31
|
+
Config,
|
|
32
|
+
)
|
|
33
|
+
except ImportError as err:
|
|
34
|
+
raise SystemExit(SDK_NOT_FOUND) from err
|
|
35
|
+
|
|
36
|
+
key_id = os.environ.get("ALIBABA_CLOUD_ACCESS_KEY_ID")
|
|
37
|
+
key_secret = os.environ.get("ALIBABA_CLOUD_ACCESS_KEY_SECRET")
|
|
38
|
+
|
|
39
|
+
if not key_id or not key_secret:
|
|
40
|
+
try:
|
|
41
|
+
import json
|
|
42
|
+
from pathlib import Path
|
|
43
|
+
|
|
44
|
+
config_path = Path.home() / ".aliyun" / "config.json"
|
|
45
|
+
data = json.loads(config_path.read_text())
|
|
46
|
+
for p in data.get("profiles", []):
|
|
47
|
+
if p.get("name") == data.get("current"):
|
|
48
|
+
key_id = p.get("access_key_id")
|
|
49
|
+
key_secret = p.get("access_key_secret")
|
|
50
|
+
if not region or region == "cn-hangzhou":
|
|
51
|
+
region = p.get("region_id", region)
|
|
52
|
+
break
|
|
53
|
+
except Exception:
|
|
54
|
+
pass
|
|
55
|
+
|
|
56
|
+
if not key_id or not key_secret:
|
|
57
|
+
raise SystemExit(CREDENTIALS_NOT_FOUND)
|
|
58
|
+
|
|
59
|
+
self._region = region
|
|
60
|
+
self._namespace = namespace
|
|
61
|
+
|
|
62
|
+
config = Config(
|
|
63
|
+
access_key_id=key_id,
|
|
64
|
+
access_key_secret=key_secret,
|
|
65
|
+
region_id=region,
|
|
66
|
+
)
|
|
67
|
+
config.endpoint = f"{self._region}.fc.aliyuncs.com"
|
|
68
|
+
self._fc = Client(config)
|
|
69
|
+
|
|
70
|
+
self._cr = ContainerRegistry(
|
|
71
|
+
region=region,
|
|
72
|
+
namespace=namespace,
|
|
73
|
+
)
|
|
74
|
+
|
|
75
|
+
@staticmethod
|
|
76
|
+
def _sanitize_name(name: str) -> str:
|
|
77
|
+
return name.replace("_", "-").lower()
|
|
78
|
+
|
|
79
|
+
def deploy(self, name: str, **kwargs) -> None:
|
|
80
|
+
"""Deploy to Alibaba Function Compute."""
|
|
81
|
+
name = self._sanitize_name(name)
|
|
82
|
+
print(
|
|
83
|
+
settings.INFO_ALERT,
|
|
84
|
+
f"Deploying to Alibaba FC '{name}'...",
|
|
85
|
+
)
|
|
86
|
+
|
|
87
|
+
image_uri = self._cr.push(name)
|
|
88
|
+
self._create_or_update_function(name, image_uri)
|
|
89
|
+
|
|
90
|
+
print(settings.INFO_ALERT, "Done.")
|
|
91
|
+
|
|
92
|
+
def _create_or_update_function(self, name: str, image_uri: str) -> None:
|
|
93
|
+
"""Create or update FC function."""
|
|
94
|
+
from alibabacloud_fc_open20210406 import models
|
|
95
|
+
|
|
96
|
+
try:
|
|
97
|
+
self._fc.get_function(name)
|
|
98
|
+
print(f" {settings.STEP_ICON} Updating function '{name}'...")
|
|
99
|
+
self._fc.update_function(
|
|
100
|
+
name,
|
|
101
|
+
models.UpdateFunctionRequest(
|
|
102
|
+
runtime="custom-container",
|
|
103
|
+
handler="handler.handler",
|
|
104
|
+
timeout=900,
|
|
105
|
+
memory_size=512,
|
|
106
|
+
custom_container_config=(
|
|
107
|
+
models.CustomContainerConfig(
|
|
108
|
+
image=image_uri,
|
|
109
|
+
port=9000,
|
|
110
|
+
)
|
|
111
|
+
),
|
|
112
|
+
),
|
|
113
|
+
)
|
|
114
|
+
except Exception as err:
|
|
115
|
+
if not (hasattr(err, "status_code") and err.status_code == 404):
|
|
116
|
+
raise
|
|
117
|
+
print(f" {settings.STEP_ICON} Creating function '{name}'...")
|
|
118
|
+
self._fc.create_function(
|
|
119
|
+
models.CreateFunctionRequest(
|
|
120
|
+
function_name=name,
|
|
121
|
+
runtime="custom-container",
|
|
122
|
+
handler="handler.handler",
|
|
123
|
+
timeout=900,
|
|
124
|
+
cpu=1,
|
|
125
|
+
memory_size=512,
|
|
126
|
+
disk_size=512,
|
|
127
|
+
custom_container_config=(
|
|
128
|
+
models.CustomContainerConfig(
|
|
129
|
+
image=image_uri,
|
|
130
|
+
port=9000,
|
|
131
|
+
)
|
|
132
|
+
),
|
|
133
|
+
)
|
|
134
|
+
)
|
|
@@ -0,0 +1,78 @@
|
|
|
1
|
+
"""Alibaba Function Compute Scheduled deployment."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from rich import print # type: ignore
|
|
6
|
+
|
|
7
|
+
from dotflow.cloud.alibaba.deployers.fc import (
|
|
8
|
+
AliyunFCDeployer,
|
|
9
|
+
)
|
|
10
|
+
from dotflow.settings import Settings as settings
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
class AliyunFCScheduledDeployer(AliyunFCDeployer):
|
|
14
|
+
"""Deploy dotflow pipeline as Alibaba FC + timer trigger."""
|
|
15
|
+
|
|
16
|
+
def deploy(self, name: str, **kwargs) -> None:
|
|
17
|
+
"""Deploy FC function with timer trigger."""
|
|
18
|
+
name = self._sanitize_name(name)
|
|
19
|
+
schedule = kwargs.get("schedule")
|
|
20
|
+
|
|
21
|
+
print(
|
|
22
|
+
settings.INFO_ALERT,
|
|
23
|
+
f"Deploying to Alibaba FC scheduled '{name}'...",
|
|
24
|
+
)
|
|
25
|
+
|
|
26
|
+
image_uri = self._cr.push(name)
|
|
27
|
+
self._create_or_update_function(name, image_uri)
|
|
28
|
+
|
|
29
|
+
if schedule:
|
|
30
|
+
self._configure_trigger(name, schedule)
|
|
31
|
+
|
|
32
|
+
print(settings.INFO_ALERT, "Done.")
|
|
33
|
+
|
|
34
|
+
def _configure_trigger(self, name: str, schedule: str) -> None:
|
|
35
|
+
"""Create timer trigger on FC function."""
|
|
36
|
+
from alibabacloud_fc_open20210406 import models
|
|
37
|
+
|
|
38
|
+
trigger_name = f"{name}-timer"
|
|
39
|
+
|
|
40
|
+
print(
|
|
41
|
+
f" {settings.STEP_ICON} "
|
|
42
|
+
f"Creating timer trigger '{trigger_name}'..."
|
|
43
|
+
)
|
|
44
|
+
|
|
45
|
+
try:
|
|
46
|
+
self._fc.get_trigger(name, trigger_name)
|
|
47
|
+
self._fc.update_trigger(
|
|
48
|
+
name,
|
|
49
|
+
trigger_name,
|
|
50
|
+
models.UpdateTriggerRequest(
|
|
51
|
+
trigger_config=(
|
|
52
|
+
models.TimerTriggerConfig(
|
|
53
|
+
cron_expression=schedule,
|
|
54
|
+
payload="{}",
|
|
55
|
+
enable=True,
|
|
56
|
+
)
|
|
57
|
+
),
|
|
58
|
+
),
|
|
59
|
+
)
|
|
60
|
+
except Exception as err:
|
|
61
|
+
if not (hasattr(err, "status_code") and err.status_code == 404):
|
|
62
|
+
raise
|
|
63
|
+
self._fc.create_trigger(
|
|
64
|
+
name,
|
|
65
|
+
models.CreateTriggerRequest(
|
|
66
|
+
trigger_name=trigger_name,
|
|
67
|
+
trigger_type="timer",
|
|
68
|
+
trigger_config=(
|
|
69
|
+
models.TimerTriggerConfig(
|
|
70
|
+
cron_expression=schedule,
|
|
71
|
+
payload="{}",
|
|
72
|
+
enable=True,
|
|
73
|
+
)
|
|
74
|
+
),
|
|
75
|
+
),
|
|
76
|
+
)
|
|
77
|
+
|
|
78
|
+
print(f" {settings.STEP_ICON} Cron: {schedule}")
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
"""Alibaba Cloud services."""
|