tinybird 0.0.1.dev245__py3-none-any.whl → 0.0.1.dev247__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of tinybird might be problematic. Click here for more details.
- tinybird/ch_utils/constants.py +2 -0
- tinybird/tb/__cli__.py +2 -2
- tinybird/tb/modules/agent/agent.py +136 -27
- tinybird/tb/modules/agent/models.py +6 -0
- tinybird/tb/modules/agent/prompts.py +71 -43
- tinybird/tb/modules/agent/tools/append.py +55 -0
- tinybird/tb/modules/agent/tools/build.py +20 -0
- tinybird/tb/modules/agent/tools/create_datafile.py +84 -5
- tinybird/tb/modules/agent/tools/deploy.py +45 -0
- tinybird/tb/modules/agent/tools/deploy_check.py +19 -0
- tinybird/tb/modules/agent/tools/mock.py +59 -0
- tinybird/tb/modules/agent/tools/plan.py +1 -1
- tinybird/tb/modules/agent/tools/read_fixture_data.py +28 -0
- tinybird/tb/modules/agent/utils.py +9 -2
- tinybird/tb/modules/build.py +4 -1
- tinybird/tb/modules/build_common.py +2 -3
- tinybird/tb/modules/cli.py +9 -1
- tinybird/tb/modules/create.py +1 -1
- tinybird/tb/modules/deployment.py +9 -381
- tinybird/tb/modules/deployment_common.py +413 -0
- tinybird/tb/modules/feedback_manager.py +8 -6
- tinybird/tb/modules/llm.py +1 -1
- tinybird/tb/modules/mock.py +3 -69
- tinybird/tb/modules/mock_common.py +71 -0
- tinybird/tb/modules/project.py +9 -0
- {tinybird-0.0.1.dev245.dist-info → tinybird-0.0.1.dev247.dist-info}/METADATA +1 -1
- {tinybird-0.0.1.dev245.dist-info → tinybird-0.0.1.dev247.dist-info}/RECORD +30 -22
- {tinybird-0.0.1.dev245.dist-info → tinybird-0.0.1.dev247.dist-info}/WHEEL +0 -0
- {tinybird-0.0.1.dev245.dist-info → tinybird-0.0.1.dev247.dist-info}/entry_points.txt +0 -0
- {tinybird-0.0.1.dev245.dist-info → tinybird-0.0.1.dev247.dist-info}/top_level.txt +0 -0
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
import click
|
|
2
|
+
from pydantic_ai import RunContext
|
|
3
|
+
|
|
4
|
+
from tinybird.tb.modules.agent.utils import TinybirdAgentContext
|
|
5
|
+
from tinybird.tb.modules.feedback_manager import FeedbackManager
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
def deploy_check(ctx: RunContext[TinybirdAgentContext]) -> str:
|
|
9
|
+
"""Check that project can be deployed"""
|
|
10
|
+
try:
|
|
11
|
+
ctx.deps.thinking_animation.stop()
|
|
12
|
+
ctx.deps.deploy_check_project()
|
|
13
|
+
ctx.deps.thinking_animation.start()
|
|
14
|
+
return "Project can be deployed"
|
|
15
|
+
except Exception as e:
|
|
16
|
+
ctx.deps.thinking_animation.stop()
|
|
17
|
+
click.echo(FeedbackManager.error(message=e))
|
|
18
|
+
ctx.deps.thinking_animation.start()
|
|
19
|
+
return f"Project cannot be deployed: {e}"
|
|
@@ -0,0 +1,59 @@
|
|
|
1
|
+
import click
|
|
2
|
+
from pydantic_ai import RunContext
|
|
3
|
+
|
|
4
|
+
from tinybird.tb.modules.agent.utils import TinybirdAgentContext, show_options
|
|
5
|
+
from tinybird.tb.modules.datafile.fixture import persist_fixture
|
|
6
|
+
from tinybird.tb.modules.feedback_manager import FeedbackManager
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
def get_mock_confirmation(datasource_name: str) -> bool:
|
|
10
|
+
"""Get user confirmation for creating mock data"""
|
|
11
|
+
while True:
|
|
12
|
+
result = show_options(
|
|
13
|
+
options=["Yes, create mock data", "No, and tell Tinybird Code what to do"],
|
|
14
|
+
title=f"Do you want to generate mock data for datasource {datasource_name}?",
|
|
15
|
+
)
|
|
16
|
+
|
|
17
|
+
if result is None: # Cancelled
|
|
18
|
+
return False
|
|
19
|
+
|
|
20
|
+
if result.startswith("Yes"):
|
|
21
|
+
return True
|
|
22
|
+
elif result.startswith("No"):
|
|
23
|
+
return False
|
|
24
|
+
|
|
25
|
+
return False
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
def mock(ctx: RunContext[TinybirdAgentContext], datasource_name: str, data_format: str, rows: int) -> str:
|
|
29
|
+
"""Create mock data for a datasource
|
|
30
|
+
|
|
31
|
+
Args:
|
|
32
|
+
datasource_name: Name of the datasource to create mock data for
|
|
33
|
+
data_format: Format of the mock data to create. Options: ndjson, csv
|
|
34
|
+
rows: Number of rows to create. If not provided, the default is 10
|
|
35
|
+
|
|
36
|
+
Returns:
|
|
37
|
+
str: Message indicating the success or failure of the mock data generation
|
|
38
|
+
"""
|
|
39
|
+
try:
|
|
40
|
+
ctx.deps.thinking_animation.stop()
|
|
41
|
+
confirmation = ctx.deps.dangerously_skip_permissions or get_mock_confirmation(datasource_name)
|
|
42
|
+
ctx.deps.thinking_animation.start()
|
|
43
|
+
|
|
44
|
+
if not confirmation:
|
|
45
|
+
return "User rejected mock data generation. Skipping..."
|
|
46
|
+
|
|
47
|
+
ctx.deps.thinking_animation.stop()
|
|
48
|
+
click.echo(FeedbackManager.highlight(message=f"\n» Generating mock data for {datasource_name}..."))
|
|
49
|
+
data = ctx.deps.mock_data(datasource_name=datasource_name, data_format=data_format, rows=rows)
|
|
50
|
+
fixture_path = persist_fixture(datasource_name, data, ctx.deps.folder, format=data_format)
|
|
51
|
+
ctx.deps.append_data(datasource_name=datasource_name, path=str(fixture_path))
|
|
52
|
+
click.echo(FeedbackManager.success(message=f"✓ Data generated for {datasource_name}"))
|
|
53
|
+
ctx.deps.thinking_animation.start()
|
|
54
|
+
return f"Mock data generated successfully for datasource {datasource_name}"
|
|
55
|
+
except Exception as e:
|
|
56
|
+
ctx.deps.thinking_animation.stop()
|
|
57
|
+
click.echo(FeedbackManager.error(message=e))
|
|
58
|
+
ctx.deps.thinking_animation.start()
|
|
59
|
+
return f"Error generating mock data: {e}"
|
|
@@ -35,7 +35,7 @@ def plan(ctx: RunContext[TinybirdAgentContext], plan: str) -> str:
|
|
|
35
35
|
try:
|
|
36
36
|
ctx.deps.thinking_animation.stop()
|
|
37
37
|
click.echo(plan)
|
|
38
|
-
confirmation = get_plan_confirmation()
|
|
38
|
+
confirmation = ctx.deps.dangerously_skip_permissions or get_plan_confirmation()
|
|
39
39
|
ctx.deps.thinking_animation.start()
|
|
40
40
|
|
|
41
41
|
if not confirmation:
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
import json
|
|
2
|
+
from pathlib import Path
|
|
3
|
+
|
|
4
|
+
from pydantic_ai import RunContext
|
|
5
|
+
|
|
6
|
+
from tinybird.tb.modules.agent.utils import TinybirdAgentContext
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
def read_fixture_data(ctx: RunContext[TinybirdAgentContext], fixture_pathname: str):
|
|
10
|
+
"""Read fixture data in the project folder
|
|
11
|
+
|
|
12
|
+
Args:
|
|
13
|
+
fixture_pathname (str): a path to a fixture file. Required.
|
|
14
|
+
|
|
15
|
+
Returns:
|
|
16
|
+
str: The content of the fixture data file.
|
|
17
|
+
"""
|
|
18
|
+
fixture_path = Path(ctx.deps.folder) / fixture_pathname.lstrip("/")
|
|
19
|
+
|
|
20
|
+
if not fixture_path.exists():
|
|
21
|
+
return f"No fixture data found for {fixture_pathname}. Please check the name of the fixture and try again."
|
|
22
|
+
|
|
23
|
+
response = ctx.deps.analyze_fixture(fixture_path=str(fixture_path))
|
|
24
|
+
# limit content to first 10 rows
|
|
25
|
+
data = response["preview"]["data"][:10]
|
|
26
|
+
schema = response["analysis"]["schema"]
|
|
27
|
+
|
|
28
|
+
return f"#Result of analysis of {fixture_pathname}:\n##Data sample:\n{json.dumps(data)}\n##Schema:\n{schema}"
|
|
@@ -18,11 +18,18 @@ from pydantic import BaseModel, Field
|
|
|
18
18
|
|
|
19
19
|
|
|
20
20
|
class TinybirdAgentContext(BaseModel):
|
|
21
|
-
explore_data: Callable[[str], str]
|
|
22
21
|
folder: str
|
|
23
|
-
|
|
22
|
+
workspace_name: str
|
|
24
23
|
thinking_animation: Any
|
|
25
24
|
get_project_files: Callable[[], List[str]]
|
|
25
|
+
explore_data: Callable[[str], str]
|
|
26
|
+
build_project: Callable[..., None]
|
|
27
|
+
deploy_project: Callable[[], None]
|
|
28
|
+
deploy_check_project: Callable[[], None]
|
|
29
|
+
mock_data: Callable[..., list[dict[str, Any]]]
|
|
30
|
+
append_data: Callable[..., None]
|
|
31
|
+
analyze_fixture: Callable[..., dict[str, Any]]
|
|
32
|
+
dangerously_skip_permissions: bool
|
|
26
33
|
|
|
27
34
|
|
|
28
35
|
default_style = Style.from_dict(
|
tinybird/tb/modules/build.py
CHANGED
|
@@ -3,7 +3,7 @@ import time
|
|
|
3
3
|
from copy import deepcopy
|
|
4
4
|
from functools import partial
|
|
5
5
|
from pathlib import Path
|
|
6
|
-
from typing import Callable, List
|
|
6
|
+
from typing import Any, Callable, Dict, List
|
|
7
7
|
from urllib.parse import urlencode
|
|
8
8
|
|
|
9
9
|
import click
|
|
@@ -32,8 +32,11 @@ def build(ctx: click.Context, watch: bool) -> None:
|
|
|
32
32
|
"""
|
|
33
33
|
Validate and build the project server side.
|
|
34
34
|
"""
|
|
35
|
+
obj: Dict[str, Any] = ctx.ensure_object(dict)
|
|
35
36
|
project: Project = ctx.ensure_object(dict)["project"]
|
|
36
37
|
tb_client: TinyB = ctx.ensure_object(dict)["client"]
|
|
38
|
+
if obj["env"] == "cloud":
|
|
39
|
+
raise click.ClickException(FeedbackManager.error_build_only_supported_in_local())
|
|
37
40
|
|
|
38
41
|
if project.has_deeper_level():
|
|
39
42
|
click.echo(
|
|
@@ -3,7 +3,7 @@ import logging
|
|
|
3
3
|
import time
|
|
4
4
|
from pathlib import Path
|
|
5
5
|
from typing import Optional
|
|
6
|
-
from urllib.parse import urlencode
|
|
6
|
+
from urllib.parse import urlencode, urljoin
|
|
7
7
|
|
|
8
8
|
import click
|
|
9
9
|
import requests
|
|
@@ -25,7 +25,6 @@ def process(
|
|
|
25
25
|
file_changed: Optional[str] = None,
|
|
26
26
|
diff: Optional[str] = None,
|
|
27
27
|
silent: bool = False,
|
|
28
|
-
error: bool = False,
|
|
29
28
|
build_status: Optional[BuildStatus] = None,
|
|
30
29
|
exit_on_error: bool = True,
|
|
31
30
|
) -> Optional[str]:
|
|
@@ -190,7 +189,7 @@ def build_project(project: Project, tb_client: TinyB, silent: bool = False) -> O
|
|
|
190
189
|
".pipe": "text/plain",
|
|
191
190
|
".connection": "text/plain",
|
|
192
191
|
}
|
|
193
|
-
TINYBIRD_API_URL = tb_client.host
|
|
192
|
+
TINYBIRD_API_URL = urljoin(tb_client.host, "/v1/build")
|
|
194
193
|
logging.debug(TINYBIRD_API_URL)
|
|
195
194
|
TINYBIRD_API_KEY = tb_client.token
|
|
196
195
|
error: Optional[str] = None
|
tinybird/tb/modules/cli.py
CHANGED
|
@@ -79,6 +79,13 @@ agent_mode_flag = os.environ.get("TB_AGENT_MODE", "false") == "true"
|
|
|
79
79
|
"--output", type=click.Choice(["human", "json", "csv"], case_sensitive=False), default="human", help="Output format"
|
|
80
80
|
)
|
|
81
81
|
@click.option("--max-depth", type=int, default=3, help="Maximum depth of the project files.")
|
|
82
|
+
@click.option(
|
|
83
|
+
"--dangerously-skip-permissions",
|
|
84
|
+
is_flag=True,
|
|
85
|
+
default=False,
|
|
86
|
+
help="Skip permissions check in agent mode.",
|
|
87
|
+
hidden=True,
|
|
88
|
+
)
|
|
82
89
|
@click.version_option(version=VERSION)
|
|
83
90
|
@click.pass_context
|
|
84
91
|
def cli(
|
|
@@ -93,6 +100,7 @@ def cli(
|
|
|
93
100
|
staging: bool,
|
|
94
101
|
output: str,
|
|
95
102
|
max_depth: int,
|
|
103
|
+
dangerously_skip_permissions: bool,
|
|
96
104
|
) -> None:
|
|
97
105
|
"""
|
|
98
106
|
Use `OBFUSCATE_REGEX_PATTERN` and `OBFUSCATE_PATTERN_SEPARATOR` environment variables to define a regex pattern and a separator (in case of a single string with multiple regex) to obfuscate secrets in the CLI output.
|
|
@@ -196,7 +204,7 @@ def cli(
|
|
|
196
204
|
is_agent_mode = agent_mode_flag and ctx.invoked_subcommand is None
|
|
197
205
|
|
|
198
206
|
if is_agent_mode:
|
|
199
|
-
run_agent(config, project)
|
|
207
|
+
run_agent(config, project, dangerously_skip_permissions)
|
|
200
208
|
|
|
201
209
|
|
|
202
210
|
@cli.command(hidden=True)
|
tinybird/tb/modules/create.py
CHANGED
|
@@ -21,7 +21,7 @@ from tinybird.tb.modules.feedback_manager import FeedbackManager
|
|
|
21
21
|
from tinybird.tb.modules.llm import LLM
|
|
22
22
|
from tinybird.tb.modules.llm_utils import extract_xml, parse_xml
|
|
23
23
|
from tinybird.tb.modules.local_common import get_tinybird_local_client
|
|
24
|
-
from tinybird.tb.modules.
|
|
24
|
+
from tinybird.tb.modules.mock_common import create_mock_data
|
|
25
25
|
from tinybird.tb.modules.project import Project
|
|
26
26
|
|
|
27
27
|
|
|
@@ -1,10 +1,8 @@
|
|
|
1
1
|
import json
|
|
2
2
|
import logging
|
|
3
|
-
import sys
|
|
4
|
-
import time
|
|
5
3
|
from datetime import datetime
|
|
6
4
|
from pathlib import Path
|
|
7
|
-
from typing import Any, Dict, Optional
|
|
5
|
+
from typing import Any, Dict, Optional
|
|
8
6
|
|
|
9
7
|
import click
|
|
10
8
|
import requests
|
|
@@ -12,10 +10,14 @@ import requests
|
|
|
12
10
|
from tinybird.tb.modules.cli import cli
|
|
13
11
|
from tinybird.tb.modules.common import (
|
|
14
12
|
echo_safe_humanfriendly_tables_format_smart_table,
|
|
15
|
-
get_display_cloud_host,
|
|
16
13
|
sys_exit,
|
|
17
14
|
)
|
|
18
|
-
from tinybird.tb.modules.
|
|
15
|
+
from tinybird.tb.modules.deployment_common import (
|
|
16
|
+
create_deployment,
|
|
17
|
+
discard_deployment,
|
|
18
|
+
promote_deployment,
|
|
19
|
+
)
|
|
20
|
+
from tinybird.tb.modules.feedback_manager import FeedbackManager
|
|
19
21
|
from tinybird.tb.modules.project import Project
|
|
20
22
|
|
|
21
23
|
|
|
@@ -148,156 +150,6 @@ def api_fetch(url: str, headers: dict) -> dict:
|
|
|
148
150
|
return {}
|
|
149
151
|
|
|
150
152
|
|
|
151
|
-
def api_post(
|
|
152
|
-
url: str,
|
|
153
|
-
headers: dict,
|
|
154
|
-
files: Optional[list] = None,
|
|
155
|
-
params: Optional[dict] = None,
|
|
156
|
-
) -> dict:
|
|
157
|
-
r = requests.post(url, headers=headers, files=files, params=params)
|
|
158
|
-
if r.status_code < 300:
|
|
159
|
-
logging.debug(json.dumps(r.json(), indent=2))
|
|
160
|
-
return r.json()
|
|
161
|
-
# Try to parse and print the error from the response
|
|
162
|
-
try:
|
|
163
|
-
result = r.json()
|
|
164
|
-
logging.debug(json.dumps(result, indent=2))
|
|
165
|
-
error = result.get("error")
|
|
166
|
-
if error:
|
|
167
|
-
click.echo(FeedbackManager.error(message=f"Error: {error}"))
|
|
168
|
-
sys_exit("deployment_error", error)
|
|
169
|
-
return result
|
|
170
|
-
except Exception:
|
|
171
|
-
message = "Error parsing response from API"
|
|
172
|
-
click.echo(FeedbackManager.error(message=message))
|
|
173
|
-
sys_exit("deployment_error", message)
|
|
174
|
-
return {}
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
# TODO(eclbg): This logic should be in the server, and there should be a dedicated endpoint for promoting a deployment
|
|
178
|
-
# potato
|
|
179
|
-
def promote_deployment(host: Optional[str], headers: dict, wait: bool) -> None:
|
|
180
|
-
TINYBIRD_API_URL = f"{host}/v1/deployments"
|
|
181
|
-
result = api_fetch(TINYBIRD_API_URL, headers)
|
|
182
|
-
|
|
183
|
-
deployments = result.get("deployments")
|
|
184
|
-
if not deployments:
|
|
185
|
-
message = "No deployments found"
|
|
186
|
-
click.echo(FeedbackManager.error(message=message))
|
|
187
|
-
sys_exit("deployment_error", message)
|
|
188
|
-
return
|
|
189
|
-
|
|
190
|
-
if len(deployments) < 2:
|
|
191
|
-
message = "Only one deployment found"
|
|
192
|
-
click.echo(FeedbackManager.error(message=message))
|
|
193
|
-
sys_exit("deployment_error", message)
|
|
194
|
-
return
|
|
195
|
-
|
|
196
|
-
last_deployment, candidate_deployment = deployments[0], deployments[1]
|
|
197
|
-
|
|
198
|
-
if candidate_deployment.get("status") != "data_ready":
|
|
199
|
-
click.echo(FeedbackManager.error(message="Current deployment is not ready"))
|
|
200
|
-
deploy_errors = candidate_deployment.get("errors", [])
|
|
201
|
-
for deploy_error in deploy_errors:
|
|
202
|
-
click.echo(FeedbackManager.error(message=f"* {deploy_error}"))
|
|
203
|
-
sys_exit("deployment_error", "Current deployment is not ready: " + str(deploy_errors))
|
|
204
|
-
return
|
|
205
|
-
|
|
206
|
-
if candidate_deployment.get("live"):
|
|
207
|
-
click.echo(FeedbackManager.error(message="Candidate deployment is already live"))
|
|
208
|
-
else:
|
|
209
|
-
TINYBIRD_API_URL = f"{host}/v1/deployments/{candidate_deployment.get('id')}/set-live"
|
|
210
|
-
result = api_post(TINYBIRD_API_URL, headers=headers)
|
|
211
|
-
|
|
212
|
-
click.echo(FeedbackManager.highlight(message="» Removing old deployment"))
|
|
213
|
-
|
|
214
|
-
TINYBIRD_API_URL = f"{host}/v1/deployments/{last_deployment.get('id')}"
|
|
215
|
-
r = requests.delete(TINYBIRD_API_URL, headers=headers)
|
|
216
|
-
result = r.json()
|
|
217
|
-
logging.debug(json.dumps(result, indent=2))
|
|
218
|
-
if result.get("error"):
|
|
219
|
-
click.echo(FeedbackManager.error(message=result.get("error")))
|
|
220
|
-
sys_exit("deployment_error", result.get("error", "Unknown error"))
|
|
221
|
-
click.echo(FeedbackManager.info(message="✓ Old deployment removed"))
|
|
222
|
-
|
|
223
|
-
click.echo(FeedbackManager.highlight(message="» Waiting for deployment to be promoted..."))
|
|
224
|
-
|
|
225
|
-
if wait:
|
|
226
|
-
while True:
|
|
227
|
-
TINYBIRD_API_URL = f"{host}/v1/deployments/{last_deployment.get('id')}"
|
|
228
|
-
result = api_fetch(TINYBIRD_API_URL, headers=headers)
|
|
229
|
-
|
|
230
|
-
last_deployment = result.get("deployment")
|
|
231
|
-
if last_deployment.get("status") == "deleted":
|
|
232
|
-
click.echo(FeedbackManager.success(message=f"✓ Deployment #{candidate_deployment.get('id')} is live!"))
|
|
233
|
-
break
|
|
234
|
-
|
|
235
|
-
time.sleep(5)
|
|
236
|
-
if last_deployment.get("id") == "0":
|
|
237
|
-
# This is the first deployment, so we prompt the user to ingest data
|
|
238
|
-
click.echo(
|
|
239
|
-
FeedbackManager.info(
|
|
240
|
-
message="A deployment with no data is useless. Learn how to ingest at https://www.tinybird.co/docs/forward/get-data-in"
|
|
241
|
-
)
|
|
242
|
-
)
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
# TODO(eclbg): This logic should be in the server, and there should be a dedicated endpoint for discarding a
|
|
246
|
-
# deployment
|
|
247
|
-
def discard_deployment(host: Optional[str], headers: dict, wait: bool) -> None:
|
|
248
|
-
TINYBIRD_API_URL = f"{host}/v1/deployments"
|
|
249
|
-
result = api_fetch(TINYBIRD_API_URL, headers=headers)
|
|
250
|
-
|
|
251
|
-
deployments = result.get("deployments")
|
|
252
|
-
if not deployments:
|
|
253
|
-
click.echo(FeedbackManager.error(message="No deployments found"))
|
|
254
|
-
return
|
|
255
|
-
|
|
256
|
-
if len(deployments) < 2:
|
|
257
|
-
click.echo(FeedbackManager.error(message="Only one deployment found"))
|
|
258
|
-
return
|
|
259
|
-
|
|
260
|
-
previous_deployment, current_deployment = deployments[0], deployments[1]
|
|
261
|
-
|
|
262
|
-
if previous_deployment.get("status") != "data_ready":
|
|
263
|
-
click.echo(FeedbackManager.error(message="Previous deployment is not ready"))
|
|
264
|
-
deploy_errors = previous_deployment.get("errors", [])
|
|
265
|
-
for deploy_error in deploy_errors:
|
|
266
|
-
click.echo(FeedbackManager.error(message=f"* {deploy_error}"))
|
|
267
|
-
return
|
|
268
|
-
|
|
269
|
-
if previous_deployment.get("live"):
|
|
270
|
-
click.echo(FeedbackManager.error(message="Previous deployment is already live"))
|
|
271
|
-
else:
|
|
272
|
-
click.echo(FeedbackManager.success(message="Promoting previous deployment"))
|
|
273
|
-
|
|
274
|
-
TINYBIRD_API_URL = f"{host}/v1/deployments/{previous_deployment.get('id')}/set-live"
|
|
275
|
-
result = api_post(TINYBIRD_API_URL, headers=headers)
|
|
276
|
-
|
|
277
|
-
click.echo(FeedbackManager.success(message="Removing current deployment"))
|
|
278
|
-
|
|
279
|
-
TINYBIRD_API_URL = f"{host}/v1/deployments/{current_deployment.get('id')}"
|
|
280
|
-
r = requests.delete(TINYBIRD_API_URL, headers=headers)
|
|
281
|
-
result = r.json()
|
|
282
|
-
logging.debug(json.dumps(result, indent=2))
|
|
283
|
-
if result.get("error"):
|
|
284
|
-
click.echo(FeedbackManager.error(message=result.get("error")))
|
|
285
|
-
sys_exit("deployment_error", result.get("error", "Unknown error"))
|
|
286
|
-
|
|
287
|
-
click.echo(FeedbackManager.success(message="Discard process successfully started"))
|
|
288
|
-
|
|
289
|
-
if wait:
|
|
290
|
-
while True:
|
|
291
|
-
TINYBIRD_API_URL = f"{host}/v1/deployments/{current_deployment.get('id')}"
|
|
292
|
-
result = api_fetch(TINYBIRD_API_URL, headers)
|
|
293
|
-
|
|
294
|
-
current_deployment = result.get("deployment")
|
|
295
|
-
if current_deployment.get("status") == "deleted":
|
|
296
|
-
click.echo(FeedbackManager.success(message="Discard process successfully completed"))
|
|
297
|
-
break
|
|
298
|
-
time.sleep(5)
|
|
299
|
-
|
|
300
|
-
|
|
301
153
|
@cli.group(name="deployment")
|
|
302
154
|
def deployment_group() -> None:
|
|
303
155
|
"""
|
|
@@ -482,8 +334,8 @@ def create_deployment_cmd(
|
|
|
482
334
|
allow_destructive_operations: Optional[bool] = None,
|
|
483
335
|
template: Optional[str] = None,
|
|
484
336
|
) -> None:
|
|
337
|
+
project: Project = ctx.ensure_object(dict)["project"]
|
|
485
338
|
if template:
|
|
486
|
-
project = ctx.ensure_object(dict)["project"]
|
|
487
339
|
if project.get_project_files():
|
|
488
340
|
click.echo(
|
|
489
341
|
FeedbackManager.error(
|
|
@@ -503,230 +355,6 @@ def create_deployment_cmd(
|
|
|
503
355
|
click.echo(FeedbackManager.error(message=f"Error downloading template: {str(e)}"))
|
|
504
356
|
sys_exit("deployment_error", f"Failed to download template {template}")
|
|
505
357
|
click.echo(FeedbackManager.success(message="Template downloaded successfully"))
|
|
506
|
-
|
|
507
|
-
create_deployment(ctx, wait, auto, check, allow_destructive_operations)
|
|
508
|
-
|
|
509
|
-
|
|
510
|
-
def create_deployment(
|
|
511
|
-
ctx: click.Context,
|
|
512
|
-
wait: bool,
|
|
513
|
-
auto: bool,
|
|
514
|
-
check: Optional[bool] = None,
|
|
515
|
-
allow_destructive_operations: Optional[bool] = None,
|
|
516
|
-
) -> None:
|
|
517
|
-
# TODO: This code is duplicated in build_server.py
|
|
518
|
-
# Should be refactored to be shared
|
|
519
|
-
MULTIPART_BOUNDARY_DATA_PROJECT = "data_project://"
|
|
520
|
-
DATAFILE_TYPE_TO_CONTENT_TYPE = {
|
|
521
|
-
".datasource": "text/plain",
|
|
522
|
-
".pipe": "text/plain",
|
|
523
|
-
".connection": "text/plain",
|
|
524
|
-
}
|
|
525
|
-
project: Project = ctx.ensure_object(dict)["project"]
|
|
526
358
|
client = ctx.ensure_object(dict)["client"]
|
|
527
359
|
config: Dict[str, Any] = ctx.ensure_object(dict)["config"]
|
|
528
|
-
|
|
529
|
-
TINYBIRD_API_KEY = client.token
|
|
530
|
-
|
|
531
|
-
if project.has_deeper_level():
|
|
532
|
-
click.echo(
|
|
533
|
-
FeedbackManager.warning(
|
|
534
|
-
message="\nYour project contains directories nested deeper than the default scan depth (max_depth=3). "
|
|
535
|
-
"Files in these deeper directories will not be processed. "
|
|
536
|
-
"To include all nested directories, run `tb --max-depth <depth> <cmd>` with a higher depth value."
|
|
537
|
-
)
|
|
538
|
-
)
|
|
539
|
-
|
|
540
|
-
files = [
|
|
541
|
-
("context://", ("cli-version", "1.0.0", "text/plain")),
|
|
542
|
-
]
|
|
543
|
-
for file_path in project.get_project_files():
|
|
544
|
-
relative_path = Path(file_path).relative_to(project.path).as_posix()
|
|
545
|
-
with open(file_path, "rb") as fd:
|
|
546
|
-
content_type = DATAFILE_TYPE_TO_CONTENT_TYPE.get(Path(file_path).suffix, "application/unknown")
|
|
547
|
-
files.append((MULTIPART_BOUNDARY_DATA_PROJECT, (relative_path, fd.read().decode("utf-8"), content_type)))
|
|
548
|
-
|
|
549
|
-
deployment = None
|
|
550
|
-
try:
|
|
551
|
-
HEADERS = {"Authorization": f"Bearer {TINYBIRD_API_KEY}"}
|
|
552
|
-
params = {}
|
|
553
|
-
if check:
|
|
554
|
-
click.echo(FeedbackManager.highlight(message="\n» Validating deployment...\n"))
|
|
555
|
-
params["check"] = "true"
|
|
556
|
-
if allow_destructive_operations:
|
|
557
|
-
params["allow_destructive_operations"] = "true"
|
|
558
|
-
|
|
559
|
-
result = api_post(TINYBIRD_API_URL, headers=HEADERS, files=files, params=params)
|
|
560
|
-
|
|
561
|
-
print_changes(result, project)
|
|
562
|
-
|
|
563
|
-
deployment = result.get("deployment", {})
|
|
564
|
-
feedback = deployment.get("feedback", [])
|
|
565
|
-
for f in feedback:
|
|
566
|
-
if f.get("level", "").upper() == "ERROR":
|
|
567
|
-
feedback_func = FeedbackManager.error
|
|
568
|
-
feedback_icon = ""
|
|
569
|
-
else:
|
|
570
|
-
feedback_func = FeedbackManager.warning
|
|
571
|
-
feedback_icon = "△ "
|
|
572
|
-
resource = f.get("resource")
|
|
573
|
-
resource_bit = f"{resource}: " if resource else ""
|
|
574
|
-
click.echo(feedback_func(message=f"{feedback_icon}{f.get('level')}: {resource_bit}{f.get('message')}"))
|
|
575
|
-
|
|
576
|
-
deploy_errors = deployment.get("errors")
|
|
577
|
-
for deploy_error in deploy_errors:
|
|
578
|
-
if deploy_error.get("filename", None):
|
|
579
|
-
click.echo(
|
|
580
|
-
FeedbackManager.error(message=f"{deploy_error.get('filename')}\n\n{deploy_error.get('error')}")
|
|
581
|
-
)
|
|
582
|
-
else:
|
|
583
|
-
click.echo(FeedbackManager.error(message=f"{deploy_error.get('error')}"))
|
|
584
|
-
click.echo("") # For spacing
|
|
585
|
-
|
|
586
|
-
status = result.get("result")
|
|
587
|
-
if check:
|
|
588
|
-
if status == "success":
|
|
589
|
-
click.echo(FeedbackManager.success(message="\n✓ Deployment is valid"))
|
|
590
|
-
sys.exit(0)
|
|
591
|
-
elif status == "no_changes":
|
|
592
|
-
sys.exit(0)
|
|
593
|
-
|
|
594
|
-
click.echo(FeedbackManager.error(message="\n✗ Deployment is not valid"))
|
|
595
|
-
sys_exit(
|
|
596
|
-
"deployment_error",
|
|
597
|
-
f"Deployment is not valid: {str(deployment.get('errors') + deployment.get('feedback', []))}",
|
|
598
|
-
)
|
|
599
|
-
|
|
600
|
-
status = result.get("result")
|
|
601
|
-
if status == "success":
|
|
602
|
-
host = get_display_cloud_host(client.host)
|
|
603
|
-
click.echo(
|
|
604
|
-
FeedbackManager.info(message="Deployment URL: ")
|
|
605
|
-
+ f"{bcolors.UNDERLINE}{host}/{config.get('name')}/deployments/{deployment.get('id')}{bcolors.ENDC}"
|
|
606
|
-
)
|
|
607
|
-
|
|
608
|
-
if wait:
|
|
609
|
-
click.echo(FeedbackManager.info(message="\n* Deployment submitted"))
|
|
610
|
-
else:
|
|
611
|
-
click.echo(FeedbackManager.success(message="\n✓ Deployment submitted successfully"))
|
|
612
|
-
elif status == "no_changes":
|
|
613
|
-
click.echo(FeedbackManager.warning(message="△ Not deploying. No changes."))
|
|
614
|
-
sys.exit(0)
|
|
615
|
-
elif status == "failed":
|
|
616
|
-
click.echo(FeedbackManager.error(message="Deployment failed"))
|
|
617
|
-
sys_exit(
|
|
618
|
-
"deployment_error",
|
|
619
|
-
f"Deployment failed. Errors: {str(deployment.get('errors') + deployment.get('feedback', []))}",
|
|
620
|
-
)
|
|
621
|
-
else:
|
|
622
|
-
click.echo(FeedbackManager.error(message=f"Unknown deployment result {status}"))
|
|
623
|
-
except Exception as e:
|
|
624
|
-
click.echo(FeedbackManager.error_exception(error=e))
|
|
625
|
-
|
|
626
|
-
if not deployment and not check:
|
|
627
|
-
sys_exit("deployment_error", "Deployment failed")
|
|
628
|
-
|
|
629
|
-
if deployment and wait and not check:
|
|
630
|
-
click.echo(FeedbackManager.highlight(message="» Waiting for deployment to be ready..."))
|
|
631
|
-
while True:
|
|
632
|
-
url = f"{client.host}/v1/deployments/{deployment.get('id')}"
|
|
633
|
-
res = api_fetch(url, HEADERS)
|
|
634
|
-
deployment = res.get("deployment")
|
|
635
|
-
if not deployment:
|
|
636
|
-
click.echo(FeedbackManager.error(message="Error parsing deployment from response"))
|
|
637
|
-
sys_exit("deployment_error", "Error parsing deployment from response")
|
|
638
|
-
if deployment.get("status") == "failed":
|
|
639
|
-
click.echo(FeedbackManager.error(message="Deployment failed"))
|
|
640
|
-
deploy_errors = deployment.get("errors")
|
|
641
|
-
for deploy_error in deploy_errors:
|
|
642
|
-
click.echo(FeedbackManager.error(message=f"* {deploy_error}"))
|
|
643
|
-
|
|
644
|
-
if auto:
|
|
645
|
-
click.echo(FeedbackManager.error(message="Rolling back deployment"))
|
|
646
|
-
discard_deployment(client.host, HEADERS, wait=wait)
|
|
647
|
-
sys_exit(
|
|
648
|
-
"deployment_error",
|
|
649
|
-
f"Deployment failed. Errors: {str(deployment.get('errors') + deployment.get('feedback', []))}",
|
|
650
|
-
)
|
|
651
|
-
|
|
652
|
-
if deployment.get("status") == "data_ready":
|
|
653
|
-
break
|
|
654
|
-
|
|
655
|
-
if deployment.get("status") in ["deleting", "deleted"]:
|
|
656
|
-
click.echo(FeedbackManager.error(message="Deployment was deleted by another process"))
|
|
657
|
-
sys_exit("deployment_error", "Deployment was deleted by another process")
|
|
658
|
-
|
|
659
|
-
time.sleep(5)
|
|
660
|
-
|
|
661
|
-
click.echo(FeedbackManager.info(message="✓ Deployment is ready"))
|
|
662
|
-
|
|
663
|
-
if auto:
|
|
664
|
-
promote_deployment(client.host, HEADERS, wait=wait)
|
|
665
|
-
|
|
666
|
-
|
|
667
|
-
def print_changes(result: dict, project: Project) -> None:
|
|
668
|
-
deployment = result.get("deployment", {})
|
|
669
|
-
resources_columns = ["status", "name", "type", "path"]
|
|
670
|
-
resources: list[list[Union[str, None]]] = []
|
|
671
|
-
tokens_columns = ["Change", "Token name", "Added permissions", "Removed permissions"]
|
|
672
|
-
tokens: list[Tuple[str, str, str, str]] = []
|
|
673
|
-
|
|
674
|
-
for ds in deployment.get("new_datasource_names", []):
|
|
675
|
-
resources.append(["new", ds, "datasource", project.get_resource_path(ds, "datasource")])
|
|
676
|
-
|
|
677
|
-
for p in deployment.get("new_pipe_names", []):
|
|
678
|
-
path = project.get_resource_path(p, "pipe")
|
|
679
|
-
pipe_type = project.get_pipe_type(path)
|
|
680
|
-
resources.append(["new", p, pipe_type, path])
|
|
681
|
-
|
|
682
|
-
for dc in deployment.get("new_data_connector_names", []):
|
|
683
|
-
resources.append(["new", dc, "connection", project.get_resource_path(dc, "connection")])
|
|
684
|
-
|
|
685
|
-
for ds in deployment.get("changed_datasource_names", []):
|
|
686
|
-
resources.append(["modified", ds, "datasource", project.get_resource_path(ds, "datasource")])
|
|
687
|
-
|
|
688
|
-
for p in deployment.get("changed_pipe_names", []):
|
|
689
|
-
path = project.get_resource_path(p, "pipe")
|
|
690
|
-
pipe_type = project.get_pipe_type(path)
|
|
691
|
-
resources.append(["modified", p, pipe_type, path])
|
|
692
|
-
|
|
693
|
-
for dc in deployment.get("changed_data_connector_names", []):
|
|
694
|
-
resources.append(["modified", dc, "connection", project.get_resource_path(dc, "connection")])
|
|
695
|
-
|
|
696
|
-
for ds in deployment.get("disconnected_data_source_names", []):
|
|
697
|
-
resources.append(["modified", ds, "datasource", project.get_resource_path(ds, "datasource")])
|
|
698
|
-
|
|
699
|
-
for ds in deployment.get("deleted_datasource_names", []):
|
|
700
|
-
resources.append(["deleted", ds, "datasource", project.get_resource_path(ds, "datasource")])
|
|
701
|
-
|
|
702
|
-
for p in deployment.get("deleted_pipe_names", []):
|
|
703
|
-
path = project.get_resource_path(p, "pipe")
|
|
704
|
-
pipe_type = project.get_pipe_type(path)
|
|
705
|
-
resources.append(["deleted", p, pipe_type, path])
|
|
706
|
-
|
|
707
|
-
for dc in deployment.get("deleted_data_connector_names", []):
|
|
708
|
-
resources.append(["deleted", dc, "connection", project.get_resource_path(dc, "connection")])
|
|
709
|
-
|
|
710
|
-
for token_change in deployment.get("token_changes", []):
|
|
711
|
-
token_name = token_change.get("token_name")
|
|
712
|
-
change_type = token_change.get("change_type")
|
|
713
|
-
added_perms = []
|
|
714
|
-
removed_perms = []
|
|
715
|
-
permission_changes = token_change.get("permission_changes", {})
|
|
716
|
-
for perm in permission_changes.get("added_permissions", []):
|
|
717
|
-
added_perms.append(f"{perm['resource_name']}.{perm['resource_type']}:{perm['permission']}")
|
|
718
|
-
for perm in permission_changes.get("removed_permissions", []):
|
|
719
|
-
removed_perms.append(f"{perm['resource_name']}.{perm['resource_type']}:{perm['permission']}")
|
|
720
|
-
|
|
721
|
-
tokens.append((change_type, token_name, "\n".join(added_perms), "\n".join(removed_perms)))
|
|
722
|
-
|
|
723
|
-
if resources:
|
|
724
|
-
click.echo(FeedbackManager.info(message="\n* Changes to be deployed:"))
|
|
725
|
-
echo_safe_humanfriendly_tables_format_smart_table(resources, column_names=resources_columns)
|
|
726
|
-
else:
|
|
727
|
-
click.echo(FeedbackManager.gray(message="\n* No changes to be deployed"))
|
|
728
|
-
if tokens:
|
|
729
|
-
click.echo(FeedbackManager.info(message="\n* Changes in tokens to be deployed:"))
|
|
730
|
-
echo_safe_humanfriendly_tables_format_smart_table(tokens, column_names=tokens_columns)
|
|
731
|
-
else:
|
|
732
|
-
click.echo(FeedbackManager.gray(message="* No changes in tokens to be deployed"))
|
|
360
|
+
create_deployment(project, client, config, wait, auto, check, allow_destructive_operations)
|