tinybird 0.0.1.dev291__py3-none-any.whl → 1.0.5__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- tinybird/ch_utils/constants.py +5 -0
- tinybird/connectors.py +1 -7
- tinybird/context.py +3 -3
- tinybird/datafile/common.py +10 -8
- tinybird/datafile/parse_pipe.py +2 -2
- tinybird/feedback_manager.py +3 -0
- tinybird/prompts.py +1 -0
- tinybird/service_datasources.py +223 -0
- tinybird/sql_template.py +26 -11
- tinybird/sql_template_fmt.py +14 -4
- tinybird/tb/__cli__.py +2 -2
- tinybird/tb/cli.py +1 -0
- tinybird/tb/client.py +104 -26
- tinybird/tb/config.py +24 -0
- tinybird/tb/modules/agent/agent.py +103 -67
- tinybird/tb/modules/agent/banner.py +15 -15
- tinybird/tb/modules/agent/explore_agent.py +5 -0
- tinybird/tb/modules/agent/mock_agent.py +5 -1
- tinybird/tb/modules/agent/models.py +6 -2
- tinybird/tb/modules/agent/prompts.py +49 -2
- tinybird/tb/modules/agent/tools/deploy.py +1 -1
- tinybird/tb/modules/agent/tools/execute_query.py +15 -18
- tinybird/tb/modules/agent/tools/request_endpoint.py +1 -1
- tinybird/tb/modules/agent/tools/run_command.py +9 -0
- tinybird/tb/modules/agent/utils.py +38 -48
- tinybird/tb/modules/branch.py +150 -0
- tinybird/tb/modules/build.py +58 -13
- tinybird/tb/modules/build_common.py +209 -25
- tinybird/tb/modules/cli.py +129 -16
- tinybird/tb/modules/common.py +172 -146
- tinybird/tb/modules/connection.py +125 -194
- tinybird/tb/modules/connection_kafka.py +382 -0
- tinybird/tb/modules/copy.py +3 -1
- tinybird/tb/modules/create.py +83 -150
- tinybird/tb/modules/datafile/build.py +27 -38
- tinybird/tb/modules/datafile/build_datasource.py +21 -25
- tinybird/tb/modules/datafile/diff.py +1 -1
- tinybird/tb/modules/datafile/format_pipe.py +46 -7
- tinybird/tb/modules/datafile/playground.py +59 -68
- tinybird/tb/modules/datafile/pull.py +2 -3
- tinybird/tb/modules/datasource.py +477 -308
- tinybird/tb/modules/deployment.py +2 -0
- tinybird/tb/modules/deployment_common.py +84 -44
- tinybird/tb/modules/deprecations.py +4 -4
- tinybird/tb/modules/dev_server.py +33 -12
- tinybird/tb/modules/exceptions.py +14 -0
- tinybird/tb/modules/feedback_manager.py +1 -1
- tinybird/tb/modules/info.py +69 -12
- tinybird/tb/modules/infra.py +4 -5
- tinybird/tb/modules/job_common.py +15 -0
- tinybird/tb/modules/local.py +143 -23
- tinybird/tb/modules/local_common.py +347 -19
- tinybird/tb/modules/local_logs.py +209 -0
- tinybird/tb/modules/login.py +21 -2
- tinybird/tb/modules/login_common.py +254 -12
- tinybird/tb/modules/mock.py +5 -54
- tinybird/tb/modules/mock_common.py +0 -54
- tinybird/tb/modules/open.py +10 -5
- tinybird/tb/modules/project.py +14 -5
- tinybird/tb/modules/shell.py +15 -7
- tinybird/tb/modules/sink.py +3 -1
- tinybird/tb/modules/telemetry.py +11 -3
- tinybird/tb/modules/test.py +13 -9
- tinybird/tb/modules/test_common.py +13 -87
- tinybird/tb/modules/tinyunit/tinyunit.py +0 -14
- tinybird/tb/modules/tinyunit/tinyunit_lib.py +0 -6
- tinybird/tb/modules/watch.py +5 -3
- tinybird/tb_cli_modules/common.py +2 -2
- tinybird/tb_cli_modules/telemetry.py +1 -1
- tinybird/tornado_template.py +6 -7
- {tinybird-0.0.1.dev291.dist-info → tinybird-1.0.5.dist-info}/METADATA +32 -6
- tinybird-1.0.5.dist-info/RECORD +132 -0
- {tinybird-0.0.1.dev291.dist-info → tinybird-1.0.5.dist-info}/WHEEL +1 -1
- tinybird-0.0.1.dev291.dist-info/RECORD +0 -128
- {tinybird-0.0.1.dev291.dist-info → tinybird-1.0.5.dist-info}/entry_points.txt +0 -0
- {tinybird-0.0.1.dev291.dist-info → tinybird-1.0.5.dist-info}/top_level.txt +0 -0
tinybird/tb/client.py
CHANGED
|
@@ -300,33 +300,33 @@ class TinyB:
|
|
|
300
300
|
response = self._req(f"/v0/connectors?{urlencode(params)}")
|
|
301
301
|
return response["connectors"]
|
|
302
302
|
|
|
303
|
-
def connections(self, connector: Optional[str] = None):
|
|
303
|
+
def connections(self, connector: Optional[str] = None, datasources: Optional[List[Dict[str, Any]]] = None):
|
|
304
304
|
response = self._req("/v0/connectors")
|
|
305
305
|
connectors = response["connectors"]
|
|
306
|
+
connectors_to_return = []
|
|
307
|
+
for c in connectors:
|
|
308
|
+
if connector and c["service"] != connector:
|
|
309
|
+
continue
|
|
310
|
+
if connector == "gcscheduler":
|
|
311
|
+
continue
|
|
312
|
+
if datasources and len(datasources) > 0:
|
|
313
|
+
datasource_ids = [linker["datasource_id"] for linker in c["linkers"]]
|
|
314
|
+
datasource_names = [ds["name"] for ds in datasources if ds["id"] in datasource_ids]
|
|
315
|
+
connected_datasources = ", ".join(datasource_names) if len(datasource_names) > 0 else ""
|
|
316
|
+
else:
|
|
317
|
+
connected_datasources = str(len(c["linkers"]))
|
|
306
318
|
|
|
307
|
-
|
|
308
|
-
return [
|
|
319
|
+
connectors_to_return.append(
|
|
309
320
|
{
|
|
310
321
|
"id": c["id"],
|
|
311
322
|
"service": c["service"],
|
|
312
323
|
"name": c["name"],
|
|
313
|
-
"connected_datasources":
|
|
324
|
+
"connected_datasources": connected_datasources,
|
|
314
325
|
**c["settings"],
|
|
315
326
|
}
|
|
316
|
-
|
|
317
|
-
|
|
318
|
-
|
|
319
|
-
return [
|
|
320
|
-
{
|
|
321
|
-
"id": c["id"],
|
|
322
|
-
"service": c["service"],
|
|
323
|
-
"name": c["name"],
|
|
324
|
-
"connected_datasources": len(c["linkers"]),
|
|
325
|
-
**c["settings"],
|
|
326
|
-
}
|
|
327
|
-
for c in connectors
|
|
328
|
-
if c["service"] != "gcscheduler"
|
|
329
|
-
]
|
|
327
|
+
)
|
|
328
|
+
|
|
329
|
+
return connectors_to_return
|
|
330
330
|
|
|
331
331
|
def get_datasource(self, ds_name: str, used_by: bool = False) -> Dict[str, Any]:
|
|
332
332
|
params = {
|
|
@@ -711,7 +711,7 @@ class TinyB:
|
|
|
711
711
|
return self._req(f"/{version}/user/workspaces/?with_environments=true&only_environments=true")
|
|
712
712
|
|
|
713
713
|
def branches(self):
|
|
714
|
-
return self._req("/
|
|
714
|
+
return self._req("/v1/environments")
|
|
715
715
|
|
|
716
716
|
def releases(self, workspace_id):
|
|
717
717
|
return self._req(f"/v0/workspaces/{workspace_id}/releases")
|
|
@@ -740,7 +740,7 @@ class TinyB:
|
|
|
740
740
|
}
|
|
741
741
|
if ignore_datasources:
|
|
742
742
|
params["ignore_datasources"] = ",".join(ignore_datasources)
|
|
743
|
-
return self._req(f"/
|
|
743
|
+
return self._req(f"/v1/environments?{urlencode(params)}", method="POST", data=b"")
|
|
744
744
|
|
|
745
745
|
def branch_workspace_data(
|
|
746
746
|
self,
|
|
@@ -996,10 +996,92 @@ class TinyB:
|
|
|
996
996
|
data=json.dumps(connection_params),
|
|
997
997
|
)
|
|
998
998
|
|
|
999
|
-
def kafka_list_topics(self, connection_id: str, timeout=
|
|
1000
|
-
resp = self._req(
|
|
999
|
+
def kafka_list_topics(self, connection_id: str, timeout=10, retries=3):
|
|
1000
|
+
resp = self._req(
|
|
1001
|
+
f"/v0/connectors/{connection_id}/preview?preview_activity=false",
|
|
1002
|
+
timeout=timeout,
|
|
1003
|
+
retries=retries,
|
|
1004
|
+
)
|
|
1001
1005
|
return [x["topic"] for x in resp["preview"]]
|
|
1002
1006
|
|
|
1007
|
+
def kafka_preview_group(self, connection_id: str, topic: str, group_id: str, timeout=30):
|
|
1008
|
+
params = {
|
|
1009
|
+
"log": "previewGroup",
|
|
1010
|
+
"kafka_group_id": group_id,
|
|
1011
|
+
"kafka_topic": topic,
|
|
1012
|
+
"preview_group": "true",
|
|
1013
|
+
}
|
|
1014
|
+
return self._req(f"/v0/connectors/{connection_id}/preview?{urlencode(params)}", method="GET", timeout=timeout)
|
|
1015
|
+
|
|
1016
|
+
def kafka_preview_topic(self, connection_id: str, topic: str, group_id: str, timeout: int = 30) -> Dict[str, Any]:
|
|
1017
|
+
"""Preview a Kafka topic and return structured preview data.
|
|
1018
|
+
|
|
1019
|
+
Args:
|
|
1020
|
+
connection_id: The ID of the Kafka connection
|
|
1021
|
+
topic: The Kafka topic name to preview
|
|
1022
|
+
group_id: The Kafka consumer group ID
|
|
1023
|
+
timeout: Request timeout in seconds
|
|
1024
|
+
|
|
1025
|
+
Returns:
|
|
1026
|
+
A dictionary containing:
|
|
1027
|
+
- analysis: Dictionary with columns information
|
|
1028
|
+
- preview: Dictionary with data and meta arrays
|
|
1029
|
+
- earliestTimestamp: The earliest message timestamp (if available)
|
|
1030
|
+
"""
|
|
1031
|
+
params = {
|
|
1032
|
+
"max_records": "12",
|
|
1033
|
+
"preview_activity": "true",
|
|
1034
|
+
"preview_earliest_timestamp": "true",
|
|
1035
|
+
"kafka_topic": topic,
|
|
1036
|
+
"kafka_group_id": group_id,
|
|
1037
|
+
"log": "previewTopic",
|
|
1038
|
+
}
|
|
1039
|
+
response = self._req(
|
|
1040
|
+
f"/v0/connectors/{connection_id}/preview?{urlencode(params)}", method="GET", timeout=timeout
|
|
1041
|
+
)
|
|
1042
|
+
|
|
1043
|
+
if not response:
|
|
1044
|
+
return {
|
|
1045
|
+
"analysis": {"columns": []},
|
|
1046
|
+
"preview": {"data": [], "meta": []},
|
|
1047
|
+
"earliestTimestamp": "",
|
|
1048
|
+
}
|
|
1049
|
+
|
|
1050
|
+
# Extract preview data (similar to TypeScript previewKafkaTopic)
|
|
1051
|
+
preview_data = response.get("preview", [])
|
|
1052
|
+
if not preview_data:
|
|
1053
|
+
return {
|
|
1054
|
+
"analysis": {"columns": []},
|
|
1055
|
+
"preview": {"data": [], "meta": []},
|
|
1056
|
+
"earliestTimestamp": "",
|
|
1057
|
+
}
|
|
1058
|
+
|
|
1059
|
+
topic_preview = preview_data[0]
|
|
1060
|
+
analysis = topic_preview.get("analysis", {})
|
|
1061
|
+
deserialized = topic_preview.get("deserialized", {})
|
|
1062
|
+
|
|
1063
|
+
# Extract columns from analysis
|
|
1064
|
+
columns = analysis.get("columns", []) if analysis else []
|
|
1065
|
+
|
|
1066
|
+
# Extract data and meta from deserialized
|
|
1067
|
+
base_data = deserialized.get("data", []) if deserialized else []
|
|
1068
|
+
base_meta = deserialized.get("meta", []) if deserialized else []
|
|
1069
|
+
|
|
1070
|
+
# Extract earliest timestamp
|
|
1071
|
+
earliest = response.get("earliest", [])
|
|
1072
|
+
earliest_timestamp = earliest[0].get("timestamp", "") if earliest else ""
|
|
1073
|
+
|
|
1074
|
+
return {
|
|
1075
|
+
"analysis": {
|
|
1076
|
+
"columns": columns,
|
|
1077
|
+
},
|
|
1078
|
+
"preview": {
|
|
1079
|
+
"data": base_data,
|
|
1080
|
+
"meta": base_meta,
|
|
1081
|
+
},
|
|
1082
|
+
"earliestTimestamp": earliest_timestamp,
|
|
1083
|
+
}
|
|
1084
|
+
|
|
1003
1085
|
def get_gcp_service_account_details(self) -> Dict[str, Any]:
|
|
1004
1086
|
return self._req("/v0/datasources-bigquery-credentials")
|
|
1005
1087
|
|
|
@@ -1390,7 +1472,3 @@ class TinyB:
|
|
|
1390
1472
|
|
|
1391
1473
|
def delete_tag(self, name: str):
|
|
1392
1474
|
self._req(f"/v0/tags/{name}", method="DELETE")
|
|
1393
|
-
|
|
1394
|
-
def explore_data(self, prompt: str) -> str:
|
|
1395
|
-
params = urlencode({"prompt": prompt, "host": self.host, "origin": "cli"})
|
|
1396
|
-
return self._req(f"/v1/agents/explore?{params}")
|
tinybird/tb/config.py
CHANGED
|
@@ -40,6 +40,23 @@ CLOUD_HOSTS = {
|
|
|
40
40
|
"https://ui.europe-west2.gcp.tinybird.co": "https://cloud.tinybird.co/gcp/europe-west2",
|
|
41
41
|
}
|
|
42
42
|
|
|
43
|
+
CH_HOSTS = {
|
|
44
|
+
"https://api.tinybird.co": "https://clickhouse.tinybird.co",
|
|
45
|
+
"https://api.us-east.tinybird.co": "https://clickhouse.us-east.tinybird.co",
|
|
46
|
+
"https://api.us-east.aws.tinybird.co": "https://clickhouse.us-east-1.aws.tinybird.co",
|
|
47
|
+
"https://api.us-west-2.aws.tinybird.co": "https://clickhouse.us-west-2.aws.tinybird.co",
|
|
48
|
+
"https://api.eu-central-1.aws.tinybird.co": "https://clickhouse.eu-central-1.aws.tinybird.co",
|
|
49
|
+
"https://api.eu-west-1.aws.tinybird.co": "https://clickhouse.eu-west-1.aws.tinybird.co",
|
|
50
|
+
"https://api.europe-west2.gcp.tinybird.co": "https://clickhouse.europe-west2.gcp.tinybird.co",
|
|
51
|
+
"https://api.ap-east.aws.tinybird.co": "https://clickhouse.ap-east.aws.tinybird.co",
|
|
52
|
+
"https://ui.tinybird.co": "https://clickhouse.tinybird.co",
|
|
53
|
+
"https://ui.us-east.tinybird.co": "https://clickhouse.us-east.tinybird.co",
|
|
54
|
+
"https://ui.us-east.aws.tinybird.co": "https://clickhouse.us-east.aws.tinybird.co",
|
|
55
|
+
"https://ui.us-west-2.aws.tinybird.co": "https://clickhouse.us-west-2.aws.tinybird.co",
|
|
56
|
+
"https://ui.eu-central-1.aws.tinybird.co": "https://clickhouse.eu-central-1.aws.tinybird.co",
|
|
57
|
+
"https://ui.europe-west2.gcp.tinybird.co": "https://clickhouse.europe-west2.gcp.tinybird.co",
|
|
58
|
+
}
|
|
59
|
+
|
|
43
60
|
|
|
44
61
|
def get_config(
|
|
45
62
|
host: Optional[str], token: Optional[str], semver: Optional[str] = None, config_file: Optional[str] = None
|
|
@@ -85,6 +102,13 @@ def get_display_cloud_host(api_host: str) -> str:
|
|
|
85
102
|
return CLOUD_HOSTS.get(api_host, api_host)
|
|
86
103
|
|
|
87
104
|
|
|
105
|
+
def get_clickhouse_host(api_host: str) -> str:
|
|
106
|
+
is_local = "localhost" in api_host
|
|
107
|
+
if is_local:
|
|
108
|
+
return "http://localhost:7182"
|
|
109
|
+
return f"{CH_HOSTS.get(api_host, api_host.replace('api.', 'clickhouse.'))}:443"
|
|
110
|
+
|
|
111
|
+
|
|
88
112
|
class FeatureFlags:
|
|
89
113
|
@classmethod
|
|
90
114
|
def ignore_sql_errors(cls) -> bool: # Context: #1155
|
|
@@ -6,8 +6,7 @@ import sys
|
|
|
6
6
|
import urllib.parse
|
|
7
7
|
import uuid
|
|
8
8
|
from functools import partial
|
|
9
|
-
from
|
|
10
|
-
from typing import Any, Optional
|
|
9
|
+
from typing import Any, Callable, Optional
|
|
11
10
|
|
|
12
11
|
import click
|
|
13
12
|
import humanfriendly
|
|
@@ -17,7 +16,7 @@ from requests import Response
|
|
|
17
16
|
|
|
18
17
|
from tinybird.tb.check_pypi import CheckPypi
|
|
19
18
|
from tinybird.tb.client import TinyB
|
|
20
|
-
from tinybird.tb.config import CURRENT_VERSION, get_display_cloud_host
|
|
19
|
+
from tinybird.tb.config import CURRENT_VERSION, get_clickhouse_host, get_display_cloud_host
|
|
21
20
|
from tinybird.tb.modules.agent.animations import ThinkingAnimation
|
|
22
21
|
from tinybird.tb.modules.agent.banner import display_banner
|
|
23
22
|
from tinybird.tb.modules.agent.command_agent import CommandAgent
|
|
@@ -27,17 +26,18 @@ from tinybird.tb.modules.agent.file_agent import FileAgent
|
|
|
27
26
|
from tinybird.tb.modules.agent.memory import (
|
|
28
27
|
clear_history,
|
|
29
28
|
clear_messages,
|
|
30
|
-
get_last_messages_from_last_user_prompt,
|
|
31
29
|
save_messages,
|
|
32
30
|
)
|
|
33
31
|
from tinybird.tb.modules.agent.mock_agent import MockAgent
|
|
34
32
|
from tinybird.tb.modules.agent.models import create_model
|
|
35
33
|
from tinybird.tb.modules.agent.prompts import (
|
|
36
34
|
agent_system_prompt,
|
|
35
|
+
fixtures_prompt,
|
|
37
36
|
load_custom_project_rules,
|
|
38
37
|
resources_prompt,
|
|
39
38
|
secrets_prompt,
|
|
40
39
|
service_datasources_prompt,
|
|
40
|
+
vendor_files_prompt,
|
|
41
41
|
)
|
|
42
42
|
from tinybird.tb.modules.agent.testing_agent import TestingAgent
|
|
43
43
|
from tinybird.tb.modules.agent.tools.analyze import analyze_file, analyze_url
|
|
@@ -65,16 +65,17 @@ from tinybird.tb.modules.common import (
|
|
|
65
65
|
echo_safe_humanfriendly_tables_format_pretty_table,
|
|
66
66
|
get_region_from_host,
|
|
67
67
|
get_regions,
|
|
68
|
+
sys_exit,
|
|
68
69
|
update_cli,
|
|
69
70
|
)
|
|
70
71
|
from tinybird.tb.modules.config import CLIConfig
|
|
71
72
|
from tinybird.tb.modules.deployment_common import create_deployment
|
|
72
|
-
from tinybird.tb.modules.exceptions import CLIBuildException, CLIDeploymentException
|
|
73
|
+
from tinybird.tb.modules.exceptions import CLIBuildException, CLIDeploymentException
|
|
73
74
|
from tinybird.tb.modules.feedback_manager import FeedbackManager
|
|
74
75
|
from tinybird.tb.modules.llm import LLM
|
|
75
76
|
from tinybird.tb.modules.local_common import get_tinybird_local_client
|
|
76
77
|
from tinybird.tb.modules.login_common import login
|
|
77
|
-
from tinybird.tb.modules.mock_common import append_mock_data
|
|
78
|
+
from tinybird.tb.modules.mock_common import append_mock_data
|
|
78
79
|
from tinybird.tb.modules.project import Project
|
|
79
80
|
from tinybird.tb.modules.test_common import run_tests as run_tests_common
|
|
80
81
|
|
|
@@ -89,6 +90,7 @@ class TinybirdAgent:
|
|
|
89
90
|
project: Project,
|
|
90
91
|
dangerously_skip_permissions: bool,
|
|
91
92
|
prompt_mode: bool,
|
|
93
|
+
feature: Optional[str] = None,
|
|
92
94
|
):
|
|
93
95
|
self.token = token
|
|
94
96
|
self.user_token = user_token
|
|
@@ -98,14 +100,12 @@ class TinybirdAgent:
|
|
|
98
100
|
self.project = project
|
|
99
101
|
self.thinking_animation = ThinkingAnimation()
|
|
100
102
|
self.confirmed_plan_id: Optional[str] = None
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
else:
|
|
104
|
-
self.messages = []
|
|
103
|
+
self.feature = feature
|
|
104
|
+
self.messages: list[ModelMessage] = []
|
|
105
105
|
cli_config = CLIConfig.get_project_config()
|
|
106
106
|
regions = get_regions(cli_config)
|
|
107
107
|
self.agent = Agent(
|
|
108
|
-
model=create_model(user_token, host, workspace_id),
|
|
108
|
+
model=create_model(user_token, host, workspace_id, feature=feature),
|
|
109
109
|
deps_type=TinybirdAgentContext,
|
|
110
110
|
instructions=[agent_system_prompt],
|
|
111
111
|
tools=[
|
|
@@ -282,7 +282,14 @@ class TinybirdAgent:
|
|
|
282
282
|
- API Host: {ctx.deps.local_host}
|
|
283
283
|
- Token: {ctx.deps.local_token}
|
|
284
284
|
- UI Dashboard URL: {get_display_cloud_host(ctx.deps.local_host)}/{ctx.deps.workspace_name}
|
|
285
|
-
|
|
285
|
+
- ClickHouse native HTTP interface:
|
|
286
|
+
- Protocol: HTTP
|
|
287
|
+
- Host: localhost
|
|
288
|
+
- Port: 7182
|
|
289
|
+
- Full URL: http://localhost:7182
|
|
290
|
+
- Username: {ctx.deps.workspace_name} # Optional, for identification purposes
|
|
291
|
+
- Password: __TB_CLOUD_TOKEN__ # Your Tinybird auth token
|
|
292
|
+
"""
|
|
286
293
|
|
|
287
294
|
@self.agent.instructions
|
|
288
295
|
def get_cloud_host(ctx: RunContext[TinybirdAgentContext]) -> str:
|
|
@@ -300,6 +307,7 @@ class TinybirdAgent:
|
|
|
300
307
|
|
|
301
308
|
region_provider = region["provider"]
|
|
302
309
|
region_name = region["name"]
|
|
310
|
+
ch_host = get_clickhouse_host(ctx.deps.host)
|
|
303
311
|
return f"""
|
|
304
312
|
# Tinybird Cloud info (region details):
|
|
305
313
|
- API Host: {ctx.deps.host}
|
|
@@ -308,6 +316,14 @@ class TinybirdAgent:
|
|
|
308
316
|
- Region provider: {region_provider}
|
|
309
317
|
- Region name: {region_name}
|
|
310
318
|
- UI Dashboard URL: {get_display_cloud_host(ctx.deps.host)}/{ctx.deps.workspace_name}
|
|
319
|
+
- ClickHouse native HTTP interface:
|
|
320
|
+
- Protocol: HTTPS
|
|
321
|
+
- Host: {ch_host.replace("https://", "").replace(":443", "")}
|
|
322
|
+
- Port: 443 (HTTPS)
|
|
323
|
+
- SSL/TLS: Required (enabled)
|
|
324
|
+
- Full URL: {ch_host}
|
|
325
|
+
- Username: {ctx.deps.workspace_name} # Optional, for identification purposes
|
|
326
|
+
- Password: __TB_CLOUD_TOKEN__ # Your Tinybird auth token
|
|
311
327
|
"""
|
|
312
328
|
|
|
313
329
|
@self.agent.instructions
|
|
@@ -322,6 +338,14 @@ class TinybirdAgent:
|
|
|
322
338
|
def get_project_files(ctx: RunContext[TinybirdAgentContext]) -> str:
|
|
323
339
|
return resources_prompt(self.project)
|
|
324
340
|
|
|
341
|
+
@self.agent.instructions
|
|
342
|
+
def get_vendor_files(ctx: RunContext[TinybirdAgentContext]) -> str:
|
|
343
|
+
return vendor_files_prompt(self.project)
|
|
344
|
+
|
|
345
|
+
@self.agent.instructions
|
|
346
|
+
def get_fixture_files(ctx: RunContext[TinybirdAgentContext]) -> str:
|
|
347
|
+
return fixtures_prompt(self.project)
|
|
348
|
+
|
|
325
349
|
@self.agent.instructions
|
|
326
350
|
def get_service_datasources(ctx: RunContext[TinybirdAgentContext]) -> str:
|
|
327
351
|
return service_datasources_prompt()
|
|
@@ -355,7 +379,6 @@ class TinybirdAgent:
|
|
|
355
379
|
build_project=partial(build_project, project=project, config=config),
|
|
356
380
|
deploy_project=partial(deploy_project, project=project, config=config),
|
|
357
381
|
deploy_check_project=partial(deploy_check_project, project=project, config=config),
|
|
358
|
-
mock_data=partial(mock_data, project=project, config=config),
|
|
359
382
|
append_data_local=partial(append_data_local, config=config),
|
|
360
383
|
append_data_cloud=partial(append_data_cloud, config=config),
|
|
361
384
|
analyze_fixture=partial(analyze_fixture, config=config),
|
|
@@ -363,7 +386,7 @@ class TinybirdAgent:
|
|
|
363
386
|
execute_query_local=partial(execute_query_local, config=config),
|
|
364
387
|
request_endpoint_cloud=partial(request_endpoint_cloud, config=config),
|
|
365
388
|
request_endpoint_local=partial(request_endpoint_local, config=config),
|
|
366
|
-
build_project_test=partial(build_project_test, project=project, client=test_client),
|
|
389
|
+
build_project_test=partial(build_project_test, project=project, client=test_client, config=config),
|
|
367
390
|
get_pipe_data_test=partial(get_pipe_data_test, client=test_client),
|
|
368
391
|
get_datasource_datafile_cloud=partial(get_datasource_datafile_cloud, config=config),
|
|
369
392
|
get_datasource_datafile_local=partial(get_datasource_datafile_local, config=config),
|
|
@@ -372,7 +395,7 @@ class TinybirdAgent:
|
|
|
372
395
|
get_connection_datafile_cloud=partial(get_connection_datafile_cloud, config=config),
|
|
373
396
|
get_connection_datafile_local=partial(get_connection_datafile_local, config=config),
|
|
374
397
|
get_project_files=project.get_project_files,
|
|
375
|
-
run_tests=partial(run_tests, project=project, client=test_client),
|
|
398
|
+
run_tests=partial(run_tests, project=project, client=test_client, config=config),
|
|
376
399
|
folder=folder,
|
|
377
400
|
thinking_animation=self.thinking_animation,
|
|
378
401
|
workspace_id=self.workspace_id,
|
|
@@ -402,7 +425,6 @@ class TinybirdAgent:
|
|
|
402
425
|
save_messages(new_messages)
|
|
403
426
|
self.thinking_animation.stop()
|
|
404
427
|
click.echo(result.output)
|
|
405
|
-
self.echo_usage(config)
|
|
406
428
|
|
|
407
429
|
async def run_iter(self, user_prompt: str, config: dict[str, Any], run_id: Optional[str] = None) -> None:
|
|
408
430
|
model = create_model(self.user_token, self.host, self.workspace_id, run_id=run_id)
|
|
@@ -429,34 +451,67 @@ class TinybirdAgent:
|
|
|
429
451
|
self.messages.extend(new_messages)
|
|
430
452
|
save_messages(new_messages)
|
|
431
453
|
self.thinking_animation.stop()
|
|
432
|
-
self.echo_usage(config)
|
|
433
454
|
|
|
434
|
-
def echo_usage(self, config: dict[str, Any]) -> None:
|
|
455
|
+
def echo_usage(self, config: dict[str, Any], show_credits: bool = False) -> None:
|
|
435
456
|
try:
|
|
436
457
|
client = _get_tb_client(config["user_token"], config["host"])
|
|
437
458
|
workspace_id = config.get("id", "")
|
|
438
459
|
workspace = client.workspace(workspace_id, with_organization=True, version="v1")
|
|
460
|
+
is_free_plan = workspace["organization"]["plan"].get("billing") == "free_shared_infrastructure_usage"
|
|
461
|
+
|
|
462
|
+
if not is_free_plan and not show_credits:
|
|
463
|
+
return
|
|
464
|
+
|
|
439
465
|
limits_data = client.organization_limits(workspace["organization"]["id"])
|
|
440
|
-
|
|
441
|
-
|
|
442
|
-
|
|
443
|
-
remaining_credits = round(max(
|
|
444
|
-
|
|
445
|
-
|
|
466
|
+
llm_usage_limits = limits_data.get("limits", {}).get("llm_usage", {})
|
|
467
|
+
current_llm_usage = llm_usage_limits.get("quantity") or 0
|
|
468
|
+
llm_usage = llm_usage_limits.get("max") or 0
|
|
469
|
+
remaining_credits = round(max(llm_usage - current_llm_usage, 0), 2)
|
|
470
|
+
current_llm_usage = round(min(llm_usage, current_llm_usage), 2)
|
|
471
|
+
|
|
472
|
+
if not llm_usage:
|
|
446
473
|
return
|
|
447
|
-
|
|
448
|
-
|
|
474
|
+
|
|
475
|
+
def get_message(current_llm_usage, llm_usage: int) -> tuple[Callable[..., str], str, bool]:
|
|
476
|
+
warning_threshold = llm_usage * 0.8
|
|
477
|
+
ui_host = get_display_cloud_host(config["host"])
|
|
478
|
+
|
|
479
|
+
if is_free_plan:
|
|
480
|
+
upgrade_link = f"{ui_host}/organizations/{workspace['organization']['name']}/upgrade?from=agent"
|
|
481
|
+
if current_llm_usage >= llm_usage:
|
|
482
|
+
return (
|
|
483
|
+
FeedbackManager.error,
|
|
484
|
+
f" You have reached the maximum number of credits. Please upgrade to continue using Tinybird Code: {upgrade_link}",
|
|
485
|
+
True,
|
|
486
|
+
)
|
|
487
|
+
if current_llm_usage >= warning_threshold:
|
|
488
|
+
return (
|
|
489
|
+
FeedbackManager.warning,
|
|
490
|
+
f" You are reaching the maximum number of credits. Please upgrade to continue using Tinybird Code: {upgrade_link}",
|
|
491
|
+
False,
|
|
492
|
+
)
|
|
493
|
+
return FeedbackManager.gray, "", False
|
|
494
|
+
|
|
495
|
+
message_color, upgrade_message, should_exit = get_message(current_llm_usage, llm_usage)
|
|
449
496
|
click.echo(
|
|
450
497
|
message_color(
|
|
451
|
-
message=f"{remaining_credits} credits left ({
|
|
498
|
+
message=f"{remaining_credits} credits left ({current_llm_usage}/{llm_usage}).{upgrade_message}"
|
|
452
499
|
)
|
|
453
500
|
)
|
|
501
|
+
|
|
502
|
+
if should_exit:
|
|
503
|
+
sys_exit("tinybird_code_error", "Maximum number of credits reached")
|
|
504
|
+
|
|
454
505
|
except Exception:
|
|
455
506
|
pass
|
|
456
507
|
|
|
457
508
|
|
|
458
509
|
def run_agent(
|
|
459
|
-
config: dict[str, Any],
|
|
510
|
+
config: dict[str, Any],
|
|
511
|
+
project: Project,
|
|
512
|
+
dangerously_skip_permissions: bool,
|
|
513
|
+
prompt: Optional[str] = None,
|
|
514
|
+
feature: Optional[str] = None,
|
|
460
515
|
):
|
|
461
516
|
if not prompt:
|
|
462
517
|
latest_version = CheckPypi().get_latest_version()
|
|
@@ -471,10 +526,13 @@ def run_agent(
|
|
|
471
526
|
)
|
|
472
527
|
if yes:
|
|
473
528
|
update_cli()
|
|
474
|
-
|
|
475
|
-
|
|
476
|
-
|
|
477
|
-
|
|
529
|
+
|
|
530
|
+
if not prompt:
|
|
531
|
+
click.echo(FeedbackManager.highlight(message="» Initializing Tinybird Code..."))
|
|
532
|
+
|
|
533
|
+
token = config.get("token", "")
|
|
534
|
+
host = config.get("host", "")
|
|
535
|
+
user_token = config.get("user_token", "")
|
|
478
536
|
workspace_id = config.get("id", "")
|
|
479
537
|
workspace_name = config.get("name", "")
|
|
480
538
|
try:
|
|
@@ -491,8 +549,8 @@ def run_agent(
|
|
|
491
549
|
login(host, auth_host="https://cloud.tinybird.co", workspace=None, interactive=False, method="browser")
|
|
492
550
|
cli_config = CLIConfig.get_project_config()
|
|
493
551
|
config = {**config, **cli_config.to_dict()}
|
|
494
|
-
token = cli_config.get_token()
|
|
495
|
-
user_token = cli_config.get_user_token()
|
|
552
|
+
token = cli_config.get_token() or ""
|
|
553
|
+
user_token = cli_config.get_user_token() or ""
|
|
496
554
|
host = cli_config.get_host()
|
|
497
555
|
workspace_id = cli_config.get("id", "")
|
|
498
556
|
workspace_name = cli_config.get("name", "")
|
|
@@ -530,6 +588,7 @@ def run_agent(
|
|
|
530
588
|
project,
|
|
531
589
|
dangerously_skip_permissions,
|
|
532
590
|
prompt_mode,
|
|
591
|
+
feature,
|
|
533
592
|
)
|
|
534
593
|
|
|
535
594
|
# Print mode: run once with the provided prompt and exit
|
|
@@ -551,7 +610,6 @@ def run_agent(
|
|
|
551
610
|
)
|
|
552
611
|
)
|
|
553
612
|
agent.echo_usage(config)
|
|
554
|
-
click.echo()
|
|
555
613
|
|
|
556
614
|
except Exception as e:
|
|
557
615
|
click.echo(FeedbackManager.error(message=f"Failed to initialize agent: {e}"))
|
|
@@ -604,6 +662,9 @@ def run_agent(
|
|
|
604
662
|
elif user_input.lower() == "/help":
|
|
605
663
|
subprocess.run(["tb", "--help"], check=True)
|
|
606
664
|
continue
|
|
665
|
+
elif user_input.lower() == "/usage":
|
|
666
|
+
agent.echo_usage(config, show_credits=True)
|
|
667
|
+
continue
|
|
607
668
|
elif user_input.strip() == "":
|
|
608
669
|
continue
|
|
609
670
|
else:
|
|
@@ -642,6 +703,7 @@ def build_project(
|
|
|
642
703
|
build_error = build_process(
|
|
643
704
|
project=project,
|
|
644
705
|
tb_client=client,
|
|
706
|
+
config=config,
|
|
645
707
|
watch=False,
|
|
646
708
|
silent=silent,
|
|
647
709
|
exit_on_error=False,
|
|
@@ -654,11 +716,13 @@ def build_project(
|
|
|
654
716
|
def build_project_test(
|
|
655
717
|
client: TinyB,
|
|
656
718
|
project: Project,
|
|
719
|
+
config: dict[str, Any],
|
|
657
720
|
silent: bool = False,
|
|
658
721
|
) -> None:
|
|
659
722
|
build_error = build_process(
|
|
660
723
|
project=project,
|
|
661
724
|
tb_client=client,
|
|
725
|
+
config=config,
|
|
662
726
|
watch=False,
|
|
663
727
|
silent=silent,
|
|
664
728
|
exit_on_error=False,
|
|
@@ -703,36 +767,6 @@ def append_data_cloud(config: dict[str, Any], datasource_name: str, path: str) -
|
|
|
703
767
|
append_mock_data(client, datasource_name, path)
|
|
704
768
|
|
|
705
769
|
|
|
706
|
-
def mock_data(
|
|
707
|
-
config: dict[str, Any],
|
|
708
|
-
project: Project,
|
|
709
|
-
datasource_name: str,
|
|
710
|
-
data_format: str,
|
|
711
|
-
rows: int,
|
|
712
|
-
context: Optional[str] = None,
|
|
713
|
-
) -> list[dict[str, Any]]:
|
|
714
|
-
client = get_tinybird_local_client(config, test=False, silent=False)
|
|
715
|
-
cli_config = CLIConfig.get_project_config()
|
|
716
|
-
datasource_path = project.get_resource_path(datasource_name, "datasource")
|
|
717
|
-
|
|
718
|
-
if not datasource_path:
|
|
719
|
-
raise CLIMockException(f"Datasource {datasource_name} not found")
|
|
720
|
-
|
|
721
|
-
datasource_content = Path(datasource_path).read_text()
|
|
722
|
-
return create_mock_data(
|
|
723
|
-
datasource_name,
|
|
724
|
-
datasource_content,
|
|
725
|
-
rows,
|
|
726
|
-
context or "",
|
|
727
|
-
cli_config,
|
|
728
|
-
config,
|
|
729
|
-
cli_config.get_user_token() or "",
|
|
730
|
-
client,
|
|
731
|
-
data_format,
|
|
732
|
-
project.folder,
|
|
733
|
-
)
|
|
734
|
-
|
|
735
|
-
|
|
736
770
|
def analyze_fixture(config: dict[str, Any], fixture_path: str, format: str = "json") -> dict[str, Any]:
|
|
737
771
|
local_client = get_tinybird_local_client(config, test=False, silent=True)
|
|
738
772
|
meta, _data = _analyze(fixture_path, local_client, format)
|
|
@@ -811,9 +845,11 @@ def get_connection_datafile_local(config: dict[str, Any], connection_name: str)
|
|
|
811
845
|
return "Connection not found"
|
|
812
846
|
|
|
813
847
|
|
|
814
|
-
def run_tests(
|
|
848
|
+
def run_tests(
|
|
849
|
+
client: TinyB, project: Project, config: dict[str, Any], pipe_name: Optional[str] = None
|
|
850
|
+
) -> Optional[str]:
|
|
815
851
|
try:
|
|
816
|
-
return run_tests_common(name=(pipe_name,) if pipe_name else (), project=project, client=client)
|
|
852
|
+
return run_tests_common(name=(pipe_name,) if pipe_name else (), project=project, client=client, config=config)
|
|
817
853
|
except SystemExit as e:
|
|
818
854
|
raise Exception(e.args[0])
|
|
819
855
|
|
|
@@ -54,20 +54,20 @@ def display_banner():
|
|
|
54
54
|
"""Convert RGB values to ANSI escape code"""
|
|
55
55
|
if use_truecolor:
|
|
56
56
|
return f"\033[38;2;{r};{g};{b}m"
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
57
|
+
|
|
58
|
+
# Convert to 8-bit color (256 color palette)
|
|
59
|
+
# Simple approximation: map RGB to 216-color cube + grayscale
|
|
60
|
+
if r == g == b:
|
|
61
|
+
# Grayscale
|
|
62
|
+
gray = int(r / 255 * 23) + 232
|
|
63
|
+
return f"\033[38;5;{gray}m"
|
|
64
|
+
|
|
65
|
+
# Color cube (6x6x6)
|
|
66
|
+
r_idx = int(r / 255 * 5)
|
|
67
|
+
g_idx = int(g / 255 * 5)
|
|
68
|
+
b_idx = int(b / 255 * 5)
|
|
69
|
+
color_idx = 16 + (36 * r_idx) + (6 * g_idx) + b_idx
|
|
70
|
+
return f"\033[38;5;{color_idx}m"
|
|
71
71
|
|
|
72
72
|
# Define solid color (corresponding to #27f795)
|
|
73
73
|
solid_color = [39, 247, 149] # #27f795 in RGB
|
|
@@ -84,4 +84,4 @@ def display_banner():
|
|
|
84
84
|
colored_line += f"{color_code}{char}"
|
|
85
85
|
|
|
86
86
|
click.echo(colored_line + reset)
|
|
87
|
-
click.echo()
|
|
87
|
+
click.echo("")
|
|
@@ -12,6 +12,7 @@ from tinybird.tb.modules.agent.prompts import (
|
|
|
12
12
|
resources_prompt,
|
|
13
13
|
service_datasources_prompt,
|
|
14
14
|
tone_and_style_instructions,
|
|
15
|
+
vendor_files_prompt,
|
|
15
16
|
)
|
|
16
17
|
from tinybird.tb.modules.agent.tools.diff_resource import diff_resource
|
|
17
18
|
from tinybird.tb.modules.agent.tools.execute_query import execute_query
|
|
@@ -76,6 +77,10 @@ Once you finish the task, return a valid response for the task to complete.
|
|
|
76
77
|
def get_project_files(ctx: RunContext[TinybirdAgentContext]) -> str:
|
|
77
78
|
return resources_prompt(self.project)
|
|
78
79
|
|
|
80
|
+
@self.agent.instructions
|
|
81
|
+
def get_vendor_files(ctx: RunContext[TinybirdAgentContext]) -> str:
|
|
82
|
+
return vendor_files_prompt(self.project)
|
|
83
|
+
|
|
79
84
|
@self.agent.instructions
|
|
80
85
|
def get_service_datasources(ctx: RunContext[TinybirdAgentContext]) -> str:
|
|
81
86
|
return service_datasources_prompt()
|
|
@@ -6,7 +6,7 @@ from pydantic_ai.usage import Usage
|
|
|
6
6
|
|
|
7
7
|
from tinybird.tb.modules.agent.animations import ThinkingAnimation
|
|
8
8
|
from tinybird.tb.modules.agent.models import create_model
|
|
9
|
-
from tinybird.tb.modules.agent.prompts import resources_prompt
|
|
9
|
+
from tinybird.tb.modules.agent.prompts import fixtures_prompt, resources_prompt
|
|
10
10
|
from tinybird.tb.modules.agent.tools.mock import generate_mock_fixture
|
|
11
11
|
from tinybird.tb.modules.agent.utils import TinybirdAgentContext
|
|
12
12
|
from tinybird.tb.modules.project import Project
|
|
@@ -194,6 +194,10 @@ Today is {datetime.now().strftime("%Y-%m-%d")}
|
|
|
194
194
|
def get_project_files(ctx: RunContext[TinybirdAgentContext]) -> str:
|
|
195
195
|
return resources_prompt(self.project)
|
|
196
196
|
|
|
197
|
+
@self.agent.instructions
|
|
198
|
+
def get_fixture_files(ctx: RunContext[TinybirdAgentContext]) -> str:
|
|
199
|
+
return fixtures_prompt(self.project)
|
|
200
|
+
|
|
197
201
|
def run(self, task: str, deps: TinybirdAgentContext, usage: Usage):
|
|
198
202
|
result = self.agent.run_sync(
|
|
199
203
|
task,
|