tinybird 0.0.1.dev246__py3-none-any.whl → 0.0.1.dev247__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of tinybird might be problematic. Click here for more details.
- tinybird/ch_utils/constants.py +2 -0
- tinybird/tb/__cli__.py +2 -2
- tinybird/tb/modules/agent/agent.py +74 -8
- tinybird/tb/modules/agent/models.py +6 -0
- tinybird/tb/modules/agent/prompts.py +57 -29
- tinybird/tb/modules/agent/tools/append.py +55 -0
- tinybird/tb/modules/agent/tools/build.py +1 -0
- tinybird/tb/modules/agent/tools/create_datafile.py +81 -2
- tinybird/tb/modules/agent/tools/deploy.py +1 -1
- tinybird/tb/modules/agent/tools/mock.py +59 -0
- tinybird/tb/modules/agent/tools/plan.py +1 -1
- tinybird/tb/modules/agent/tools/read_fixture_data.py +28 -0
- tinybird/tb/modules/agent/utils.py +4 -0
- tinybird/tb/modules/build.py +4 -1
- tinybird/tb/modules/build_common.py +2 -3
- tinybird/tb/modules/cli.py +9 -1
- tinybird/tb/modules/create.py +1 -1
- tinybird/tb/modules/feedback_manager.py +1 -0
- tinybird/tb/modules/llm.py +1 -1
- tinybird/tb/modules/mock.py +3 -69
- tinybird/tb/modules/mock_common.py +71 -0
- tinybird/tb/modules/project.py +9 -0
- {tinybird-0.0.1.dev246.dist-info → tinybird-0.0.1.dev247.dist-info}/METADATA +1 -1
- {tinybird-0.0.1.dev246.dist-info → tinybird-0.0.1.dev247.dist-info}/RECORD +27 -23
- {tinybird-0.0.1.dev246.dist-info → tinybird-0.0.1.dev247.dist-info}/WHEEL +0 -0
- {tinybird-0.0.1.dev246.dist-info → tinybird-0.0.1.dev247.dist-info}/entry_points.txt +0 -0
- {tinybird-0.0.1.dev246.dist-info → tinybird-0.0.1.dev247.dist-info}/top_level.txt +0 -0
tinybird/ch_utils/constants.py
CHANGED
tinybird/tb/__cli__.py
CHANGED
|
@@ -4,5 +4,5 @@ __description__ = 'Tinybird Command Line Tool'
|
|
|
4
4
|
__url__ = 'https://www.tinybird.co/docs/forward/commands'
|
|
5
5
|
__author__ = 'Tinybird'
|
|
6
6
|
__author_email__ = 'support@tinybird.co'
|
|
7
|
-
__version__ = '0.0.1.
|
|
8
|
-
__revision__ = '
|
|
7
|
+
__version__ = '0.0.1.dev247'
|
|
8
|
+
__revision__ = '379a827'
|
|
@@ -2,6 +2,7 @@ import subprocess
|
|
|
2
2
|
import sys
|
|
3
3
|
from datetime import datetime
|
|
4
4
|
from functools import partial
|
|
5
|
+
from pathlib import Path
|
|
5
6
|
from typing import Any
|
|
6
7
|
|
|
7
8
|
import click
|
|
@@ -28,34 +29,40 @@ from tinybird.tb.client import TinyB
|
|
|
28
29
|
from tinybird.tb.modules.agent.animations import ThinkingAnimation
|
|
29
30
|
from tinybird.tb.modules.agent.banner import display_banner
|
|
30
31
|
from tinybird.tb.modules.agent.memory import clear_history, load_history
|
|
31
|
-
from tinybird.tb.modules.agent.models import create_model
|
|
32
|
+
from tinybird.tb.modules.agent.models import create_model, model_costs
|
|
32
33
|
from tinybird.tb.modules.agent.prompts import (
|
|
33
34
|
datafile_instructions,
|
|
34
35
|
plan_instructions,
|
|
35
36
|
resources_prompt,
|
|
36
37
|
sql_instructions,
|
|
37
38
|
)
|
|
39
|
+
from tinybird.tb.modules.agent.tools.append import append
|
|
38
40
|
from tinybird.tb.modules.agent.tools.build import build
|
|
39
41
|
from tinybird.tb.modules.agent.tools.create_datafile import create_datafile
|
|
40
42
|
from tinybird.tb.modules.agent.tools.deploy import deploy
|
|
41
43
|
from tinybird.tb.modules.agent.tools.deploy_check import deploy_check
|
|
42
44
|
from tinybird.tb.modules.agent.tools.explore import explore_data
|
|
45
|
+
from tinybird.tb.modules.agent.tools.mock import mock
|
|
43
46
|
from tinybird.tb.modules.agent.tools.plan import plan
|
|
44
47
|
from tinybird.tb.modules.agent.tools.preview_datafile import preview_datafile
|
|
48
|
+
from tinybird.tb.modules.agent.tools.read_fixture_data import read_fixture_data
|
|
45
49
|
from tinybird.tb.modules.agent.utils import TinybirdAgentContext
|
|
46
50
|
from tinybird.tb.modules.build_common import process as build_process
|
|
47
|
-
from tinybird.tb.modules.common import _get_tb_client
|
|
51
|
+
from tinybird.tb.modules.common import _analyze, _get_tb_client
|
|
52
|
+
from tinybird.tb.modules.config import CLIConfig
|
|
48
53
|
from tinybird.tb.modules.deployment_common import create_deployment
|
|
49
|
-
from tinybird.tb.modules.exceptions import CLIBuildException
|
|
54
|
+
from tinybird.tb.modules.exceptions import CLIBuildException, CLIMockException
|
|
50
55
|
from tinybird.tb.modules.feedback_manager import FeedbackManager
|
|
51
56
|
from tinybird.tb.modules.local_common import get_tinybird_local_client
|
|
57
|
+
from tinybird.tb.modules.mock_common import append_mock_data, create_mock_data
|
|
52
58
|
from tinybird.tb.modules.project import Project
|
|
53
59
|
|
|
54
60
|
|
|
55
61
|
class TinybirdAgent:
|
|
56
|
-
def __init__(self, token: str, host: str, project: Project):
|
|
62
|
+
def __init__(self, token: str, host: str, project: Project, dangerously_skip_permissions: bool):
|
|
57
63
|
self.token = token
|
|
58
64
|
self.host = host
|
|
65
|
+
self.dangerously_skip_permissions = dangerously_skip_permissions
|
|
59
66
|
self.project = project
|
|
60
67
|
self.messages: list[ModelMessage] = []
|
|
61
68
|
self.agent = Agent(
|
|
@@ -94,6 +101,9 @@ You have access to the following tools:
|
|
|
94
101
|
5. `build` - Build the project.
|
|
95
102
|
6. `deploy` - Deploy the project to Tinybird Cloud.
|
|
96
103
|
7. `deploy_check` - Check if the project can be deployed to Tinybird Cloud before deploying it.
|
|
104
|
+
8. `mock` - Create mock data for a landing datasource.
|
|
105
|
+
9. `read_fixture_data` - Read a fixture data file present in the project folder.
|
|
106
|
+
10. `append` - Append existing fixture to a datasource.
|
|
97
107
|
|
|
98
108
|
# When creating or updating datafiles:
|
|
99
109
|
1. Use `plan` tool to plan the creation or update of resources.
|
|
@@ -150,6 +160,9 @@ Today is {datetime.now().strftime("%Y-%m-%d")}
|
|
|
150
160
|
Tool(build, docstring_format="google", require_parameter_descriptions=True, takes_ctx=True),
|
|
151
161
|
Tool(deploy, docstring_format="google", require_parameter_descriptions=True, takes_ctx=True),
|
|
152
162
|
Tool(deploy_check, docstring_format="google", require_parameter_descriptions=True, takes_ctx=True),
|
|
163
|
+
Tool(mock, docstring_format="google", require_parameter_descriptions=True, takes_ctx=True),
|
|
164
|
+
Tool(read_fixture_data, docstring_format="google", require_parameter_descriptions=True, takes_ctx=True),
|
|
165
|
+
Tool(append, docstring_format="google", require_parameter_descriptions=True, takes_ctx=True),
|
|
153
166
|
],
|
|
154
167
|
)
|
|
155
168
|
|
|
@@ -158,7 +171,7 @@ Today is {datetime.now().strftime("%Y-%m-%d")}
|
|
|
158
171
|
return self.messages[-5:] if len(self.messages) > 5 else self.messages
|
|
159
172
|
|
|
160
173
|
def run(self, user_prompt: str, config: dict[str, Any], project: Project) -> None:
|
|
161
|
-
user_prompt = f"{user_prompt}\n\n
|
|
174
|
+
user_prompt = f"{user_prompt}\n\n{resources_prompt(project)}"
|
|
162
175
|
client = TinyB(token=self.token, host=self.host)
|
|
163
176
|
folder = self.project.folder
|
|
164
177
|
|
|
@@ -172,28 +185,44 @@ Today is {datetime.now().strftime("%Y-%m-%d")}
|
|
|
172
185
|
build_project=partial(build_project, project=project, config=config),
|
|
173
186
|
deploy_project=partial(deploy_project, project=project, config=config),
|
|
174
187
|
deploy_check_project=partial(deploy_check_project, project=project, config=config),
|
|
188
|
+
mock_data=partial(mock_data, project=project, config=config),
|
|
189
|
+
append_data=partial(append_data, config=config),
|
|
190
|
+
analyze_fixture=partial(analyze_fixture, config=config),
|
|
175
191
|
get_project_files=project.get_project_files,
|
|
176
192
|
folder=folder,
|
|
177
193
|
thinking_animation=thinking_animation,
|
|
178
194
|
workspace_name=self.project.workspace_name,
|
|
195
|
+
dangerously_skip_permissions=self.dangerously_skip_permissions,
|
|
179
196
|
),
|
|
180
197
|
message_history=self.messages,
|
|
181
198
|
)
|
|
182
199
|
new_messages = result.new_messages()
|
|
183
200
|
self.messages.extend(new_messages)
|
|
184
201
|
thinking_animation.stop()
|
|
185
|
-
|
|
202
|
+
usage = result.usage()
|
|
203
|
+
request_tokens = usage.request_tokens or 0
|
|
204
|
+
response_tokens = usage.response_tokens or 0
|
|
205
|
+
total_tokens = usage.total_tokens or 0
|
|
206
|
+
cost = (
|
|
207
|
+
request_tokens * model_costs["input_cost_per_token"]
|
|
208
|
+
+ response_tokens * model_costs["output_cost_per_token"]
|
|
209
|
+
)
|
|
210
|
+
click.echo()
|
|
186
211
|
click.echo(result.output)
|
|
187
212
|
click.echo("\n")
|
|
213
|
+
click.echo(f"Input tokens: {request_tokens}")
|
|
214
|
+
click.echo(f"Output tokens: {response_tokens}")
|
|
215
|
+
click.echo(f"Total tokens: {total_tokens}")
|
|
216
|
+
click.echo(f"Cost: ${cost:.6f}")
|
|
188
217
|
|
|
189
218
|
|
|
190
|
-
def run_agent(config: dict[str, Any], project: Project):
|
|
219
|
+
def run_agent(config: dict[str, Any], project: Project, dangerously_skip_permissions: bool):
|
|
191
220
|
display_banner()
|
|
192
221
|
|
|
193
222
|
try:
|
|
194
223
|
token = config["token"]
|
|
195
224
|
host = config["host"]
|
|
196
|
-
agent = TinybirdAgent(token, host, project)
|
|
225
|
+
agent = TinybirdAgent(token, host, project, dangerously_skip_permissions)
|
|
197
226
|
click.echo()
|
|
198
227
|
if config.get("token"):
|
|
199
228
|
click.echo(FeedbackManager.info(message="Describe what you want to create and I'll help you build it"))
|
|
@@ -289,3 +318,40 @@ def deploy_check_project(config: dict[str, Any], project: Project) -> None:
|
|
|
289
318
|
wait=True,
|
|
290
319
|
auto=True,
|
|
291
320
|
)
|
|
321
|
+
|
|
322
|
+
|
|
323
|
+
def append_data(config: dict[str, Any], datasource_name: str, path: str) -> None:
|
|
324
|
+
client = get_tinybird_local_client(config, test=False, silent=False)
|
|
325
|
+
append_mock_data(client, datasource_name, path)
|
|
326
|
+
|
|
327
|
+
|
|
328
|
+
def mock_data(
|
|
329
|
+
config: dict[str, Any], project: Project, datasource_name: str, data_format: str, rows: int
|
|
330
|
+
) -> list[dict[str, Any]]:
|
|
331
|
+
client = get_tinybird_local_client(config, test=False, silent=False)
|
|
332
|
+
cli_config = CLIConfig.get_project_config()
|
|
333
|
+
datasource_path = project.get_resource_path(datasource_name, "datasource")
|
|
334
|
+
|
|
335
|
+
if not datasource_path:
|
|
336
|
+
raise CLIMockException(f"Datasource {datasource_name} not found")
|
|
337
|
+
|
|
338
|
+
datasource_content = Path(datasource_path).read_text()
|
|
339
|
+
prompt = ""
|
|
340
|
+
return create_mock_data(
|
|
341
|
+
datasource_name,
|
|
342
|
+
datasource_content,
|
|
343
|
+
rows,
|
|
344
|
+
prompt,
|
|
345
|
+
cli_config,
|
|
346
|
+
config,
|
|
347
|
+
cli_config.get_user_token() or "",
|
|
348
|
+
client,
|
|
349
|
+
data_format,
|
|
350
|
+
project.folder,
|
|
351
|
+
)
|
|
352
|
+
|
|
353
|
+
|
|
354
|
+
def analyze_fixture(config: dict[str, Any], fixture_path: str) -> dict[str, Any]:
|
|
355
|
+
local_client = get_tinybird_local_client(config, test=False, silent=True)
|
|
356
|
+
meta, _data = _analyze(fixture_path, local_client, Path(fixture_path).suffix.lstrip("."))
|
|
357
|
+
return meta
|
|
@@ -8,7 +8,7 @@ When asked to create a plan, you MUST respond with this EXACT format and NOTHING
|
|
|
8
8
|
Plan description: [One sentence describing what will be built]
|
|
9
9
|
|
|
10
10
|
Steps:
|
|
11
|
-
1. Connection: [name] - [description]
|
|
11
|
+
1. Connection: [name] - [description]
|
|
12
12
|
2. Datasource: [name] - [description] - Depends on: [connection_name (optional)]
|
|
13
13
|
3. Endpoint: [name] - [description] - Depends on: [resources]
|
|
14
14
|
4. Materialized pipe: [name] - [description] - Depends on: [resources]
|
|
@@ -16,9 +16,15 @@ Steps:
|
|
|
16
16
|
6. Sink: [name] - [description] - Depends on: [resources]
|
|
17
17
|
7. Copy: [name] - [description] - Depends on: [resources]
|
|
18
18
|
8. Build project
|
|
19
|
+
9. Generate mock data: [datasource_name]
|
|
20
|
+
10. Append existing fixture: [fixture_pathname] - Target: [datasource_name]
|
|
19
21
|
|
|
22
|
+
<dev_notes>
|
|
20
23
|
You can skip steps where resources will not be created or updated.
|
|
21
|
-
Always add
|
|
24
|
+
Always add 'Build project' step after generating resources.
|
|
25
|
+
Always add 'Generate mock data' step after building project if a landing datasource was created.
|
|
26
|
+
Always add 'Append existing fixture' step after building project if a fixture file was provided at the beginning of the plan.
|
|
27
|
+
</dev_notes>
|
|
22
28
|
|
|
23
29
|
Resource dependencies:
|
|
24
30
|
[resource_name]: [resources]
|
|
@@ -89,31 +95,53 @@ datafile_instructions = """
|
|
|
89
95
|
|
|
90
96
|
def resources_prompt(project: Project) -> str:
|
|
91
97
|
files = project.get_project_files()
|
|
98
|
+
fixture_files = project.get_fixture_files()
|
|
92
99
|
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
""
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
100
|
+
resources_content = "# Existing resources in the project:\n"
|
|
101
|
+
if files:
|
|
102
|
+
paths = [Path(file_path) for file_path in files]
|
|
103
|
+
|
|
104
|
+
resources_content += "\n".join(
|
|
105
|
+
[
|
|
106
|
+
f"""
|
|
107
|
+
<resource>
|
|
108
|
+
<path>{file_path.relative_to(project.folder)}</path>
|
|
109
|
+
<type>{get_resource_type(file_path)}</type>
|
|
110
|
+
<name>{file_path.stem}</name>
|
|
111
|
+
<content>{file_path.read_text()}</content>
|
|
112
|
+
</resource>
|
|
113
|
+
"""
|
|
114
|
+
for file_path in paths
|
|
115
|
+
]
|
|
116
|
+
)
|
|
117
|
+
else:
|
|
118
|
+
resources_content += "No resources found"
|
|
119
|
+
|
|
120
|
+
fixture_content = "# Fixture files in the project:\n"
|
|
121
|
+
if fixture_files:
|
|
122
|
+
paths = [Path(file_path) for file_path in fixture_files]
|
|
123
|
+
fixture_content += "\n".join(
|
|
124
|
+
[
|
|
125
|
+
f"""
|
|
126
|
+
<fixture>
|
|
127
|
+
<path>{file_path.relative_to(project.folder)}</path>
|
|
128
|
+
<name>{file_path.stem}</name>
|
|
129
|
+
</fixture>
|
|
130
|
+
"""
|
|
131
|
+
for file_path in paths
|
|
132
|
+
]
|
|
133
|
+
)
|
|
134
|
+
else:
|
|
135
|
+
fixture_content += "No fixture files found"
|
|
136
|
+
|
|
137
|
+
return resources_content + "\n" + fixture_content
|
|
138
|
+
|
|
139
|
+
|
|
140
|
+
def get_resource_type(path: Path) -> str:
|
|
141
|
+
if path.suffix.lower() == ".pipe":
|
|
142
|
+
return Project.get_pipe_type(str(path))
|
|
143
|
+
elif path.suffix.lower() == ".datasource":
|
|
144
|
+
return "datasource"
|
|
145
|
+
elif path.suffix.lower() == ".connection":
|
|
146
|
+
return "connection"
|
|
147
|
+
return "unknown"
|
|
@@ -0,0 +1,55 @@
|
|
|
1
|
+
import click
|
|
2
|
+
from pydantic_ai import RunContext
|
|
3
|
+
|
|
4
|
+
from tinybird.tb.modules.agent.utils import TinybirdAgentContext, show_options
|
|
5
|
+
from tinybird.tb.modules.feedback_manager import FeedbackManager
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
def get_append_confirmation(datasource_name: str) -> bool:
|
|
9
|
+
"""Get user confirmation for appending existing fixture"""
|
|
10
|
+
while True:
|
|
11
|
+
result = show_options(
|
|
12
|
+
options=["Yes, append existing fixture", "No, and tell Tinybird Code what to do"],
|
|
13
|
+
title=f"Do you want to append existing fixture for datasource {datasource_name}?",
|
|
14
|
+
)
|
|
15
|
+
|
|
16
|
+
if result is None: # Cancelled
|
|
17
|
+
return False
|
|
18
|
+
|
|
19
|
+
if result.startswith("Yes"):
|
|
20
|
+
return True
|
|
21
|
+
elif result.startswith("No"):
|
|
22
|
+
return False
|
|
23
|
+
|
|
24
|
+
return False
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
def append(ctx: RunContext[TinybirdAgentContext], datasource_name: str, fixture_pathname: str) -> str:
|
|
28
|
+
"""Append existing fixture to a datasource
|
|
29
|
+
|
|
30
|
+
Args:
|
|
31
|
+
datasource_name: Name of the datasource to append fixture to
|
|
32
|
+
fixture_pathname: Path to the fixture file to append
|
|
33
|
+
|
|
34
|
+
Returns:
|
|
35
|
+
str: Message indicating the success or failure of the appending
|
|
36
|
+
"""
|
|
37
|
+
try:
|
|
38
|
+
ctx.deps.thinking_animation.stop()
|
|
39
|
+
confirmation = ctx.deps.dangerously_skip_permissions or get_append_confirmation(datasource_name)
|
|
40
|
+
ctx.deps.thinking_animation.start()
|
|
41
|
+
|
|
42
|
+
if not confirmation:
|
|
43
|
+
return "User rejected appending existing fixture. Skipping..."
|
|
44
|
+
|
|
45
|
+
ctx.deps.thinking_animation.stop()
|
|
46
|
+
click.echo(FeedbackManager.highlight(message=f"\n» Appending {fixture_pathname} to {datasource_name}..."))
|
|
47
|
+
ctx.deps.append_data(datasource_name=datasource_name, path=fixture_pathname)
|
|
48
|
+
click.echo(FeedbackManager.success(message=f"✓ Data appended to {datasource_name}"))
|
|
49
|
+
ctx.deps.thinking_animation.start()
|
|
50
|
+
return f"Data appended to {datasource_name}"
|
|
51
|
+
except Exception as e:
|
|
52
|
+
ctx.deps.thinking_animation.stop()
|
|
53
|
+
click.echo(FeedbackManager.error(message=e))
|
|
54
|
+
ctx.deps.thinking_animation.start()
|
|
55
|
+
return f"Error appending fixture {fixture_pathname} to {datasource_name}: {e}"
|
|
@@ -9,6 +9,7 @@ def build(ctx: RunContext[TinybirdAgentContext]) -> str:
|
|
|
9
9
|
"""Build the project"""
|
|
10
10
|
try:
|
|
11
11
|
ctx.deps.thinking_animation.stop()
|
|
12
|
+
click.echo(FeedbackManager.highlight(message="\n» Building project..."))
|
|
12
13
|
ctx.deps.build_project(test=False, silent=False)
|
|
13
14
|
ctx.deps.thinking_animation.start()
|
|
14
15
|
return "Project built successfully"
|
|
@@ -1,5 +1,18 @@
|
|
|
1
|
+
import difflib
|
|
1
2
|
from pathlib import Path
|
|
2
3
|
|
|
4
|
+
try:
|
|
5
|
+
from colorama import Back, Fore, Style, init
|
|
6
|
+
|
|
7
|
+
init()
|
|
8
|
+
except ImportError: # fallback so that the imported classes always exist
|
|
9
|
+
|
|
10
|
+
class ColorFallback:
|
|
11
|
+
def __getattr__(self, name):
|
|
12
|
+
return ""
|
|
13
|
+
|
|
14
|
+
Fore = Back = Style = ColorFallback()
|
|
15
|
+
|
|
3
16
|
import click
|
|
4
17
|
from pydantic_ai import RunContext
|
|
5
18
|
|
|
@@ -8,6 +21,67 @@ from tinybird.tb.modules.exceptions import CLIBuildException
|
|
|
8
21
|
from tinybird.tb.modules.feedback_manager import FeedbackManager
|
|
9
22
|
|
|
10
23
|
|
|
24
|
+
def create_line_numbered_diff(original_content: str, new_content: str, filename: str) -> str:
|
|
25
|
+
"""Create a diff with line numbers similar to the example format"""
|
|
26
|
+
original_lines = original_content.splitlines()
|
|
27
|
+
new_lines = new_content.splitlines()
|
|
28
|
+
|
|
29
|
+
# Create a SequenceMatcher to find the differences
|
|
30
|
+
matcher = difflib.SequenceMatcher(None, original_lines, new_lines)
|
|
31
|
+
|
|
32
|
+
result = []
|
|
33
|
+
result.append(f"╭{'─' * 88}╮")
|
|
34
|
+
result.append(f"│ {filename:<86} │")
|
|
35
|
+
result.append(f"│{' ' * 88}│")
|
|
36
|
+
|
|
37
|
+
# Process the opcodes to build the diff
|
|
38
|
+
for tag, i1, i2, j1, j2 in matcher.get_opcodes():
|
|
39
|
+
if tag == "equal":
|
|
40
|
+
# Show context lines
|
|
41
|
+
for i, line in enumerate(original_lines[i1:i2]):
|
|
42
|
+
line_num = i1 + i + 1
|
|
43
|
+
result.append(f"│ {line_num:4} {line:<74} │")
|
|
44
|
+
elif tag == "replace":
|
|
45
|
+
# Show removed lines
|
|
46
|
+
for i, line in enumerate(original_lines[i1:i2]):
|
|
47
|
+
line_num = i1 + i + 1
|
|
48
|
+
result.append(f"│ {Back.RED}{line_num:4} - {line:<74}{Back.RESET} │")
|
|
49
|
+
# Show added lines
|
|
50
|
+
for i, line in enumerate(new_lines[j1:j2]):
|
|
51
|
+
line_num = i1 + i + 1
|
|
52
|
+
result.append(f"│ {Back.GREEN}{line_num:4} + {line:<74}{Back.RESET} │")
|
|
53
|
+
elif tag == "delete":
|
|
54
|
+
# Show removed lines
|
|
55
|
+
for i, line in enumerate(original_lines[i1:i2]):
|
|
56
|
+
line_num = i1 + i + 1
|
|
57
|
+
result.append(f"│ {Back.RED}{line_num:4} - {line:<74}{Back.RESET} │")
|
|
58
|
+
elif tag == "insert":
|
|
59
|
+
# Show added lines
|
|
60
|
+
for i, line in enumerate(new_lines[j1:j2]):
|
|
61
|
+
# Use the line number from the original position
|
|
62
|
+
line_num = i1 + i + 1
|
|
63
|
+
result.append(f"│ {Back.GREEN}{line_num:4} + {line:<74}{Back.RESET} │")
|
|
64
|
+
|
|
65
|
+
result.append(f"╰{'─' * 88}╯")
|
|
66
|
+
return "\n".join(result)
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
def create_line_numbered_content(content: str, filename: str) -> str:
|
|
70
|
+
"""Create a formatted display of file content with line numbers"""
|
|
71
|
+
lines = content.splitlines()
|
|
72
|
+
|
|
73
|
+
result = []
|
|
74
|
+
result.append(f"╭{'─' * 88}╮")
|
|
75
|
+
result.append(f"│ {filename:<86} │")
|
|
76
|
+
result.append(f"│{' ' * 88}│")
|
|
77
|
+
|
|
78
|
+
for i, line in enumerate(lines, 1):
|
|
79
|
+
result.append(f"│ {i:4} {line:<74} │")
|
|
80
|
+
|
|
81
|
+
result.append(f"╰{'─' * 88}╯")
|
|
82
|
+
return "\n".join(result)
|
|
83
|
+
|
|
84
|
+
|
|
11
85
|
def get_resource_confirmation(resource: Datafile, exists: bool) -> bool:
|
|
12
86
|
"""Get user confirmation for creating a resource"""
|
|
13
87
|
while True:
|
|
@@ -39,11 +113,16 @@ def create_datafile(ctx: RunContext[TinybirdAgentContext], resource: Datafile) -
|
|
|
39
113
|
"""
|
|
40
114
|
try:
|
|
41
115
|
ctx.deps.thinking_animation.stop()
|
|
42
|
-
click.echo(resource.content)
|
|
43
116
|
resource.pathname = resource.pathname.removeprefix("/")
|
|
44
117
|
path = Path(ctx.deps.folder) / resource.pathname
|
|
118
|
+
content = resource.content
|
|
45
119
|
exists = str(path) in ctx.deps.get_project_files()
|
|
46
|
-
|
|
120
|
+
if exists:
|
|
121
|
+
content = create_line_numbered_diff(path.read_text(), resource.content, resource.pathname)
|
|
122
|
+
else:
|
|
123
|
+
content = create_line_numbered_content(resource.content, resource.pathname)
|
|
124
|
+
click.echo(content)
|
|
125
|
+
confirmation = ctx.deps.dangerously_skip_permissions or get_resource_confirmation(resource, exists)
|
|
47
126
|
|
|
48
127
|
if not confirmation:
|
|
49
128
|
ctx.deps.thinking_animation.start()
|
|
@@ -28,7 +28,7 @@ def deploy(ctx: RunContext[TinybirdAgentContext]) -> str:
|
|
|
28
28
|
"""Deploy the project"""
|
|
29
29
|
try:
|
|
30
30
|
ctx.deps.thinking_animation.stop()
|
|
31
|
-
confirmation = get_deploy_confirmation()
|
|
31
|
+
confirmation = ctx.deps.dangerously_skip_permissions or get_deploy_confirmation()
|
|
32
32
|
ctx.deps.thinking_animation.start()
|
|
33
33
|
|
|
34
34
|
if not confirmation:
|
|
@@ -0,0 +1,59 @@
|
|
|
1
|
+
import click
|
|
2
|
+
from pydantic_ai import RunContext
|
|
3
|
+
|
|
4
|
+
from tinybird.tb.modules.agent.utils import TinybirdAgentContext, show_options
|
|
5
|
+
from tinybird.tb.modules.datafile.fixture import persist_fixture
|
|
6
|
+
from tinybird.tb.modules.feedback_manager import FeedbackManager
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
def get_mock_confirmation(datasource_name: str) -> bool:
|
|
10
|
+
"""Get user confirmation for creating mock data"""
|
|
11
|
+
while True:
|
|
12
|
+
result = show_options(
|
|
13
|
+
options=["Yes, create mock data", "No, and tell Tinybird Code what to do"],
|
|
14
|
+
title=f"Do you want to generate mock data for datasource {datasource_name}?",
|
|
15
|
+
)
|
|
16
|
+
|
|
17
|
+
if result is None: # Cancelled
|
|
18
|
+
return False
|
|
19
|
+
|
|
20
|
+
if result.startswith("Yes"):
|
|
21
|
+
return True
|
|
22
|
+
elif result.startswith("No"):
|
|
23
|
+
return False
|
|
24
|
+
|
|
25
|
+
return False
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
def mock(ctx: RunContext[TinybirdAgentContext], datasource_name: str, data_format: str, rows: int) -> str:
|
|
29
|
+
"""Create mock data for a datasource
|
|
30
|
+
|
|
31
|
+
Args:
|
|
32
|
+
datasource_name: Name of the datasource to create mock data for
|
|
33
|
+
data_format: Format of the mock data to create. Options: ndjson, csv
|
|
34
|
+
rows: Number of rows to create. If not provided, the default is 10
|
|
35
|
+
|
|
36
|
+
Returns:
|
|
37
|
+
str: Message indicating the success or failure of the mock data generation
|
|
38
|
+
"""
|
|
39
|
+
try:
|
|
40
|
+
ctx.deps.thinking_animation.stop()
|
|
41
|
+
confirmation = ctx.deps.dangerously_skip_permissions or get_mock_confirmation(datasource_name)
|
|
42
|
+
ctx.deps.thinking_animation.start()
|
|
43
|
+
|
|
44
|
+
if not confirmation:
|
|
45
|
+
return "User rejected mock data generation. Skipping..."
|
|
46
|
+
|
|
47
|
+
ctx.deps.thinking_animation.stop()
|
|
48
|
+
click.echo(FeedbackManager.highlight(message=f"\n» Generating mock data for {datasource_name}..."))
|
|
49
|
+
data = ctx.deps.mock_data(datasource_name=datasource_name, data_format=data_format, rows=rows)
|
|
50
|
+
fixture_path = persist_fixture(datasource_name, data, ctx.deps.folder, format=data_format)
|
|
51
|
+
ctx.deps.append_data(datasource_name=datasource_name, path=str(fixture_path))
|
|
52
|
+
click.echo(FeedbackManager.success(message=f"✓ Data generated for {datasource_name}"))
|
|
53
|
+
ctx.deps.thinking_animation.start()
|
|
54
|
+
return f"Mock data generated successfully for datasource {datasource_name}"
|
|
55
|
+
except Exception as e:
|
|
56
|
+
ctx.deps.thinking_animation.stop()
|
|
57
|
+
click.echo(FeedbackManager.error(message=e))
|
|
58
|
+
ctx.deps.thinking_animation.start()
|
|
59
|
+
return f"Error generating mock data: {e}"
|
|
@@ -35,7 +35,7 @@ def plan(ctx: RunContext[TinybirdAgentContext], plan: str) -> str:
|
|
|
35
35
|
try:
|
|
36
36
|
ctx.deps.thinking_animation.stop()
|
|
37
37
|
click.echo(plan)
|
|
38
|
-
confirmation = get_plan_confirmation()
|
|
38
|
+
confirmation = ctx.deps.dangerously_skip_permissions or get_plan_confirmation()
|
|
39
39
|
ctx.deps.thinking_animation.start()
|
|
40
40
|
|
|
41
41
|
if not confirmation:
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
import json
|
|
2
|
+
from pathlib import Path
|
|
3
|
+
|
|
4
|
+
from pydantic_ai import RunContext
|
|
5
|
+
|
|
6
|
+
from tinybird.tb.modules.agent.utils import TinybirdAgentContext
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
def read_fixture_data(ctx: RunContext[TinybirdAgentContext], fixture_pathname: str):
|
|
10
|
+
"""Read fixture data in the project folder
|
|
11
|
+
|
|
12
|
+
Args:
|
|
13
|
+
fixture_pathname (str): a path to a fixture file. Required.
|
|
14
|
+
|
|
15
|
+
Returns:
|
|
16
|
+
str: The content of the fixture data file.
|
|
17
|
+
"""
|
|
18
|
+
fixture_path = Path(ctx.deps.folder) / fixture_pathname.lstrip("/")
|
|
19
|
+
|
|
20
|
+
if not fixture_path.exists():
|
|
21
|
+
return f"No fixture data found for {fixture_pathname}. Please check the name of the fixture and try again."
|
|
22
|
+
|
|
23
|
+
response = ctx.deps.analyze_fixture(fixture_path=str(fixture_path))
|
|
24
|
+
# limit content to first 10 rows
|
|
25
|
+
data = response["preview"]["data"][:10]
|
|
26
|
+
schema = response["analysis"]["schema"]
|
|
27
|
+
|
|
28
|
+
return f"#Result of analysis of {fixture_pathname}:\n##Data sample:\n{json.dumps(data)}\n##Schema:\n{schema}"
|
|
@@ -26,6 +26,10 @@ class TinybirdAgentContext(BaseModel):
|
|
|
26
26
|
build_project: Callable[..., None]
|
|
27
27
|
deploy_project: Callable[[], None]
|
|
28
28
|
deploy_check_project: Callable[[], None]
|
|
29
|
+
mock_data: Callable[..., list[dict[str, Any]]]
|
|
30
|
+
append_data: Callable[..., None]
|
|
31
|
+
analyze_fixture: Callable[..., dict[str, Any]]
|
|
32
|
+
dangerously_skip_permissions: bool
|
|
29
33
|
|
|
30
34
|
|
|
31
35
|
default_style = Style.from_dict(
|
tinybird/tb/modules/build.py
CHANGED
|
@@ -3,7 +3,7 @@ import time
|
|
|
3
3
|
from copy import deepcopy
|
|
4
4
|
from functools import partial
|
|
5
5
|
from pathlib import Path
|
|
6
|
-
from typing import Callable, List
|
|
6
|
+
from typing import Any, Callable, Dict, List
|
|
7
7
|
from urllib.parse import urlencode
|
|
8
8
|
|
|
9
9
|
import click
|
|
@@ -32,8 +32,11 @@ def build(ctx: click.Context, watch: bool) -> None:
|
|
|
32
32
|
"""
|
|
33
33
|
Validate and build the project server side.
|
|
34
34
|
"""
|
|
35
|
+
obj: Dict[str, Any] = ctx.ensure_object(dict)
|
|
35
36
|
project: Project = ctx.ensure_object(dict)["project"]
|
|
36
37
|
tb_client: TinyB = ctx.ensure_object(dict)["client"]
|
|
38
|
+
if obj["env"] == "cloud":
|
|
39
|
+
raise click.ClickException(FeedbackManager.error_build_only_supported_in_local())
|
|
37
40
|
|
|
38
41
|
if project.has_deeper_level():
|
|
39
42
|
click.echo(
|
|
@@ -3,7 +3,7 @@ import logging
|
|
|
3
3
|
import time
|
|
4
4
|
from pathlib import Path
|
|
5
5
|
from typing import Optional
|
|
6
|
-
from urllib.parse import urlencode
|
|
6
|
+
from urllib.parse import urlencode, urljoin
|
|
7
7
|
|
|
8
8
|
import click
|
|
9
9
|
import requests
|
|
@@ -25,7 +25,6 @@ def process(
|
|
|
25
25
|
file_changed: Optional[str] = None,
|
|
26
26
|
diff: Optional[str] = None,
|
|
27
27
|
silent: bool = False,
|
|
28
|
-
error: bool = False,
|
|
29
28
|
build_status: Optional[BuildStatus] = None,
|
|
30
29
|
exit_on_error: bool = True,
|
|
31
30
|
) -> Optional[str]:
|
|
@@ -190,7 +189,7 @@ def build_project(project: Project, tb_client: TinyB, silent: bool = False) -> O
|
|
|
190
189
|
".pipe": "text/plain",
|
|
191
190
|
".connection": "text/plain",
|
|
192
191
|
}
|
|
193
|
-
TINYBIRD_API_URL = tb_client.host
|
|
192
|
+
TINYBIRD_API_URL = urljoin(tb_client.host, "/v1/build")
|
|
194
193
|
logging.debug(TINYBIRD_API_URL)
|
|
195
194
|
TINYBIRD_API_KEY = tb_client.token
|
|
196
195
|
error: Optional[str] = None
|
tinybird/tb/modules/cli.py
CHANGED
|
@@ -79,6 +79,13 @@ agent_mode_flag = os.environ.get("TB_AGENT_MODE", "false") == "true"
|
|
|
79
79
|
"--output", type=click.Choice(["human", "json", "csv"], case_sensitive=False), default="human", help="Output format"
|
|
80
80
|
)
|
|
81
81
|
@click.option("--max-depth", type=int, default=3, help="Maximum depth of the project files.")
|
|
82
|
+
@click.option(
|
|
83
|
+
"--dangerously-skip-permissions",
|
|
84
|
+
is_flag=True,
|
|
85
|
+
default=False,
|
|
86
|
+
help="Skip permissions check in agent mode.",
|
|
87
|
+
hidden=True,
|
|
88
|
+
)
|
|
82
89
|
@click.version_option(version=VERSION)
|
|
83
90
|
@click.pass_context
|
|
84
91
|
def cli(
|
|
@@ -93,6 +100,7 @@ def cli(
|
|
|
93
100
|
staging: bool,
|
|
94
101
|
output: str,
|
|
95
102
|
max_depth: int,
|
|
103
|
+
dangerously_skip_permissions: bool,
|
|
96
104
|
) -> None:
|
|
97
105
|
"""
|
|
98
106
|
Use `OBFUSCATE_REGEX_PATTERN` and `OBFUSCATE_PATTERN_SEPARATOR` environment variables to define a regex pattern and a separator (in case of a single string with multiple regex) to obfuscate secrets in the CLI output.
|
|
@@ -196,7 +204,7 @@ def cli(
|
|
|
196
204
|
is_agent_mode = agent_mode_flag and ctx.invoked_subcommand is None
|
|
197
205
|
|
|
198
206
|
if is_agent_mode:
|
|
199
|
-
run_agent(config, project)
|
|
207
|
+
run_agent(config, project, dangerously_skip_permissions)
|
|
200
208
|
|
|
201
209
|
|
|
202
210
|
@cli.command(hidden=True)
|
tinybird/tb/modules/create.py
CHANGED
|
@@ -21,7 +21,7 @@ from tinybird.tb.modules.feedback_manager import FeedbackManager
|
|
|
21
21
|
from tinybird.tb.modules.llm import LLM
|
|
22
22
|
from tinybird.tb.modules.llm_utils import extract_xml, parse_xml
|
|
23
23
|
from tinybird.tb.modules.local_common import get_tinybird_local_client
|
|
24
|
-
from tinybird.tb.modules.
|
|
24
|
+
from tinybird.tb.modules.mock_common import create_mock_data
|
|
25
25
|
from tinybird.tb.modules.project import Project
|
|
26
26
|
|
|
27
27
|
|
|
@@ -444,6 +444,7 @@ class FeedbackManager:
|
|
|
444
444
|
error_invalid_output_format = error_message(
|
|
445
445
|
"Invalid output format for this command. Supported formats are: {formats}"
|
|
446
446
|
)
|
|
447
|
+
error_build_only_supported_in_local = error_message("Builds are only supported in Tinybird Local")
|
|
447
448
|
|
|
448
449
|
info_incl_relative_path = info_message("** Relative path {path} does not exist, skipping.")
|
|
449
450
|
info_ignoring_incl_file = info_message(
|
tinybird/tb/modules/llm.py
CHANGED
tinybird/tb/modules/mock.py
CHANGED
|
@@ -1,19 +1,15 @@
|
|
|
1
1
|
import glob
|
|
2
2
|
from pathlib import Path
|
|
3
|
-
from typing import Any, Dict, List
|
|
4
3
|
|
|
5
4
|
import click
|
|
6
5
|
|
|
7
|
-
from tinybird.prompts import mock_prompt
|
|
8
6
|
from tinybird.tb.client import TinyB
|
|
9
7
|
from tinybird.tb.modules.cli import cli
|
|
10
|
-
from tinybird.tb.modules.common import push_data
|
|
11
8
|
from tinybird.tb.modules.config import CLIConfig
|
|
12
|
-
from tinybird.tb.modules.datafile.fixture import persist_fixture
|
|
9
|
+
from tinybird.tb.modules.datafile.fixture import persist_fixture
|
|
13
10
|
from tinybird.tb.modules.exceptions import CLIMockException
|
|
14
11
|
from tinybird.tb.modules.feedback_manager import FeedbackManager
|
|
15
|
-
from tinybird.tb.modules.
|
|
16
|
-
from tinybird.tb.modules.llm_utils import extract_xml
|
|
12
|
+
from tinybird.tb.modules.mock_common import append_mock_data, create_mock_data
|
|
17
13
|
from tinybird.tb.modules.project import Project
|
|
18
14
|
|
|
19
15
|
|
|
@@ -86,71 +82,9 @@ def mock(ctx: click.Context, datasource: str, rows: int, prompt: str, format_: s
|
|
|
86
82
|
fixture_path = persist_fixture(datasource_name, data, folder, format=format_)
|
|
87
83
|
click.echo(FeedbackManager.info(message=f"✓ /fixtures/{datasource_name}.{format_} created"))
|
|
88
84
|
if env == "cloud":
|
|
89
|
-
|
|
85
|
+
append_mock_data(tb_client, datasource_name, str(fixture_path))
|
|
90
86
|
|
|
91
87
|
click.echo(FeedbackManager.success(message=f"✓ Sample data for {datasource_name} created with {rows} rows"))
|
|
92
88
|
|
|
93
89
|
except Exception as e:
|
|
94
90
|
raise CLIMockException(FeedbackManager.error(message=str(e)))
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
def append_fixture(
|
|
98
|
-
tb_client: TinyB,
|
|
99
|
-
datasource_name: str,
|
|
100
|
-
url: str,
|
|
101
|
-
):
|
|
102
|
-
push_data(
|
|
103
|
-
tb_client,
|
|
104
|
-
datasource_name,
|
|
105
|
-
url,
|
|
106
|
-
mode="append",
|
|
107
|
-
concurrency=1,
|
|
108
|
-
silent=True,
|
|
109
|
-
)
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
def create_mock_data(
|
|
113
|
-
datasource_name: str,
|
|
114
|
-
datasource_content: str,
|
|
115
|
-
rows: int,
|
|
116
|
-
prompt: str,
|
|
117
|
-
config: CLIConfig,
|
|
118
|
-
ctx_config: Dict[str, Any],
|
|
119
|
-
user_token: str,
|
|
120
|
-
tb_client: TinyB,
|
|
121
|
-
format_: str,
|
|
122
|
-
folder: str,
|
|
123
|
-
) -> List[Dict[str, Any]]:
|
|
124
|
-
user_client = config.get_client(token=ctx_config.get("token"), host=ctx_config.get("host"))
|
|
125
|
-
llm = LLM(user_token=user_token, host=user_client.host)
|
|
126
|
-
prompt = f"<datasource_schema>{datasource_content}</datasource_schema>\n<user_input>{prompt}</user_input>"
|
|
127
|
-
sql = ""
|
|
128
|
-
attempts = 0
|
|
129
|
-
data = []
|
|
130
|
-
error = ""
|
|
131
|
-
sql_path = None
|
|
132
|
-
while True:
|
|
133
|
-
try:
|
|
134
|
-
response = llm.ask(system_prompt=mock_prompt(rows, error), prompt=prompt, feature="tb_mock")
|
|
135
|
-
sql = extract_xml(response, "sql")
|
|
136
|
-
sql_path = persist_fixture_sql(datasource_name, sql, folder)
|
|
137
|
-
sql_format = "JSON" if format_ == "ndjson" else "CSV"
|
|
138
|
-
result = tb_client.query(f"SELECT * FROM ({sql}) LIMIT {rows} FORMAT {sql_format}")
|
|
139
|
-
if sql_format == "JSON":
|
|
140
|
-
data = result.get("data", [])[:rows]
|
|
141
|
-
error_response = result.get("error", None)
|
|
142
|
-
if error_response:
|
|
143
|
-
raise Exception(error_response)
|
|
144
|
-
else:
|
|
145
|
-
data = result
|
|
146
|
-
break
|
|
147
|
-
except Exception as e:
|
|
148
|
-
error = str(e)
|
|
149
|
-
attempts += 1
|
|
150
|
-
if attempts > 5:
|
|
151
|
-
raise Exception(
|
|
152
|
-
f"Failed to generate a valid solution. Check {str(sql_path or '.sql path')} and try again."
|
|
153
|
-
)
|
|
154
|
-
else:
|
|
155
|
-
continue
|
|
156
|
-
return data
|
|
@@ -0,0 +1,71 @@
|
|
|
1
|
+
from typing import Any, Dict, List
|
|
2
|
+
|
|
3
|
+
from tinybird.prompts import mock_prompt
|
|
4
|
+
from tinybird.tb.client import TinyB
|
|
5
|
+
from tinybird.tb.modules.common import push_data
|
|
6
|
+
from tinybird.tb.modules.config import CLIConfig
|
|
7
|
+
from tinybird.tb.modules.datafile.fixture import persist_fixture_sql
|
|
8
|
+
from tinybird.tb.modules.llm import LLM
|
|
9
|
+
from tinybird.tb.modules.llm_utils import extract_xml
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
def append_mock_data(
|
|
13
|
+
tb_client: TinyB,
|
|
14
|
+
datasource_name: str,
|
|
15
|
+
url: str,
|
|
16
|
+
):
|
|
17
|
+
push_data(
|
|
18
|
+
tb_client,
|
|
19
|
+
datasource_name,
|
|
20
|
+
url,
|
|
21
|
+
mode="append",
|
|
22
|
+
concurrency=1,
|
|
23
|
+
silent=True,
|
|
24
|
+
)
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
def create_mock_data(
|
|
28
|
+
datasource_name: str,
|
|
29
|
+
datasource_content: str,
|
|
30
|
+
rows: int,
|
|
31
|
+
prompt: str,
|
|
32
|
+
config: CLIConfig,
|
|
33
|
+
ctx_config: Dict[str, Any],
|
|
34
|
+
user_token: str,
|
|
35
|
+
tb_client: TinyB,
|
|
36
|
+
format_: str,
|
|
37
|
+
folder: str,
|
|
38
|
+
) -> List[Dict[str, Any]]:
|
|
39
|
+
user_client = config.get_client(token=ctx_config.get("token"), host=ctx_config.get("host"))
|
|
40
|
+
llm = LLM(user_token=user_token, host=user_client.host)
|
|
41
|
+
prompt = f"<datasource_schema>{datasource_content}</datasource_schema>\n<user_input>{prompt}</user_input>"
|
|
42
|
+
sql = ""
|
|
43
|
+
attempts = 0
|
|
44
|
+
data = []
|
|
45
|
+
error = ""
|
|
46
|
+
sql_path = None
|
|
47
|
+
while True:
|
|
48
|
+
try:
|
|
49
|
+
response = llm.ask(system_prompt=mock_prompt(rows, error), prompt=prompt, feature="tb_mock")
|
|
50
|
+
sql = extract_xml(response, "sql")
|
|
51
|
+
sql_path = persist_fixture_sql(datasource_name, sql, folder)
|
|
52
|
+
sql_format = "JSON" if format_ == "ndjson" else "CSV"
|
|
53
|
+
result = tb_client.query(f"SELECT * FROM ({sql}) LIMIT {rows} FORMAT {sql_format}")
|
|
54
|
+
if sql_format == "JSON":
|
|
55
|
+
data = result.get("data", [])[:rows]
|
|
56
|
+
error_response = result.get("error", None)
|
|
57
|
+
if error_response:
|
|
58
|
+
raise Exception(error_response)
|
|
59
|
+
else:
|
|
60
|
+
data = result
|
|
61
|
+
break
|
|
62
|
+
except Exception as e:
|
|
63
|
+
error = str(e)
|
|
64
|
+
attempts += 1
|
|
65
|
+
if attempts > 5:
|
|
66
|
+
raise Exception(
|
|
67
|
+
f"Failed to generate a valid solution. Check {str(sql_path or '.sql path')} and try again."
|
|
68
|
+
)
|
|
69
|
+
else:
|
|
70
|
+
continue
|
|
71
|
+
return data
|
tinybird/tb/modules/project.py
CHANGED
|
@@ -50,6 +50,15 @@ class Project:
|
|
|
50
50
|
project_files.append(project_file)
|
|
51
51
|
return project_files
|
|
52
52
|
|
|
53
|
+
def get_fixture_files(self) -> List[str]:
|
|
54
|
+
fixture_files: List[str] = []
|
|
55
|
+
for extension in ["csv", "ndjson", "parquet"]:
|
|
56
|
+
for fixture_file in self.get_files(extension):
|
|
57
|
+
if self.vendor_path in fixture_file:
|
|
58
|
+
continue
|
|
59
|
+
fixture_files.append(fixture_file)
|
|
60
|
+
return fixture_files
|
|
61
|
+
|
|
53
62
|
def get_resource_path(self, resource_name: str, resource_type: str) -> str:
|
|
54
63
|
full_path = next(
|
|
55
64
|
(p for p in self.get_project_files() if p.endswith("/" + resource_name + f".{resource_type}")), ""
|
|
@@ -10,27 +10,27 @@ tinybird/sql_template_fmt.py,sha256=KUHdj5rYCYm_rKKdXYSJAE9vIyXUQLB0YSZnUXHeBlY,
|
|
|
10
10
|
tinybird/sql_toolset.py,sha256=M2rpLYkgV2W8NnYEYPC1tJdpy4uZHXVF64NBSKLQka4,19549
|
|
11
11
|
tinybird/syncasync.py,sha256=IPnOx6lMbf9SNddN1eBtssg8vCLHMt76SuZ6YNYm-Yk,27761
|
|
12
12
|
tinybird/tornado_template.py,sha256=jjNVDMnkYFWXflmT8KU_Ssbo5vR8KQq3EJMk5vYgXRw,41959
|
|
13
|
-
tinybird/ch_utils/constants.py,sha256=
|
|
13
|
+
tinybird/ch_utils/constants.py,sha256=fPgZtwbr1ymxaW7uqVWHKmAbt7uGj3SxCCS3xsEMJqA,4151
|
|
14
14
|
tinybird/ch_utils/engine.py,sha256=X4tE9OrfaUy6kO9cqVEzyI9cDcmOF3IAssRRzsTsfEQ,40781
|
|
15
15
|
tinybird/datafile/common.py,sha256=J0Oydru3Nh3oCx66EEEHGxm2-r5_NfxvZxiFhBjgAH0,98428
|
|
16
16
|
tinybird/datafile/exceptions.py,sha256=8rw2umdZjtby85QbuRKFO5ETz_eRHwUY5l7eHsy1wnI,556
|
|
17
17
|
tinybird/datafile/parse_connection.py,sha256=tRyn2Rpr1TeWet5BXmMoQgaotbGdYep1qiTak_OqC5E,1825
|
|
18
18
|
tinybird/datafile/parse_datasource.py,sha256=ssW8QeFSgglVFi3sDZj_HgkJiTJ2069v2JgqnH3CkDE,1825
|
|
19
19
|
tinybird/datafile/parse_pipe.py,sha256=xf4m0Tw44QWJzHzAm7Z7FwUoUUtr7noMYjU1NiWnX0k,3880
|
|
20
|
-
tinybird/tb/__cli__.py,sha256=
|
|
20
|
+
tinybird/tb/__cli__.py,sha256=vsGGI9kP2i8TgwhV7ZNt0lpfebdoTNrbxYl00oYXMjE,247
|
|
21
21
|
tinybird/tb/check_pypi.py,sha256=Gp0HkHHDFMSDL6nxKlOY51z7z1Uv-2LRexNTZSHHGmM,552
|
|
22
22
|
tinybird/tb/cli.py,sha256=FdDFEIayjmsZEVsVSSvRiVYn_FHOVg_zWQzchnzfWho,1008
|
|
23
23
|
tinybird/tb/client.py,sha256=pJbdkWMXGAqKseNAvdsRRnl_c7I-DCMB0dWCQnG82nU,54146
|
|
24
24
|
tinybird/tb/config.py,sha256=mhMTGnMB5KcxGoh3dewIr2Jjsa6pHE183gCPAQWyp6o,3973
|
|
25
|
-
tinybird/tb/modules/build.py,sha256=
|
|
26
|
-
tinybird/tb/modules/build_common.py,sha256=
|
|
25
|
+
tinybird/tb/modules/build.py,sha256=efD-vamK1NPaDo9R86Hn8be2DYoW0Hh5bZiH7knK5dk,7790
|
|
26
|
+
tinybird/tb/modules/build_common.py,sha256=rWhemU8bk0ZE2eiwZDaTmV9cPabDGGlyc2WnRxfhT0M,12859
|
|
27
27
|
tinybird/tb/modules/cicd.py,sha256=0KLKccha9IP749QvlXBmzdWv1On3mFwMY4DUcJlBxiE,7326
|
|
28
|
-
tinybird/tb/modules/cli.py,sha256=
|
|
28
|
+
tinybird/tb/modules/cli.py,sha256=DnNCC1HBe42CNCYIa8FR7YfbUNo9Mbd4ATTVShK8f6Q,16573
|
|
29
29
|
tinybird/tb/modules/common.py,sha256=jTTaDDHrZREt--032XhP6GkbfFwC79YJ5aH1Sl7bmbo,81925
|
|
30
30
|
tinybird/tb/modules/config.py,sha256=gK7rgaWTDd4ZKCrNEg_Uemr26EQjqWt6TjyQKujxOws,11462
|
|
31
31
|
tinybird/tb/modules/connection.py,sha256=-MY56NUAai6EMC4-wpi7bT0_nz_SA8QzTmHkV7HB1IQ,17810
|
|
32
32
|
tinybird/tb/modules/copy.py,sha256=dPZkcIDvxjJrlQUIvToO0vsEEEs4EYumbNV77-BzNoU,4404
|
|
33
|
-
tinybird/tb/modules/create.py,sha256=
|
|
33
|
+
tinybird/tb/modules/create.py,sha256=pJxHXG69c9Z_21s-7VuJ3RZOF_nJU51LEwiAkvI3dZY,23251
|
|
34
34
|
tinybird/tb/modules/datasource.py,sha256=cxq0VVjjidxq-v_JSIIAH7L90XNRctgNKsHRoQ_42OI,41632
|
|
35
35
|
tinybird/tb/modules/deployment.py,sha256=Fw9wSNqmLBGCpKwmZsn3KPsy-6kmQzI8YzSdXWoDb6k,12046
|
|
36
36
|
tinybird/tb/modules/deployment_common.py,sha256=Y0r3g-3d6AcihsVVa0OHer3ow3xHSV1VPskF1eI03KI,17644
|
|
@@ -38,21 +38,22 @@ tinybird/tb/modules/deprecations.py,sha256=rrszC1f_JJeJ8mUxGoCxckQTJFBCR8wREf4XX
|
|
|
38
38
|
tinybird/tb/modules/dev_server.py,sha256=57FCKuWpErwYUYgHspYDkLWEm9F4pbvVOtMrFXX1fVU,10129
|
|
39
39
|
tinybird/tb/modules/endpoint.py,sha256=ksRj6mfDb9Xv63PhTkV_uKSosgysHElqagg3RTt21Do,11958
|
|
40
40
|
tinybird/tb/modules/exceptions.py,sha256=5jK91w1LPmtqIUfDpHe_Op5OxGz8-p1BPgtLREMIni0,5217
|
|
41
|
-
tinybird/tb/modules/feedback_manager.py,sha256=
|
|
41
|
+
tinybird/tb/modules/feedback_manager.py,sha256=xkZjwGOV50-oc0y9h58ei4MZ1mwOMOaN_kUcGrtNH48,78132
|
|
42
42
|
tinybird/tb/modules/info.py,sha256=F5vY4kHS_kyO2uSBKac92HoOb447oDeRlzpwtAHTuKc,6872
|
|
43
43
|
tinybird/tb/modules/infra.py,sha256=JE9oLIyF4bi_JBoe-BgZ5HhKp_lQgSihuSV1KIS02Qs,32709
|
|
44
44
|
tinybird/tb/modules/job.py,sha256=wBsnu8UPTOha2rkLvucgmw4xYv73ubmui3eeSIF68ZM,3107
|
|
45
|
-
tinybird/tb/modules/llm.py,sha256=
|
|
45
|
+
tinybird/tb/modules/llm.py,sha256=CpTq2YAk88E8ENpQA94-mas3UDN1aqa--9Al8GdwQtc,1575
|
|
46
46
|
tinybird/tb/modules/llm_utils.py,sha256=nS9r4FAElJw8yXtmdYrx-rtI2zXR8qXfi1QqUDCfxvg,3469
|
|
47
47
|
tinybird/tb/modules/local.py,sha256=tpiw_F_qOIp42h3kTBwTm5GQDyuVLF0QNF1jmB0zR94,6845
|
|
48
48
|
tinybird/tb/modules/local_common.py,sha256=_WODjW3oPshgsZ1jDFFx2nr0zrLi3Gxz5dlahWPobM8,17464
|
|
49
49
|
tinybird/tb/modules/login.py,sha256=glqj5RWH26AseEoBl8XfrSDEjQTdko17i_pVWOIMoGc,12497
|
|
50
50
|
tinybird/tb/modules/logout.py,sha256=sniI4JNxpTrVeRCp0oGJuQ3yRerG4hH5uz6oBmjv724,1009
|
|
51
51
|
tinybird/tb/modules/materialization.py,sha256=0O2JUCxLzz-DrXTUewVHlIyC6-Kyymw0hGXXDicMSHE,5403
|
|
52
|
-
tinybird/tb/modules/mock.py,sha256=
|
|
52
|
+
tinybird/tb/modules/mock.py,sha256=ET8sRpmXnQsd2sSJXH_KCdREU1_XQgkORru6T357Akc,3260
|
|
53
|
+
tinybird/tb/modules/mock_common.py,sha256=72yKp--Zo40hrycUtiajSRW2BojOsgOZFqUorQ_KQ3w,2279
|
|
53
54
|
tinybird/tb/modules/open.py,sha256=LYiuO8Z1I9O_v6pv58qpUCWFD6BT00BdeO21fRa4I4Y,1315
|
|
54
55
|
tinybird/tb/modules/pipe.py,sha256=xPKtezhnWZ6k_g82r4XpgKslofhuIxb_PvynH4gdUzI,2393
|
|
55
|
-
tinybird/tb/modules/project.py,sha256=
|
|
56
|
+
tinybird/tb/modules/project.py,sha256=YyoVnRzqEa8h5uXF4SjkhltChcVaYuUnE_jakKYue3A,6023
|
|
56
57
|
tinybird/tb/modules/regions.py,sha256=QjsL5H6Kg-qr0aYVLrvb1STeJ5Sx_sjvbOYO0LrEGMk,166
|
|
57
58
|
tinybird/tb/modules/secret.py,sha256=9BIdh2PZDAbY2wRbf4ZDvkEltygztz1RMxgDmY1D0LI,3521
|
|
58
59
|
tinybird/tb/modules/secret_common.py,sha256=HyCLAI9WniDLwfK6SAb7ZUWorWjtf8j_GghlaTaos_I,1829
|
|
@@ -66,21 +67,24 @@ tinybird/tb/modules/watch.py,sha256=No0bK1M1_3CYuMaIgylxf7vYFJ72lTJe3brz6xQ-mJo,
|
|
|
66
67
|
tinybird/tb/modules/workspace.py,sha256=Q_8HcxMsNg8QG9aBlwcWS2umrDP5IkTIHqqz3sfmGuc,11341
|
|
67
68
|
tinybird/tb/modules/workspace_members.py,sha256=5JdkJgfuEwbq-t6vxkBhYwgsiTDxF790wsa6Xfif9nk,8608
|
|
68
69
|
tinybird/tb/modules/agent/__init__.py,sha256=i3oe3vDIWWPaicdCM0zs7D7BJ1W0k7th93ooskHAV00,54
|
|
69
|
-
tinybird/tb/modules/agent/agent.py,sha256=
|
|
70
|
+
tinybird/tb/modules/agent/agent.py,sha256=3mP1HwjEGWuyuUByyA7xe2FrW2l-ZiChRKDxMs7vJIg,16887
|
|
70
71
|
tinybird/tb/modules/agent/animations.py,sha256=z0MNLf8TnUO8qAjgYvth_wc9a9283pNVz1Z4jl15Ggs,2558
|
|
71
72
|
tinybird/tb/modules/agent/banner.py,sha256=KX_e467uiy1gWOZ4ofTZt0GCFGQqHQ_8Ob27XLQqda0,3053
|
|
72
73
|
tinybird/tb/modules/agent/memory.py,sha256=H6SJK--2L5C87B7AJd_jMqsq3sCvFvZwZXmajuT0GBE,1171
|
|
73
|
-
tinybird/tb/modules/agent/models.py,sha256=
|
|
74
|
-
tinybird/tb/modules/agent/prompts.py,sha256=
|
|
75
|
-
tinybird/tb/modules/agent/utils.py,sha256=
|
|
74
|
+
tinybird/tb/modules/agent/models.py,sha256=Of74wcU8oX05ricTqmhMHVHfeYo_pQbnbCI_q3mlx5E,682
|
|
75
|
+
tinybird/tb/modules/agent/prompts.py,sha256=fZMTbTbq8SHWob8-wA5fQFnZ9lJa7Y_66_9JvJT3xuc,6818
|
|
76
|
+
tinybird/tb/modules/agent/utils.py,sha256=K0mszvO27wqsmxjWZOjrshUIVb3qxpYr1hM_enlbrxw,13441
|
|
76
77
|
tinybird/tb/modules/agent/tools/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
77
|
-
tinybird/tb/modules/agent/tools/
|
|
78
|
-
tinybird/tb/modules/agent/tools/
|
|
79
|
-
tinybird/tb/modules/agent/tools/
|
|
78
|
+
tinybird/tb/modules/agent/tools/append.py,sha256=2q5y32jeNHJgbMsVmABn7y-KCoKwkqbMFJAm8OZ4zQc,2161
|
|
79
|
+
tinybird/tb/modules/agent/tools/build.py,sha256=LhzJMx6tbxC7gogIrxhfKJc-SDgoSR-FC6IunfaCdn8,758
|
|
80
|
+
tinybird/tb/modules/agent/tools/create_datafile.py,sha256=IeLB81HNQ5k8fy1x3ZDkKbi0iPqSMZz-xNi2Kga0I1M,5431
|
|
81
|
+
tinybird/tb/modules/agent/tools/deploy.py,sha256=Vv1SHalxZsl5QttaON0jBwJenj1cVOQiQ-cMiK2ULZg,1443
|
|
80
82
|
tinybird/tb/modules/agent/tools/deploy_check.py,sha256=VqMYC7l3_cihmmM_pi8w1t8rJ3P0xDc7pHs_st9k-9Q,684
|
|
81
83
|
tinybird/tb/modules/agent/tools/explore.py,sha256=ihALc_kBcsjrKT3hZyicqyIowB0g_K3AtNNi-5uz9-8,412
|
|
82
|
-
tinybird/tb/modules/agent/tools/
|
|
84
|
+
tinybird/tb/modules/agent/tools/mock.py,sha256=c4fY8_D92tOUBr0DoqoA5lEE3FgvUQHP6JE75mfTBko,2491
|
|
85
|
+
tinybird/tb/modules/agent/tools/plan.py,sha256=wQY4gNtFTOEy2yZUGf8VqefPUbbz5DgMZdrzGRk-wiE,1365
|
|
83
86
|
tinybird/tb/modules/agent/tools/preview_datafile.py,sha256=e9q5fR0afApcrntzFrnuHmd10ex7MG_GM6T0Pwc9bRI,850
|
|
87
|
+
tinybird/tb/modules/agent/tools/read_fixture_data.py,sha256=QCmdXccVVRqkDYmUdPst94qr29dmlE20KSMyBHpzDjs,961
|
|
84
88
|
tinybird/tb/modules/datafile/build.py,sha256=NFKBrusFLU0WJNCXePAFWiEDuTaXpwc0lHlOQWEJ43s,51117
|
|
85
89
|
tinybird/tb/modules/datafile/build_common.py,sha256=2yNdxe49IMA9wNvl25NemY2Iaz8L66snjOdT64dm1is,4511
|
|
86
90
|
tinybird/tb/modules/datafile/build_datasource.py,sha256=Ra8pVQBDafbFRUKlhpgohhTsRyp_ADKZJVG8Gd69idY,17227
|
|
@@ -101,8 +105,8 @@ tinybird/tb_cli_modules/config.py,sha256=IsgdtFRnUrkY8-Zo32lmk6O7u3bHie1QCxLwgp4
|
|
|
101
105
|
tinybird/tb_cli_modules/exceptions.py,sha256=pmucP4kTF4irIt7dXiG-FcnI-o3mvDusPmch1L8RCWk,3367
|
|
102
106
|
tinybird/tb_cli_modules/regions.py,sha256=QjsL5H6Kg-qr0aYVLrvb1STeJ5Sx_sjvbOYO0LrEGMk,166
|
|
103
107
|
tinybird/tb_cli_modules/telemetry.py,sha256=Hh2Io8ZPROSunbOLuMvuIFU4TqwWPmQTqal4WS09K1A,10449
|
|
104
|
-
tinybird-0.0.1.
|
|
105
|
-
tinybird-0.0.1.
|
|
106
|
-
tinybird-0.0.1.
|
|
107
|
-
tinybird-0.0.1.
|
|
108
|
-
tinybird-0.0.1.
|
|
108
|
+
tinybird-0.0.1.dev247.dist-info/METADATA,sha256=IBLQijubQLYO67wrCCvUpdLPbEqv1JBnf7RCCEOiKG8,1733
|
|
109
|
+
tinybird-0.0.1.dev247.dist-info/WHEEL,sha256=In9FTNxeP60KnTkGw7wk6mJPYd_dQSjEZmXdBdMCI-8,91
|
|
110
|
+
tinybird-0.0.1.dev247.dist-info/entry_points.txt,sha256=LwdHU6TfKx4Qs7BqqtaczEZbImgU7Abe9Lp920zb_fo,43
|
|
111
|
+
tinybird-0.0.1.dev247.dist-info/top_level.txt,sha256=VqqqEmkAy7UNaD8-V51FCoMMWXjLUlR0IstvK7tJYVY,54
|
|
112
|
+
tinybird-0.0.1.dev247.dist-info/RECORD,,
|
|
File without changes
|
|
File without changes
|
|
File without changes
|