tinybird 0.0.1.dev253__py3-none-any.whl → 0.0.1.dev254__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of tinybird might be problematic. Click here for more details.
- tinybird/tb/__cli__.py +2 -2
- tinybird/tb/modules/agent/agent.py +20 -25
- tinybird/tb/modules/agent/prompts.py +237 -0
- tinybird/tb/modules/agent/tools/analyze.py +79 -0
- tinybird/tb/modules/agent/tools/append.py +42 -2
- {tinybird-0.0.1.dev253.dist-info → tinybird-0.0.1.dev254.dist-info}/METADATA +1 -1
- {tinybird-0.0.1.dev253.dist-info → tinybird-0.0.1.dev254.dist-info}/RECORD +10 -10
- tinybird/tb/modules/agent/tools/read_fixture_data.py +0 -36
- {tinybird-0.0.1.dev253.dist-info → tinybird-0.0.1.dev254.dist-info}/WHEEL +0 -0
- {tinybird-0.0.1.dev253.dist-info → tinybird-0.0.1.dev254.dist-info}/entry_points.txt +0 -0
- {tinybird-0.0.1.dev253.dist-info → tinybird-0.0.1.dev254.dist-info}/top_level.txt +0 -0
tinybird/tb/__cli__.py
CHANGED
|
@@ -4,5 +4,5 @@ __description__ = 'Tinybird Command Line Tool'
|
|
|
4
4
|
__url__ = 'https://www.tinybird.co/docs/forward/commands'
|
|
5
5
|
__author__ = 'Tinybird'
|
|
6
6
|
__author_email__ = 'support@tinybird.co'
|
|
7
|
-
__version__ = '0.0.1.
|
|
8
|
-
__revision__ = '
|
|
7
|
+
__version__ = '0.0.1.dev254'
|
|
8
|
+
__revision__ = '269e345'
|
|
@@ -30,11 +30,13 @@ from tinybird.tb.modules.agent.memory import clear_history
|
|
|
30
30
|
from tinybird.tb.modules.agent.models import create_model, model_costs
|
|
31
31
|
from tinybird.tb.modules.agent.prompts import (
|
|
32
32
|
datafile_instructions,
|
|
33
|
+
endpoint_optimization_instructions,
|
|
33
34
|
plan_instructions,
|
|
34
35
|
resources_prompt,
|
|
35
36
|
sql_instructions,
|
|
36
37
|
)
|
|
37
|
-
from tinybird.tb.modules.agent.tools.
|
|
38
|
+
from tinybird.tb.modules.agent.tools.analyze import analyze_file, analyze_url
|
|
39
|
+
from tinybird.tb.modules.agent.tools.append import append_file, append_url
|
|
38
40
|
from tinybird.tb.modules.agent.tools.build import build
|
|
39
41
|
from tinybird.tb.modules.agent.tools.create_datafile import create_datafile
|
|
40
42
|
from tinybird.tb.modules.agent.tools.deploy import deploy
|
|
@@ -45,7 +47,6 @@ from tinybird.tb.modules.agent.tools.get_openapi_definition import get_openapi_d
|
|
|
45
47
|
from tinybird.tb.modules.agent.tools.mock import mock
|
|
46
48
|
from tinybird.tb.modules.agent.tools.plan import plan
|
|
47
49
|
from tinybird.tb.modules.agent.tools.preview_datafile import preview_datafile
|
|
48
|
-
from tinybird.tb.modules.agent.tools.read_fixture_data import read_fixture_data
|
|
49
50
|
from tinybird.tb.modules.agent.tools.request_endpoint import request_endpoint
|
|
50
51
|
from tinybird.tb.modules.agent.utils import TinybirdAgentContext, show_input
|
|
51
52
|
from tinybird.tb.modules.build_common import process as build_process
|
|
@@ -112,11 +113,13 @@ You have access to the following tools:
|
|
|
112
113
|
5. `deploy` - Deploy the project to Tinybird Cloud.
|
|
113
114
|
6. `deploy_check` - Check if the project can be deployed to Tinybird Cloud before deploying it.
|
|
114
115
|
7. `mock` - Create mock data for a landing datasource.
|
|
115
|
-
8. `
|
|
116
|
-
9. `
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
116
|
+
8. `analyze_file` - Analyze the content of a fixture file present in the project folder.
|
|
117
|
+
9. `analyze_url` - Analyze the content of an external url.
|
|
118
|
+
9. `append_file` - Append a file present in the project to a datasource.
|
|
119
|
+
10. `append_url` - Append an external url to a datasource.
|
|
120
|
+
11. `get_endpoint_stats` - Get metrics of the requests to an endpoint.
|
|
121
|
+
12. `get_openapi_definition` - Get the OpenAPI definition for all endpoints that are built/deployed to Tinybird Cloud or Local.
|
|
122
|
+
13. `execute_query` - Execute a query against Tinybird Cloud or Local.
|
|
120
123
|
13. `request_endpoint` - Request an endpoint against Tinybird Cloud or Local.
|
|
121
124
|
|
|
122
125
|
# When creating or updating datafiles:
|
|
@@ -133,21 +136,11 @@ You have access to the following tools:
|
|
|
133
136
|
- If the user does not specify anything about the desired schema, create a schema like this:
|
|
134
137
|
SCHEMA >
|
|
135
138
|
`data` String `json:$`
|
|
139
|
+
|
|
136
140
|
- Use always json paths with .ndjson files.
|
|
137
141
|
|
|
138
142
|
# When user wants to optimize an endpoint:
|
|
139
|
-
|
|
140
|
-
Avoid when possible to update the landing datasource.
|
|
141
|
-
Check endpoint stats to analyze how the endpoint is performing. Use `get_endpoint_stats` tool to get the stats.
|
|
142
|
-
You can use `request_endpoint` tool to check if the endpoint is working as expected.
|
|
143
|
-
You can use `execute_query` tool to execute a query against Tinybird Cloud or Local, to test different queries.
|
|
144
|
-
When your data is in Tinybird, you can create intermediate data sources to preprocess data and make the endpoints faster. This can be done by using materialized views or copy pipes.
|
|
145
|
-
- Copy pipes capture the result of a pipe at a specific point in time and write it to a target data source. They can run on a schedule or run on demand, making them ideal for event-sourced snapshots, data experimentation, and deduplication with snapshots.
|
|
146
|
-
- Materialized views continuously re-evaluate a query as new events are inserted, maintaining an always up-to-date derived dataset. Unlike copy pipes which create point-in-time snapshots, materialized views provide real-time transformations of your data.
|
|
147
|
-
Each approach has its own strengths and use cases:
|
|
148
|
-
- Use copy pipes when you need scheduled or on-demand snapshots of your data.
|
|
149
|
-
- Use materialized views when you need continuous, real-time transformations.
|
|
150
|
-
Finally, update the existing endpoint itself, do not add a new one.
|
|
143
|
+
{endpoint_optimization_instructions}
|
|
151
144
|
|
|
152
145
|
IMPORTANT: If the user cancels some of the steps or there is an error in file creation, DO NOT continue with the plan. Stop the process and wait for the user before using any other tool.
|
|
153
146
|
IMPORTANT: Every time you finish a plan and start a new resource creation or update process, create a new plan before starting with the changes.
|
|
@@ -200,8 +193,10 @@ Today is {datetime.now().strftime("%Y-%m-%d")}
|
|
|
200
193
|
Tool(deploy, docstring_format="google", require_parameter_descriptions=True, takes_ctx=True),
|
|
201
194
|
Tool(deploy_check, docstring_format="google", require_parameter_descriptions=True, takes_ctx=True),
|
|
202
195
|
Tool(mock, docstring_format="google", require_parameter_descriptions=True, takes_ctx=True),
|
|
203
|
-
Tool(
|
|
204
|
-
Tool(
|
|
196
|
+
Tool(analyze_file, docstring_format="google", require_parameter_descriptions=True, takes_ctx=True),
|
|
197
|
+
Tool(analyze_url, docstring_format="google", require_parameter_descriptions=True, takes_ctx=True),
|
|
198
|
+
Tool(append_file, docstring_format="google", require_parameter_descriptions=True, takes_ctx=True),
|
|
199
|
+
Tool(append_url, docstring_format="google", require_parameter_descriptions=True, takes_ctx=True),
|
|
205
200
|
Tool(
|
|
206
201
|
get_endpoint_stats, docstring_format="google", require_parameter_descriptions=True, takes_ctx=True
|
|
207
202
|
),
|
|
@@ -281,8 +276,8 @@ def run_agent(
|
|
|
281
276
|
token = config.get("token", None)
|
|
282
277
|
host = config.get("host", None)
|
|
283
278
|
user_token = config.get("user_token", None)
|
|
284
|
-
workspace_id = config.get("id",
|
|
285
|
-
workspace_name = config.get("name",
|
|
279
|
+
workspace_id = config.get("id", "")
|
|
280
|
+
workspace_name = config.get("name", "")
|
|
286
281
|
try:
|
|
287
282
|
if not token or not host or not workspace_id or not user_token:
|
|
288
283
|
yes = click.confirm(
|
|
@@ -443,9 +438,9 @@ def mock_data(
|
|
|
443
438
|
)
|
|
444
439
|
|
|
445
440
|
|
|
446
|
-
def analyze_fixture(config: dict[str, Any], fixture_path: str) -> dict[str, Any]:
|
|
441
|
+
def analyze_fixture(config: dict[str, Any], fixture_path: str, format: str = "json") -> dict[str, Any]:
|
|
447
442
|
local_client = get_tinybird_local_client(config, test=False, silent=True)
|
|
448
|
-
meta, _data = _analyze(fixture_path, local_client,
|
|
443
|
+
meta, _data = _analyze(fixture_path, local_client, format)
|
|
449
444
|
return meta
|
|
450
445
|
|
|
451
446
|
|
|
@@ -147,3 +147,240 @@ def get_resource_type(path: Path) -> str:
|
|
|
147
147
|
elif path.suffix.lower() == ".connection":
|
|
148
148
|
return "connection"
|
|
149
149
|
return "unknown"
|
|
150
|
+
|
|
151
|
+
|
|
152
|
+
endpoint_optimization_instructions = """
|
|
153
|
+
<endpoint_optimization_instructions>
|
|
154
|
+
## Endpoint Optimization Instructions
|
|
155
|
+
### Step 1: Identify Performance Issues
|
|
156
|
+
1. Analyze the endpoint's query performance metrics
|
|
157
|
+
2. Look for endpoints with high latency or excessive data scanning
|
|
158
|
+
3. Check read_bytes/write_bytes ratios to detect inefficient operations
|
|
159
|
+
|
|
160
|
+
### Step 2: Apply the 5-Question Diagnostic Framework
|
|
161
|
+
|
|
162
|
+
#### Question 1: Are you aggregating or transforming data at query time?
|
|
163
|
+
**Detection:**
|
|
164
|
+
- Look for `count()`, `sum()`, `avg()`, or data type casting in published API endpoints
|
|
165
|
+
- Check if the same calculations are performed on every request
|
|
166
|
+
|
|
167
|
+
**Fix:**
|
|
168
|
+
- Create Materialized Views to pre-aggregate data at ingestion time
|
|
169
|
+
- Move transformations from query time to ingestion time
|
|
170
|
+
- Example transformation:
|
|
171
|
+
```sql
|
|
172
|
+
-- Before (in endpoint)
|
|
173
|
+
SELECT date, count(*) as daily_count
|
|
174
|
+
FROM events
|
|
175
|
+
GROUP BY date
|
|
176
|
+
|
|
177
|
+
-- After (in Materialized View)
|
|
178
|
+
ENGINE "AggregatingMergeTree"
|
|
179
|
+
ENGINE_PARTITION_KEY "toYYYYMM(date)"
|
|
180
|
+
ENGINE_SORTING_KEY "date"
|
|
181
|
+
AS SELECT
|
|
182
|
+
date,
|
|
183
|
+
count(*) as daily_count
|
|
184
|
+
FROM events
|
|
185
|
+
GROUP BY date
|
|
186
|
+
```
|
|
187
|
+
|
|
188
|
+
#### Question 2: Are you filtering by fields in the sorting key?
|
|
189
|
+
**Detection:**
|
|
190
|
+
- Examine WHERE clauses in queries
|
|
191
|
+
- Check if filtered columns are part of the sorting key
|
|
192
|
+
- Look for filters on partition keys instead of sorting keys
|
|
193
|
+
|
|
194
|
+
**Fix:**
|
|
195
|
+
- Ensure sorting key includes frequently filtered columns
|
|
196
|
+
- Order sorting key columns by selectivity (most selective first)
|
|
197
|
+
- Guidelines:
|
|
198
|
+
- Use 3-5 columns in sorting key
|
|
199
|
+
- Place `customer_id` or tenant identifiers first for multi-tenant apps
|
|
200
|
+
- Avoid `timestamp` as the first sorting key element
|
|
201
|
+
- Never use partition key for filtering
|
|
202
|
+
|
|
203
|
+
**Example Fix:**
|
|
204
|
+
```sql
|
|
205
|
+
-- Before
|
|
206
|
+
ENGINE_SORTING_KEY "timestamp, customer_id"
|
|
207
|
+
|
|
208
|
+
-- After (better for multi-tenant filtering)
|
|
209
|
+
ENGINE_SORTING_KEY "customer_id, timestamp"
|
|
210
|
+
```
|
|
211
|
+
|
|
212
|
+
#### Question 3: Are you using the best data types?
|
|
213
|
+
**Detection:**
|
|
214
|
+
- Scan for overly large data types:
|
|
215
|
+
- String where UUID would work
|
|
216
|
+
- Int64 where UInt32 would suffice
|
|
217
|
+
- DateTime with unnecessary precision
|
|
218
|
+
- Nullable columns that could have defaults
|
|
219
|
+
|
|
220
|
+
**Fix:**
|
|
221
|
+
- Downsize data types:
|
|
222
|
+
```sql
|
|
223
|
+
-- Before
|
|
224
|
+
id String,
|
|
225
|
+
count Int64,
|
|
226
|
+
created_at DateTime64(3),
|
|
227
|
+
status Nullable(String)
|
|
228
|
+
|
|
229
|
+
-- After
|
|
230
|
+
id UUID,
|
|
231
|
+
count UInt32,
|
|
232
|
+
created_at DateTime,
|
|
233
|
+
status LowCardinality(String) DEFAULT 'pending'
|
|
234
|
+
```
|
|
235
|
+
- Use `LowCardinality()` for strings with <100k unique values
|
|
236
|
+
- Replace Nullable with default values using `coalesce()`
|
|
237
|
+
|
|
238
|
+
#### Question 4: Are you doing complex operations early in the pipeline?
|
|
239
|
+
**Detection:**
|
|
240
|
+
- Look for JOINs or aggregations before filters
|
|
241
|
+
- Check operation order in multi-node pipes
|
|
242
|
+
|
|
243
|
+
**Fix:**
|
|
244
|
+
- Reorder operations: Filter → Simple transforms → Complex operations
|
|
245
|
+
- Example:
|
|
246
|
+
```sql
|
|
247
|
+
-- Before
|
|
248
|
+
SELECT * FROM (
|
|
249
|
+
SELECT a.*, b.name
|
|
250
|
+
FROM events a
|
|
251
|
+
JOIN users b ON a.user_id = b.id
|
|
252
|
+
) WHERE date >= today() - 7
|
|
253
|
+
|
|
254
|
+
-- After
|
|
255
|
+
SELECT a.*, b.name
|
|
256
|
+
FROM (
|
|
257
|
+
SELECT * FROM events
|
|
258
|
+
WHERE date >= today() - 7
|
|
259
|
+
) a
|
|
260
|
+
JOIN users b ON a.user_id = b.id
|
|
261
|
+
```
|
|
262
|
+
|
|
263
|
+
#### Question 5: Are you joining two or more data sources?
|
|
264
|
+
**Detection:**
|
|
265
|
+
- Identify JOINs in queries
|
|
266
|
+
- Check read_bytes/write_bytes ratio in Materialized Views
|
|
267
|
+
- Look for full table scans on joined tables
|
|
268
|
+
|
|
269
|
+
**Fix Options:**
|
|
270
|
+
1. Replace JOIN with subquery:
|
|
271
|
+
```sql
|
|
272
|
+
-- Before
|
|
273
|
+
SELECT e.*, u.name
|
|
274
|
+
FROM events e
|
|
275
|
+
JOIN users u ON e.user_id = u.id
|
|
276
|
+
|
|
277
|
+
-- After
|
|
278
|
+
SELECT e.*,
|
|
279
|
+
(SELECT name FROM users WHERE id = e.user_id) as name
|
|
280
|
+
FROM events e
|
|
281
|
+
WHERE user_id IN (SELECT id FROM users)
|
|
282
|
+
```
|
|
283
|
+
|
|
284
|
+
2. Optimize Materialized View JOINs:
|
|
285
|
+
```sql
|
|
286
|
+
-- Before (inefficient)
|
|
287
|
+
SELECT a.id, a.value, b.value
|
|
288
|
+
FROM a
|
|
289
|
+
LEFT JOIN b USING id
|
|
290
|
+
|
|
291
|
+
-- After (optimized)
|
|
292
|
+
SELECT a.id, a.value, b.value
|
|
293
|
+
FROM a
|
|
294
|
+
LEFT JOIN (
|
|
295
|
+
SELECT id, value
|
|
296
|
+
FROM b
|
|
297
|
+
WHERE b.id IN (SELECT id FROM a)
|
|
298
|
+
) b USING id
|
|
299
|
+
```
|
|
300
|
+
|
|
301
|
+
### Step 3: Implementation Actions
|
|
302
|
+
|
|
303
|
+
#### For Schema Changes:
|
|
304
|
+
1. Update the datasource schema
|
|
305
|
+
2. Update the sorting keys and data types
|
|
306
|
+
3. Update dependent pipes and endpoints
|
|
307
|
+
|
|
308
|
+
#### For Query Optimizations:
|
|
309
|
+
1. Create Materialized Views for repeated aggregations
|
|
310
|
+
2. Rewrite queries following best practices
|
|
311
|
+
3. Test performance improvements
|
|
312
|
+
|
|
313
|
+
#### For JOIN Optimizations:
|
|
314
|
+
1. Evaluate if JOIN is necessary
|
|
315
|
+
2. Consider denormalization strategies
|
|
316
|
+
3. Use Copy Pipes for historical data recalculation
|
|
317
|
+
4. Implement filtered JOINs in Materialized Views
|
|
318
|
+
|
|
319
|
+
#### In general:
|
|
320
|
+
1. If you need to iterate an existing resource, do not create a new iteration, just update it with the needed changes.
|
|
321
|
+
|
|
322
|
+
## Monitoring and Validation
|
|
323
|
+
|
|
324
|
+
### Monitoring:
|
|
325
|
+
1. Set up alerts for endpoints exceeding latency thresholds
|
|
326
|
+
2. Review of tinybird.pipe_stats_rt (realtime stats of last 24h) and tinybird.pipe_stats (historical stats aggregated by day)
|
|
327
|
+
3. Track processed data patterns over time
|
|
328
|
+
4. Monitor for query pattern changes
|
|
329
|
+
|
|
330
|
+
### Success Metrics:
|
|
331
|
+
- Reduced query latency
|
|
332
|
+
- Lower data scanning (read_bytes)
|
|
333
|
+
- Improved read_bytes/write_bytes ratio
|
|
334
|
+
- Consistent sub-second API response times
|
|
335
|
+
|
|
336
|
+
## Code Templates
|
|
337
|
+
|
|
338
|
+
### Materialized View Template:
|
|
339
|
+
```sql
|
|
340
|
+
NODE materialized_view_name
|
|
341
|
+
SQL >
|
|
342
|
+
SELECT
|
|
343
|
+
-- Pre-aggregated fields
|
|
344
|
+
toDate(timestamp) as date,
|
|
345
|
+
customer_id,
|
|
346
|
+
count(*) as event_count,
|
|
347
|
+
sum(amount) as total_amount
|
|
348
|
+
FROM source_table
|
|
349
|
+
GROUP BY date, customer_id
|
|
350
|
+
|
|
351
|
+
TYPE materialized
|
|
352
|
+
DATASOURCE mv_datasource_name
|
|
353
|
+
ENGINE "AggregatingMergeTree"
|
|
354
|
+
ENGINE_PARTITION_KEY "toYYYYMM(date)"
|
|
355
|
+
ENGINE_SORTING_KEY "customer_id, date"
|
|
356
|
+
```
|
|
357
|
+
|
|
358
|
+
### Optimized Query Template:
|
|
359
|
+
```sql
|
|
360
|
+
NODE endpoint_query
|
|
361
|
+
SQL >
|
|
362
|
+
-- Step 1: Filter early
|
|
363
|
+
WITH filtered_data AS (
|
|
364
|
+
SELECT * FROM events
|
|
365
|
+
WHERE customer_id = {{ String(customer_id) }}
|
|
366
|
+
AND date >= {{ Date(start_date) }}
|
|
367
|
+
AND date <= {{ Date(end_date) }}
|
|
368
|
+
)
|
|
369
|
+
-- Step 2: Simple operations
|
|
370
|
+
SELECT
|
|
371
|
+
date,
|
|
372
|
+
sum(amount) as daily_total
|
|
373
|
+
FROM filtered_data
|
|
374
|
+
GROUP BY date
|
|
375
|
+
ORDER BY date DESC
|
|
376
|
+
```
|
|
377
|
+
|
|
378
|
+
## Best Practices Summary
|
|
379
|
+
|
|
380
|
+
1. **Think ingestion-time, not query-time** - Move computations upstream
|
|
381
|
+
2. **Index smartly** - Sorting keys should match filter patterns
|
|
382
|
+
3. **Size appropriately** - Use the smallest viable data types
|
|
383
|
+
4. **Filter first** - Reduce data before complex operations
|
|
384
|
+
5. **JOIN carefully** - Consider alternatives and optimize when necessary
|
|
385
|
+
</endpoint_optimization_instructions>
|
|
386
|
+
"""
|
|
@@ -0,0 +1,79 @@
|
|
|
1
|
+
import json
|
|
2
|
+
from pathlib import Path
|
|
3
|
+
from urllib.parse import urlparse
|
|
4
|
+
|
|
5
|
+
import click
|
|
6
|
+
from pydantic_ai import RunContext
|
|
7
|
+
|
|
8
|
+
from tinybird.tb.modules.agent.utils import TinybirdAgentContext
|
|
9
|
+
from tinybird.tb.modules.feedback_manager import FeedbackManager
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
def analyze_file(ctx: RunContext[TinybirdAgentContext], fixture_pathname: str):
|
|
13
|
+
"""Analyze a fixture data file present in the project folder
|
|
14
|
+
|
|
15
|
+
Args:
|
|
16
|
+
fixture_pathname (str): a path or an external url to a fixture file. Required.
|
|
17
|
+
|
|
18
|
+
Returns:
|
|
19
|
+
str: The content of the fixture data file.
|
|
20
|
+
"""
|
|
21
|
+
try:
|
|
22
|
+
ctx.deps.thinking_animation.stop()
|
|
23
|
+
click.echo(FeedbackManager.highlight(message=f"» Analyzing {fixture_pathname}..."))
|
|
24
|
+
fixture_path = Path(ctx.deps.folder) / fixture_pathname.lstrip("/")
|
|
25
|
+
|
|
26
|
+
if not fixture_path.exists():
|
|
27
|
+
click.echo(FeedbackManager.error(message=f"No fixture data found for {fixture_pathname}."))
|
|
28
|
+
ctx.deps.thinking_animation.start()
|
|
29
|
+
return f"No fixture data found for {fixture_pathname}. Please check the path of the fixture and try again."
|
|
30
|
+
|
|
31
|
+
fixture_extension = fixture_path.suffix.lstrip(".")
|
|
32
|
+
response = ctx.deps.analyze_fixture(fixture_path=str(fixture_path), format=fixture_extension)
|
|
33
|
+
click.echo(FeedbackManager.success(message="✓ Done!\n"))
|
|
34
|
+
ctx.deps.thinking_animation.start()
|
|
35
|
+
# limit content to first 10 rows
|
|
36
|
+
data = response["preview"]["data"][:10]
|
|
37
|
+
columns = response["analysis"]["columns"]
|
|
38
|
+
|
|
39
|
+
return f"#Result of analysis of {fixture_pathname}:\n##Columns:\n{json.dumps(columns)}\n##Data sample:\n{json.dumps(data)}"
|
|
40
|
+
except Exception as e:
|
|
41
|
+
ctx.deps.thinking_animation.stop()
|
|
42
|
+
click.echo(FeedbackManager.error(message=f"Error analyzing {fixture_pathname}: {e}"))
|
|
43
|
+
ctx.deps.thinking_animation.start()
|
|
44
|
+
return f"Error analyzing {fixture_pathname}: {e}"
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
def analyze_url(ctx: RunContext[TinybirdAgentContext], fixture_url: str):
|
|
48
|
+
"""Analyze a fixture file present in an external url
|
|
49
|
+
|
|
50
|
+
Args:
|
|
51
|
+
fixture_url (str): an external url to a fixture file. Required.
|
|
52
|
+
|
|
53
|
+
Returns:
|
|
54
|
+
str: The analysis with the columns and the first 10 rows of the fixture data file.
|
|
55
|
+
"""
|
|
56
|
+
try:
|
|
57
|
+
ctx.deps.thinking_animation.stop()
|
|
58
|
+
is_url = urlparse(fixture_url).scheme in ("http", "https")
|
|
59
|
+
click.echo(FeedbackManager.highlight(message=f"» Analyzing {fixture_url}..."))
|
|
60
|
+
if not is_url:
|
|
61
|
+
click.echo(FeedbackManager.error(message=f"{fixture_url} is not a valid url."))
|
|
62
|
+
ctx.deps.thinking_animation.start()
|
|
63
|
+
return f"{fixture_url} is not a valid url. Please check the url and try again."
|
|
64
|
+
|
|
65
|
+
fixture_extension = fixture_url.split(".")[-1]
|
|
66
|
+
|
|
67
|
+
response = ctx.deps.analyze_fixture(fixture_path=fixture_url, format=fixture_extension)
|
|
68
|
+
click.echo(FeedbackManager.success(message="✓ Done!\n"))
|
|
69
|
+
ctx.deps.thinking_animation.start()
|
|
70
|
+
# limit content to first 10 rows
|
|
71
|
+
data = response["preview"]["data"][:10]
|
|
72
|
+
columns = response["analysis"]["columns"]
|
|
73
|
+
|
|
74
|
+
return f"#Result of analysis of URL {fixture_url}:\n##Columns:\n{json.dumps(columns)}\n##Data sample:\n{json.dumps(data)}"
|
|
75
|
+
except Exception as e:
|
|
76
|
+
ctx.deps.thinking_animation.stop()
|
|
77
|
+
click.echo(FeedbackManager.error(message=f"Error analyzing {fixture_url}: {e}"))
|
|
78
|
+
ctx.deps.thinking_animation.start()
|
|
79
|
+
return f"Error analyzing {fixture_url}: {e}"
|
|
@@ -5,7 +5,7 @@ from tinybird.tb.modules.agent.utils import TinybirdAgentContext, show_confirmat
|
|
|
5
5
|
from tinybird.tb.modules.feedback_manager import FeedbackManager
|
|
6
6
|
|
|
7
7
|
|
|
8
|
-
def
|
|
8
|
+
def append_file(ctx: RunContext[TinybirdAgentContext], datasource_name: str, fixture_pathname: str) -> str:
|
|
9
9
|
"""Append existing fixture to a datasource
|
|
10
10
|
|
|
11
11
|
Args:
|
|
@@ -18,7 +18,7 @@ def append(ctx: RunContext[TinybirdAgentContext], datasource_name: str, fixture_
|
|
|
18
18
|
try:
|
|
19
19
|
ctx.deps.thinking_animation.stop()
|
|
20
20
|
confirmation = show_confirmation(
|
|
21
|
-
title=f"Append
|
|
21
|
+
title=f"Append fixture {fixture_pathname} to datasource {datasource_name}?",
|
|
22
22
|
skip_confirmation=ctx.deps.dangerously_skip_permissions,
|
|
23
23
|
)
|
|
24
24
|
|
|
@@ -45,3 +45,43 @@ def append(ctx: RunContext[TinybirdAgentContext], datasource_name: str, fixture_
|
|
|
45
45
|
click.echo(FeedbackManager.error(message=e))
|
|
46
46
|
ctx.deps.thinking_animation.start()
|
|
47
47
|
return f"Error appending fixture {fixture_pathname} to {datasource_name}: {e}"
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
def append_url(ctx: RunContext[TinybirdAgentContext], datasource_name: str, fixture_url: str) -> str:
|
|
51
|
+
"""Append existing fixture to a datasource
|
|
52
|
+
|
|
53
|
+
Args:
|
|
54
|
+
datasource_name: Name of the datasource to append fixture to
|
|
55
|
+
fixture_url: external url to the fixture file to append
|
|
56
|
+
|
|
57
|
+
Returns:
|
|
58
|
+
str: Message indicating the success or failure of the appending
|
|
59
|
+
"""
|
|
60
|
+
try:
|
|
61
|
+
ctx.deps.thinking_animation.stop()
|
|
62
|
+
confirmation = show_confirmation(
|
|
63
|
+
title=f"Append URL {fixture_url} to datasource {datasource_name}?",
|
|
64
|
+
skip_confirmation=ctx.deps.dangerously_skip_permissions,
|
|
65
|
+
)
|
|
66
|
+
|
|
67
|
+
if confirmation == "review":
|
|
68
|
+
click.echo()
|
|
69
|
+
feedback = show_input(ctx.deps.workspace_name)
|
|
70
|
+
ctx.deps.thinking_animation.start()
|
|
71
|
+
return f"User did not confirm appending URL {fixture_url} and gave the following feedback: {feedback}"
|
|
72
|
+
|
|
73
|
+
if confirmation == "cancel":
|
|
74
|
+
ctx.deps.thinking_animation.start()
|
|
75
|
+
return f"User rejected appending URL {fixture_url}. Skip this step"
|
|
76
|
+
|
|
77
|
+
ctx.deps.thinking_animation.stop()
|
|
78
|
+
click.echo(FeedbackManager.highlight(message=f"\n» Appending {fixture_url} to {datasource_name}..."))
|
|
79
|
+
ctx.deps.append_data(datasource_name=datasource_name, path=fixture_url)
|
|
80
|
+
click.echo(FeedbackManager.success(message=f"✓ Data appended to {datasource_name}"))
|
|
81
|
+
ctx.deps.thinking_animation.start()
|
|
82
|
+
return f"Data appended to {datasource_name}"
|
|
83
|
+
except Exception as e:
|
|
84
|
+
ctx.deps.thinking_animation.stop()
|
|
85
|
+
click.echo(FeedbackManager.error(message=e))
|
|
86
|
+
ctx.deps.thinking_animation.start()
|
|
87
|
+
return f"Error appending URL {fixture_url} to {datasource_name}: {e}"
|
|
@@ -17,7 +17,7 @@ tinybird/datafile/exceptions.py,sha256=8rw2umdZjtby85QbuRKFO5ETz_eRHwUY5l7eHsy1w
|
|
|
17
17
|
tinybird/datafile/parse_connection.py,sha256=tRyn2Rpr1TeWet5BXmMoQgaotbGdYep1qiTak_OqC5E,1825
|
|
18
18
|
tinybird/datafile/parse_datasource.py,sha256=ssW8QeFSgglVFi3sDZj_HgkJiTJ2069v2JgqnH3CkDE,1825
|
|
19
19
|
tinybird/datafile/parse_pipe.py,sha256=xf4m0Tw44QWJzHzAm7Z7FwUoUUtr7noMYjU1NiWnX0k,3880
|
|
20
|
-
tinybird/tb/__cli__.py,sha256=
|
|
20
|
+
tinybird/tb/__cli__.py,sha256=dY306yA2ppVsNM4k4b-zkQx3D8E5suyaU1zBkami7R0,247
|
|
21
21
|
tinybird/tb/check_pypi.py,sha256=Gp0HkHHDFMSDL6nxKlOY51z7z1Uv-2LRexNTZSHHGmM,552
|
|
22
22
|
tinybird/tb/cli.py,sha256=FdDFEIayjmsZEVsVSSvRiVYn_FHOVg_zWQzchnzfWho,1008
|
|
23
23
|
tinybird/tb/client.py,sha256=pJbdkWMXGAqKseNAvdsRRnl_c7I-DCMB0dWCQnG82nU,54146
|
|
@@ -68,15 +68,16 @@ tinybird/tb/modules/watch.py,sha256=No0bK1M1_3CYuMaIgylxf7vYFJ72lTJe3brz6xQ-mJo,
|
|
|
68
68
|
tinybird/tb/modules/workspace.py,sha256=Q_8HcxMsNg8QG9aBlwcWS2umrDP5IkTIHqqz3sfmGuc,11341
|
|
69
69
|
tinybird/tb/modules/workspace_members.py,sha256=5JdkJgfuEwbq-t6vxkBhYwgsiTDxF790wsa6Xfif9nk,8608
|
|
70
70
|
tinybird/tb/modules/agent/__init__.py,sha256=i3oe3vDIWWPaicdCM0zs7D7BJ1W0k7th93ooskHAV00,54
|
|
71
|
-
tinybird/tb/modules/agent/agent.py,sha256=
|
|
71
|
+
tinybird/tb/modules/agent/agent.py,sha256=lemqCQyYYnoSqp3gVPMAc0J7b4OrLy8sCpUk1_Vy9bo,22150
|
|
72
72
|
tinybird/tb/modules/agent/animations.py,sha256=4WOC5_2BracttmMCrV0H91tXfWcUzQHBUaIJc5FA7tE,3490
|
|
73
73
|
tinybird/tb/modules/agent/banner.py,sha256=KX_e467uiy1gWOZ4ofTZt0GCFGQqHQ_8Ob27XLQqda0,3053
|
|
74
74
|
tinybird/tb/modules/agent/memory.py,sha256=H6SJK--2L5C87B7AJd_jMqsq3sCvFvZwZXmajuT0GBE,1171
|
|
75
75
|
tinybird/tb/modules/agent/models.py,sha256=LW1D27gjcd_jwFmghEzteCgToDfodX2B6B5S8BYbysw,735
|
|
76
|
-
tinybird/tb/modules/agent/prompts.py,sha256
|
|
76
|
+
tinybird/tb/modules/agent/prompts.py,sha256=-2Gazjrh3NGjSqtw9yZ8Up-tQLDD-kKGj7rwPSk71oY,13377
|
|
77
77
|
tinybird/tb/modules/agent/utils.py,sha256=me5-kflBxV4N4psBSxI3vAfmFvAtsSnboc7ND3M9omw,26249
|
|
78
78
|
tinybird/tb/modules/agent/tools/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
79
|
-
tinybird/tb/modules/agent/tools/
|
|
79
|
+
tinybird/tb/modules/agent/tools/analyze.py,sha256=7oxJ3waCS24Qja_k5GUB59_XiHTG9pCewZogOXhH0cA,3495
|
|
80
|
+
tinybird/tb/modules/agent/tools/append.py,sha256=brd7OFSFJCC3-k6_1p2Qlcg_v2cDlpL_iBUtWtpiqfw,3821
|
|
80
81
|
tinybird/tb/modules/agent/tools/build.py,sha256=LhzJMx6tbxC7gogIrxhfKJc-SDgoSR-FC6IunfaCdn8,758
|
|
81
82
|
tinybird/tb/modules/agent/tools/create_datafile.py,sha256=wcPcChACTIFKw0lKFTlhm0sWJKhQkMLPLnGNpKyeETA,2962
|
|
82
83
|
tinybird/tb/modules/agent/tools/deploy.py,sha256=WrsSlaufKGOBx0S13uoMQQH2DnKue5LQ231Rx4RXh2I,1443
|
|
@@ -88,7 +89,6 @@ tinybird/tb/modules/agent/tools/get_openapi_definition.py,sha256=9cQ-SUeB1NVhPJN
|
|
|
88
89
|
tinybird/tb/modules/agent/tools/mock.py,sha256=Omog_gdEdm8YuBXNrJdHwxHqjL_ji9UIr75mALF4ozI,3408
|
|
89
90
|
tinybird/tb/modules/agent/tools/plan.py,sha256=pr6LnItz6vlOeCG8GE459ExsrBEG0KLx-g02SZGNjXU,1217
|
|
90
91
|
tinybird/tb/modules/agent/tools/preview_datafile.py,sha256=e9q5fR0afApcrntzFrnuHmd10ex7MG_GM6T0Pwc9bRI,850
|
|
91
|
-
tinybird/tb/modules/agent/tools/read_fixture_data.py,sha256=rvTdVlZsu3rQTSWqXzpFt4LEwnBcMLIT8hlI5C7MVN4,1430
|
|
92
92
|
tinybird/tb/modules/agent/tools/request_endpoint.py,sha256=iwzjYLtX_4YS7b9KzRwPGtVkx2UNujMAXv8m7mm8Fac,2683
|
|
93
93
|
tinybird/tb/modules/datafile/build.py,sha256=NFKBrusFLU0WJNCXePAFWiEDuTaXpwc0lHlOQWEJ43s,51117
|
|
94
94
|
tinybird/tb/modules/datafile/build_common.py,sha256=2yNdxe49IMA9wNvl25NemY2Iaz8L66snjOdT64dm1is,4511
|
|
@@ -110,8 +110,8 @@ tinybird/tb_cli_modules/config.py,sha256=IsgdtFRnUrkY8-Zo32lmk6O7u3bHie1QCxLwgp4
|
|
|
110
110
|
tinybird/tb_cli_modules/exceptions.py,sha256=pmucP4kTF4irIt7dXiG-FcnI-o3mvDusPmch1L8RCWk,3367
|
|
111
111
|
tinybird/tb_cli_modules/regions.py,sha256=QjsL5H6Kg-qr0aYVLrvb1STeJ5Sx_sjvbOYO0LrEGMk,166
|
|
112
112
|
tinybird/tb_cli_modules/telemetry.py,sha256=Hh2Io8ZPROSunbOLuMvuIFU4TqwWPmQTqal4WS09K1A,10449
|
|
113
|
-
tinybird-0.0.1.
|
|
114
|
-
tinybird-0.0.1.
|
|
115
|
-
tinybird-0.0.1.
|
|
116
|
-
tinybird-0.0.1.
|
|
117
|
-
tinybird-0.0.1.
|
|
113
|
+
tinybird-0.0.1.dev254.dist-info/METADATA,sha256=OJ8Hn_avSbp70iyOkjiWretCRbTvI-jUk514rgWiA40,1733
|
|
114
|
+
tinybird-0.0.1.dev254.dist-info/WHEEL,sha256=In9FTNxeP60KnTkGw7wk6mJPYd_dQSjEZmXdBdMCI-8,91
|
|
115
|
+
tinybird-0.0.1.dev254.dist-info/entry_points.txt,sha256=LwdHU6TfKx4Qs7BqqtaczEZbImgU7Abe9Lp920zb_fo,43
|
|
116
|
+
tinybird-0.0.1.dev254.dist-info/top_level.txt,sha256=VqqqEmkAy7UNaD8-V51FCoMMWXjLUlR0IstvK7tJYVY,54
|
|
117
|
+
tinybird-0.0.1.dev254.dist-info/RECORD,,
|
|
@@ -1,36 +0,0 @@
|
|
|
1
|
-
import json
|
|
2
|
-
from pathlib import Path
|
|
3
|
-
|
|
4
|
-
import click
|
|
5
|
-
from pydantic_ai import RunContext
|
|
6
|
-
|
|
7
|
-
from tinybird.tb.modules.agent.utils import TinybirdAgentContext
|
|
8
|
-
from tinybird.tb.modules.feedback_manager import FeedbackManager
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
def read_fixture_data(ctx: RunContext[TinybirdAgentContext], fixture_pathname: str):
|
|
12
|
-
"""Read fixture data in the project folder
|
|
13
|
-
|
|
14
|
-
Args:
|
|
15
|
-
fixture_pathname (str): a path to a fixture file. Required.
|
|
16
|
-
|
|
17
|
-
Returns:
|
|
18
|
-
str: The content of the fixture data file.
|
|
19
|
-
"""
|
|
20
|
-
ctx.deps.thinking_animation.stop()
|
|
21
|
-
click.echo(FeedbackManager.highlight(message=f"» Analyzing {fixture_pathname}..."))
|
|
22
|
-
fixture_path = Path(ctx.deps.folder) / fixture_pathname.lstrip("/")
|
|
23
|
-
|
|
24
|
-
if not fixture_path.exists():
|
|
25
|
-
click.echo(FeedbackManager.error(message=f"No fixture data found for {fixture_pathname}."))
|
|
26
|
-
ctx.deps.thinking_animation.start()
|
|
27
|
-
return f"No fixture data found for {fixture_pathname}. Please check the name of the fixture and try again."
|
|
28
|
-
|
|
29
|
-
response = ctx.deps.analyze_fixture(fixture_path=str(fixture_path))
|
|
30
|
-
click.echo(FeedbackManager.success(message="✓ Done!\n"))
|
|
31
|
-
ctx.deps.thinking_animation.start()
|
|
32
|
-
# limit content to first 10 rows
|
|
33
|
-
data = response["preview"]["data"][:10]
|
|
34
|
-
columns = response["analysis"]["columns"]
|
|
35
|
-
|
|
36
|
-
return f"#Result of analysis of {fixture_pathname}:\n##Columns:\n{json.dumps(columns)}\n##Data sample:\n{json.dumps(data)}"
|
|
File without changes
|
|
File without changes
|
|
File without changes
|