tinybird 0.0.1.dev253__tar.gz → 0.0.1.dev255__tar.gz

{tinybird-0.0.1.dev253 → tinybird-0.0.1.dev255}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: tinybird
-Version: 0.0.1.dev253
+Version: 0.0.1.dev255
 Summary: Tinybird Command Line Tool
 Home-page: https://www.tinybird.co/docs/forward/commands
 Author: Tinybird

{tinybird-0.0.1.dev253 → tinybird-0.0.1.dev255}/tinybird/prompts.py

@@ -689,7 +689,6 @@ materialized_pipe_instructions = """
 - TYPE MATERIALIZED is the type of the pipe and it is mandatory for materialized pipes.
 - The content of the .pipe file must follow the materialized_pipe_content format.
 - Use State modifier for the aggregated columns in the pipe.
-- Keep the SQL query simple and avoid using complex queries with joins, subqueries, etc.
 </materialized_pipe_instructions>
 <materialized_pipe_content>
 NODE daily_sales
@@ -812,11 +811,11 @@ TYPE endpoint
 """
 
 pipe_instructions = """
+Follow these instructions when creating or updating any type of .pipe file:
 <pipe_file_instructions>
     - The pipe names must be unique.
     - Nodes do NOT use the same name as the Pipe they belong to. So if the pipe name is "my_pipe", the nodes must be named different like "my_pipe_node_1", "my_pipe_node_2", etc.
     - Node names MUST be different from the resource names in the project.
-    - Avoid more than one node per pipe unless it is really necessary or requested by the user.
     - No indentation is allowed for property names: DESCRIPTION, NODE, SQL, TYPE, etc.
     - Allowed TYPE values are: endpoint, copy, materialized, sink.
     - Add always the output node in the TYPE section or in the last node of the pipe.

{tinybird-0.0.1.dev253 → tinybird-0.0.1.dev255}/tinybird/tb/__cli__.py

@@ -4,5 +4,5 @@ __description__ = 'Tinybird Command Line Tool'
 __url__ = 'https://www.tinybird.co/docs/forward/commands'
 __author__ = 'Tinybird'
 __author_email__ = 'support@tinybird.co'
-__version__ = '0.0.1.dev253'
-__revision__ = '3bc2143'
+__version__ = '0.0.1.dev255'
+__revision__ = '208306e'

{tinybird-0.0.1.dev253 → tinybird-0.0.1.dev255}/tinybird/tb/modules/agent/agent.py

@@ -30,22 +30,25 @@ from tinybird.tb.modules.agent.memory import clear_history
 from tinybird.tb.modules.agent.models import create_model, model_costs
 from tinybird.tb.modules.agent.prompts import (
     datafile_instructions,
+    endpoint_optimization_instructions,
     plan_instructions,
     resources_prompt,
+    sql_agent_instructions,
     sql_instructions,
 )
-from tinybird.tb.modules.agent.tools.append import append
+from tinybird.tb.modules.agent.tools.analyze import analyze_file, analyze_url
+from tinybird.tb.modules.agent.tools.append import append_file, append_url
 from tinybird.tb.modules.agent.tools.build import build
 from tinybird.tb.modules.agent.tools.create_datafile import create_datafile
 from tinybird.tb.modules.agent.tools.deploy import deploy
 from tinybird.tb.modules.agent.tools.deploy_check import deploy_check
+from tinybird.tb.modules.agent.tools.diff_resource import diff_resource
 from tinybird.tb.modules.agent.tools.execute_query import execute_query
 from tinybird.tb.modules.agent.tools.get_endpoint_stats import get_endpoint_stats
 from tinybird.tb.modules.agent.tools.get_openapi_definition import get_openapi_definition
 from tinybird.tb.modules.agent.tools.mock import mock
 from tinybird.tb.modules.agent.tools.plan import plan
 from tinybird.tb.modules.agent.tools.preview_datafile import preview_datafile
-from tinybird.tb.modules.agent.tools.read_fixture_data import read_fixture_data
 from tinybird.tb.modules.agent.tools.request_endpoint import request_endpoint
 from tinybird.tb.modules.agent.utils import TinybirdAgentContext, show_input
 from tinybird.tb.modules.build_common import process as build_process
@@ -112,12 +115,15 @@ You have access to the following tools:
 5. `deploy` - Deploy the project to Tinybird Cloud.
 6. `deploy_check` - Check if the project can be deployed to Tinybird Cloud before deploying it.
 7. `mock` - Create mock data for a landing datasource.
-8. `read_fixture_data` - Read a fixture data file present in the project folder.
-9. `append` - Append existing fixture to a datasource.
-10. `get_endpoint_stats` - Get metrics of the requests to an endpoint.
-11. `get_openapi_definition` - Get the OpenAPI definition for all endpoints that are built/deployed to Tinybird Cloud or Local.
-12. `execute_query` - Execute a query against Tinybird Cloud or Local.
+8. `analyze_file` - Analyze the content of a fixture file present in the project folder.
+9. `analyze_url` - Analyze the content of an external url.
+9. `append_file` - Append a file present in the project to a datasource.
+10. `append_url` - Append an external url to a datasource.
+11. `get_endpoint_stats` - Get metrics of the requests to an endpoint.
+12. `get_openapi_definition` - Get the OpenAPI definition for all endpoints that are built/deployed to Tinybird Cloud or Local.
+13. `execute_query` - Execute a query against Tinybird Cloud or Local.
 13. `request_endpoint` - Request an endpoint against Tinybird Cloud or Local.
+14. `diff_resource` - Diff the content of a resource in Tinybird Cloud vs Tinybird Local vs Project local file.
 
 # When creating or updating datafiles:
 1. Use `plan` tool to plan the creation or update of resources.
@@ -133,21 +139,11 @@ You have access to the following tools:
 - If the user does not specify anything about the desired schema, create a schema like this:
 SCHEMA >
     `data` String `json:$`
+
 - Use always json paths with .ndjson files.
 
 # When user wants to optimize an endpoint:
-First check if the query is optimized. E.g is filtering by a column present in the sorting key.
-Avoid when possible to update the landing datasource.
-Check endpoint stats to analyze how the endpoint is performing. Use `get_endpoint_stats` tool to get the stats.
-You can use `request_endpoint` tool to check if the endpoint is working as expected.
-You can use `execute_query` tool to execute a query against Tinybird Cloud or Local, to test different queries.
-When your data is in Tinybird, you can create intermediate data sources to preprocess data and make the endpoints faster. This can be done by using materialized views or copy pipes.
-- Copy pipes capture the result of a pipe at a specific point in time and write it to a target data source. They can run on a schedule or run on demand, making them ideal for event-sourced snapshots, data experimentation, and deduplication with snapshots.
-- Materialized views continuously re-evaluate a query as new events are inserted, maintaining an always up-to-date derived dataset. Unlike copy pipes which create point-in-time snapshots, materialized views provide real-time transformations of your data.
-Each approach has its own strengths and use cases:
-- Use copy pipes when you need scheduled or on-demand snapshots of your data.
-- Use materialized views when you need continuous, real-time transformations.
-Finally, update the existing endpoint itself, do not add a new one.
+{endpoint_optimization_instructions}
 
 IMPORTANT: If the user cancels some of the steps or there is an error in file creation, DO NOT continue with the plan. Stop the process and wait for the user before using any other tool.
 IMPORTANT: Every time you finish a plan and start a new resource creation or update process, create a new plan before starting with the changes.
@@ -179,6 +175,7 @@ IMPORTANT: Every time you finish a plan and start a new resource creation or upd
 {copy_pipe_instructions}
 
 # Working with SQL queries:
+{sql_agent_instructions}
 {sql_instructions}
 
 # Working with connections files:
@@ -189,6 +186,13 @@ Kafka: {kafka_connection_example}
 S3: {s3_connection_example}
 GCS: {gcs_connection_example}
 
+# When executing a query or requesting an endpoint:
+- You need to be sure that the selected resource is updated to the last version in the environment you are working on.
+- Use `diff_resource` tool to compare the content of the resource to compare the differences between environments.
+- Project local file is the source of truth.
+- If the resource is not present or updated to the last version in Tinybird Local, it means you need to build the project. 
+- If the resource is not present or updated to the last version in Tinybird Cloud, it means you need to deploy the project.
+
 # Info
 Today is {datetime.now().strftime("%Y-%m-%d")}
 """,
@@ -200,8 +204,10 @@ Today is {datetime.now().strftime("%Y-%m-%d")}
                 Tool(deploy, docstring_format="google", require_parameter_descriptions=True, takes_ctx=True),
                 Tool(deploy_check, docstring_format="google", require_parameter_descriptions=True, takes_ctx=True),
                 Tool(mock, docstring_format="google", require_parameter_descriptions=True, takes_ctx=True),
-                Tool(read_fixture_data, docstring_format="google", require_parameter_descriptions=True, takes_ctx=True),
-                Tool(append, docstring_format="google", require_parameter_descriptions=True, takes_ctx=True),
+                Tool(analyze_file, docstring_format="google", require_parameter_descriptions=True, takes_ctx=True),
+                Tool(analyze_url, docstring_format="google", require_parameter_descriptions=True, takes_ctx=True),
+                Tool(append_file, docstring_format="google", require_parameter_descriptions=True, takes_ctx=True),
+                Tool(append_url, docstring_format="google", require_parameter_descriptions=True, takes_ctx=True),
                 Tool(
                     get_endpoint_stats, docstring_format="google", require_parameter_descriptions=True, takes_ctx=True
                 ),
@@ -213,6 +219,7 @@ Today is {datetime.now().strftime("%Y-%m-%d")}
                 ),
                 Tool(execute_query, docstring_format="google", require_parameter_descriptions=True, takes_ctx=True),
                 Tool(request_endpoint, docstring_format="google", require_parameter_descriptions=True, takes_ctx=True),
+                Tool(diff_resource, docstring_format="google", require_parameter_descriptions=True, takes_ctx=True),
             ],
         )
 
@@ -238,10 +245,16 @@ Today is {datetime.now().strftime("%Y-%m-%d")}
                 mock_data=partial(mock_data, project=project, config=config),
                 append_data=partial(append_data, config=config),
                 analyze_fixture=partial(analyze_fixture, config=config),
-                execute_cloud_query=partial(execute_cloud_query, config=config),
-                execute_local_query=partial(execute_local_query, config=config),
+                execute_query_cloud=partial(execute_query_cloud, config=config),
+                execute_query_local=partial(execute_query_local, config=config),
                 request_endpoint_cloud=partial(request_endpoint_cloud, config=config),
                 request_endpoint_local=partial(request_endpoint_local, config=config),
+                get_datasource_datafile_cloud=partial(get_datasource_datafile_cloud, config=config),
+                get_datasource_datafile_local=partial(get_datasource_datafile_local, config=config),
+                get_pipe_datafile_cloud=partial(get_pipe_datafile_cloud, config=config),
+                get_pipe_datafile_local=partial(get_pipe_datafile_local, config=config),
+                get_connection_datafile_cloud=partial(get_connection_datafile_cloud, config=config),
+                get_connection_datafile_local=partial(get_connection_datafile_local, config=config),
                 get_project_files=project.get_project_files,
                 folder=folder,
                 thinking_animation=thinking_animation,
@@ -281,8 +294,8 @@ def run_agent(
     token = config.get("token", None)
     host = config.get("host", None)
     user_token = config.get("user_token", None)
-    workspace_id = config.get("id", None)
-    workspace_name = config.get("name", None)
+    workspace_id = config.get("id", "")
+    workspace_name = config.get("name", "")
     try:
         if not token or not host or not workspace_id or not user_token:
             yes = click.confirm(
@@ -443,18 +456,18 @@ def mock_data(
     )
 
 
-def analyze_fixture(config: dict[str, Any], fixture_path: str) -> dict[str, Any]:
+def analyze_fixture(config: dict[str, Any], fixture_path: str, format: str = "json") -> dict[str, Any]:
     local_client = get_tinybird_local_client(config, test=False, silent=True)
-    meta, _data = _analyze(fixture_path, local_client, Path(fixture_path).suffix.lstrip("."))
+    meta, _data = _analyze(fixture_path, local_client, format)
     return meta
 
 
-def execute_cloud_query(config: dict[str, Any], query: str, pipe_name: Optional[str] = None) -> dict[str, Any]:
+def execute_query_cloud(config: dict[str, Any], query: str, pipe_name: Optional[str] = None) -> dict[str, Any]:
     client = _get_tb_client(config["token"], config["host"])
     return client.query(sql=query, pipeline=pipe_name)
 
 
-def execute_local_query(config: dict[str, Any], query: str, pipe_name: Optional[str] = None) -> dict[str, Any]:
+def execute_query_local(config: dict[str, Any], query: str, pipe_name: Optional[str] = None) -> dict[str, Any]:
     local_client = get_tinybird_local_client(config, test=False, silent=True)
     return local_client.query(sql=query, pipeline=pipe_name)
 
@@ -471,3 +484,51 @@ def request_endpoint_local(
 ) -> dict[str, Any]:
     local_client = get_tinybird_local_client(config, test=False, silent=True)
     return local_client.pipe_data(endpoint_name, format="json", params=params)
+
+
+def get_datasource_datafile_cloud(config: dict[str, Any], datasource_name: str) -> str:
+    try:
+        client = _get_tb_client(config["token"], config["host"])
+        return client.datasource_file(datasource_name)
+    except Exception:
+        return "Datasource not found"
+
+
+def get_datasource_datafile_local(config: dict[str, Any], datasource_name: str) -> str:
+    try:
+        local_client = get_tinybird_local_client(config, test=False, silent=True)
+        return local_client.datasource_file(datasource_name)
+    except Exception:
+        return "Datasource not found"
+
+
+def get_pipe_datafile_cloud(config: dict[str, Any], pipe_name: str) -> str:
+    try:
+        client = _get_tb_client(config["token"], config["host"])
+        return client.pipe_file(pipe_name)
+    except Exception:
+        return "Pipe not found"
+
+
+def get_pipe_datafile_local(config: dict[str, Any], pipe_name: str) -> str:
+    try:
+        local_client = get_tinybird_local_client(config, test=False, silent=True)
+        return local_client.pipe_file(pipe_name)
+    except Exception:
+        return "Pipe not found"
+
+
+def get_connection_datafile_cloud(config: dict[str, Any], connection_name: str) -> str:
+    try:
+        client = _get_tb_client(config["token"], config["host"])
+        return client.connection_file(connection_name)
+    except Exception:
+        return "Connection not found"
+
+
+def get_connection_datafile_local(config: dict[str, Any], connection_name: str) -> str:
+    try:
+        local_client = get_tinybird_local_client(config, test=False, silent=True)
+        return local_client.connection_file(connection_name)
+    except Exception:
+        return "Connection not found"