tinybird 0.0.1.dev24__tar.gz → 0.0.1.dev26__tar.gz

{tinybird-0.0.1.dev24 → tinybird-0.0.1.dev26}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: tinybird
-Version: 0.0.1.dev24
+Version: 0.0.1.dev26
 Summary: Tinybird Command Line Tool
 Home-page: https://www.tinybird.co/docs/cli/introduction.html
 Author: Tinybird

{tinybird-0.0.1.dev24 → tinybird-0.0.1.dev26}/tinybird/prompts.py

@@ -117,6 +117,93 @@ FROM numbers({rows})
 - Do NOT add a semicolon at the end of the query
 - Do NOT add any FORMAT at the end of the query, because it will be added later by Tinybird.
 
+# Examples with different schemas, like an array field or a nested JSON field:
+
+## Example schema with an array field:
+
+### Schema:
+
+SCHEMA >
+    `order_id` UInt64 `json:$.order_id`,
+    `customer_id` UInt64 `json:$.customer_id`,
+    `order_date` DateTime `json:$.order_date`,
+    `total_amount` Float64 `json:$.total_amount`,
+    `items` Array(String) `json:$.items[:]` // This is an array field 
+
+### Desired final output of the query:
+{
+  "order_id": 123456,
+  "customer_id": 7890,
+  "order_date": "2024-11-30T10:30:00.000Z",
+  "total_amount": 150.0,
+  "items": ["item1", "item2", "item3"]
+}
+
+### Example SQL output with an array field:
+
+SELECT
+  concat('ord_', toString(rand() % 10000)) AS order_id,
+  concat('cust_', toString(rand() % 10000)) AS customer_id,
+  now() - rand() % 86400 AS order_date,
+  rand() % 1000 AS total_amount,
+  arrayMap(x -> concat('item_', toString(x)), range(1, rand() % 5 + 1)) AS items
+FROM numbers(ROWS)
+
+## Example schema with a nested JSON field:
+
+### Schema:
+
+SCHEMA >
+    `request_id` String `json:$.request_id`,
+    `timestamp` DateTime `json:$.timestamp`,
+    `model` String `json:$.request.model`,
+    `temperature` Float32 `json:$.request.options.temperature`,
+    `max_tokens` UInt32 `json:$.request.options.max_tokens`,
+    `stream` UInt8 `json:$.request.options.stream`
+
+### Desired final output of the query:
+
+Note that the important part is generating the nested fields:
+json:$.request.options.max_tokens > this means that the max_tokens field is nested inside the options field inside the request field.
+
+{
+  "request_id": "req_abc123",
+  "timestamp": "2024-11-30T10:30:00.000Z",
+  "request": {
+    "model": "gpt-4",
+    "options": {
+      "temperature": 0.7,
+      "max_tokens": 1000,
+      "stream": false
+    }
+  }
+}
+
+### Example SQL output with nested fields:
+
+SELECT
+    request_id,
+    timestamp,
+    CAST(concat('{
+        "model": "', model, '",
+        "options": {
+            "temperature": ', temperature, ',
+            "max_tokens": ', max_tokens, ',
+            "stream": ', IF(stream = 1, 'true', 'false'), '
+        }
+    }'), 'JSON') AS request
+FROM
+(
+    SELECT
+        concat('req_', lower(hex(randomString(6)))) AS request_id,
+        (now() - toIntervalDay(rand() % 30)) + toIntervalSecond(rand() % 86400) AS timestamp,
+        ['gpt-4', 'gpt-3.5-turbo', 'gpt-4-turbo'][(rand() % 3) + 1] AS model,
+        round(rand() / 10, 2) AS temperature,
+        500 + (rand() % 2500) AS max_tokens,
+        rand() % 2 AS stream
+    FROM numbers(ROWS)
+)
+
 # Extra context:
 {context}
 

{tinybird-0.0.1.dev24 → tinybird-0.0.1.dev26}/tinybird/tb/__cli__.py

@@ -4,5 +4,5 @@ __description__ = 'Tinybird Command Line Tool'
 __url__ = 'https://www.tinybird.co/docs/cli/introduction.html'
 __author__ = 'Tinybird'
 __author_email__ = 'support@tinybird.co'
-__version__ = '0.0.1.dev24'
-__revision__ = '3de7f2a'
+__version__ = '0.0.1.dev26'
+__revision__ = 'f0e9917'

{tinybird-0.0.1.dev24 → tinybird-0.0.1.dev26}/tinybird/tb/modules/build.py

@@ -105,7 +105,7 @@ def build(
     async def build_once(filenames: List[str]):
         ok = False
         try:
-            click.echo("⚡ Building project...\n")
+            click.echo(FeedbackManager.highlight(message="» Building project...\n"))
             time_start = time.time()
             await process(filenames=filenames, watch=False)
             time_end = time.time()
@@ -117,7 +117,7 @@ def build(
                     fixture_path = get_fixture_dir() / f"{name}.ndjson"
                     if fixture_path.exists():
                         await append_datasource(tb_client, ds_path.stem, str(fixture_path), silent=True)
-            click.echo(FeedbackManager.success(message=f"\n✓ Build completed in {elapsed_time:.1f}s\n"))
+            click.echo(FeedbackManager.success(message=f"\n✓ Build completed in {elapsed_time:.1f}s"))
             ok = True
         except Exception as e:
             click.echo(FeedbackManager.error_exception(error=e))
@@ -154,7 +154,7 @@ def build(
             pipes=pipes,
             endpoints=endpoints,
         )
-        click.echo(FeedbackManager.highlight(message="◎ Watching for changes..."))
+        click.echo(FeedbackManager.gray(message="\nWatching for changes..."))
         watcher_thread = threading.Thread(
             target=watch_files, args=(filenames, process, shell, folder, build_ok), daemon=True
         )

{tinybird-0.0.1.dev24 → tinybird-0.0.1.dev26}/tinybird/tb/modules/create.py

@@ -71,16 +71,16 @@ async def create(
                 )
                 return
         local_client = await get_tinybird_local_client(folder)
-        click.echo(FeedbackManager.gray(message="Creating new project structure..."))
+        click.echo(FeedbackManager.highlight(message="\n» Creating new project structure..."))
         await project_create(local_client, tb_client, user_token, data, prompt, folder)
         click.echo(FeedbackManager.success(message="✓ Scaffolding completed!\n"))
 
-        click.echo(FeedbackManager.gray(message="\nCreating CI/CD files for GitHub and GitLab..."))
+        click.echo(FeedbackManager.highlight(message="\n» Creating CI/CD files for GitHub and GitLab..."))
         init_git(folder)
         await init_cicd(data_project_dir=os.path.relpath(folder))
         click.echo(FeedbackManager.success(message="✓ Done!\n"))
 
-        click.echo(FeedbackManager.gray(message="Building fixtures..."))
+        click.echo(FeedbackManager.highlight(message="\n» Building fixtures..."))
 
         if demo:
             # Users datasource

{tinybird-0.0.1.dev24 → tinybird-0.0.1.dev26}/tinybird/tb/modules/datafile/build.py

@@ -773,11 +773,11 @@ async def get_processed(
     filenames: Iterable[str],
     changed: Optional[Dict[str, Any]] = None,
     verbose: bool = False,
-    deps: List[str] = [],
-    dep_map: Dict[str, Any] = {},
-    to_run: Dict[str, Any] = {},
+    deps: Optional[List[str]] = None,
+    dep_map: Optional[Dict[str, Any]] = None,
+    to_run: Optional[Dict[str, Any]] = None,
     vendor_paths: Optional[List[Tuple[str, str]]] = None,
-    processed: Set[str] = set(),
+    processed: Optional[Set[str]] = None,
     tb_client: TinyB = None,
     skip_connectors: bool = False,
     current_ws: Optional[Dict[str, Any]] = None,
@@ -786,6 +786,15 @@ async def get_processed(
     dir_path: Optional[str] = None,
     embedded_datasources: Optional[Dict[str, Any]] = None,
 ):
+    if deps is None:
+        deps = []
+    if dep_map is None:
+        dep_map = {}
+    if to_run is None:
+        to_run = {}
+    if processed is None:
+        processed = set()
+
     for filename in filenames:
         # just process changed filenames (tb deploy and --only-changes)
         if changed:

{tinybird-0.0.1.dev24 → tinybird-0.0.1.dev26}/tinybird/tb/modules/feedback_manager.py

@@ -25,7 +25,7 @@ def print_message(message: str, color: str = bcolors.ENDC) -> Callable[..., str]
 
 
 def error_message(message: str) -> Callable[..., str]:
-    return print_message(f"\n** {message}", bcolors.FAIL)
+    return print_message(f"{message}", bcolors.FAIL)
 
 
 def error_exception(message: str) -> Callable[..., str]:

{tinybird-0.0.1.dev24 → tinybird-0.0.1.dev26}/tinybird/tb/modules/mock.py

@@ -31,6 +31,7 @@ async def mock(ctx: click.Context, datasource: str, rows: int, prompt: str, fold
     try:
         datasource_path = Path(datasource)
         datasource_name = datasource
+        click.echo(FeedbackManager.highlight(message=f"\n» Creating fixture for {datasource_name}..."))
         if datasource_path.suffix == ".datasource":
             datasource_name = datasource_path.stem
         else:
@@ -43,10 +44,9 @@ async def mock(ctx: click.Context, datasource: str, rows: int, prompt: str, fold
             if prompt_path.exists():
                 prompt = prompt_path.read_text()
         else:
-            click.echo(FeedbackManager.gray(message=f"Overriding prompt for {datasource_name}..."))
+            click.echo(FeedbackManager.info(message="* Overriding last prompt..."))
             prompt_path.write_text(prompt)
 
-        click.echo(FeedbackManager.gray(message=f"Creating fixture for {datasource_name}..."))
         datasource_content = datasource_path.read_text()
         config = CLIConfig.get_project_config()
         user_client = config.get_client()
@@ -69,7 +69,7 @@ async def mock(ctx: click.Context, datasource: str, rows: int, prompt: str, fold
         data = result.get("data", [])[:rows]
         fixture_name = build_fixture_name(datasource_path.absolute().as_posix(), datasource_name, datasource_content)
         persist_fixture(fixture_name, data)
-        click.echo(FeedbackManager.success(message="✓ Done!"))
+        click.echo(FeedbackManager.success(message=f"✓ /fixtures/{fixture_name}.ndjson created with {rows} rows"))
 
     except Exception as e:
         raise CLIException(FeedbackManager.error_exception(error=e))

{tinybird-0.0.1.dev24 → tinybird-0.0.1.dev26}/tinybird/tb/modules/test.py

@@ -81,7 +81,7 @@ async def test_create(pipe: str, prompt: Optional[str], folder: str) -> None:
             if not pipe_path.exists():
                 pipe_path = Path("pipes", f"{pipe}.pipe")
 
-        click.echo(FeedbackManager.gray(message=f"\nCreating tests for {pipe_name} endpoint..."))
+        click.echo(FeedbackManager.highlight(message=f"\n» Creating tests for {pipe_name} endpoint..."))
         pipe_path = Path(folder) / pipe_path
         pipe_content = pipe_path.read_text()
 
@@ -96,7 +96,7 @@ async def test_create(pipe: str, prompt: Optional[str], folder: str) -> None:
         test_expectations = await llm.create_tests(
             pipe_content=pipe_content, pipe_params=pipe_params, prompt=prompt or ""
         )
-        valid_test_expectations = []
+        valid_test_expectations: List[Dict[str, Any]] = []
         for test in test_expectations.tests:
             valid_test = test.model_dump()
             test_params = (
@@ -118,9 +118,13 @@ async def test_create(pipe: str, prompt: Optional[str], folder: str) -> None:
                 valid_test["expected_result"] = response.text or ""
 
             valid_test_expectations.append(valid_test)
+
         if valid_test_expectations:
             generate_test_file(pipe_name, valid_test_expectations, folder, mode="a")
-            click.echo(FeedbackManager.info(message=f"✓ /tests/{pipe_name}.yaml"))
+            for test in valid_test_expectations:
+                test_name = test["name"]
+                click.echo(FeedbackManager.info(message=f"✓ {test_name} created"))
+
         click.echo(FeedbackManager.success(message="✓ Done!\n"))
     except Exception as e:
         raise CLIException(FeedbackManager.error_exception(error=e))
@@ -139,37 +143,43 @@ async def test_create(pipe: str, prompt: Optional[str], folder: str) -> None:
 )
 @coro
 async def test_update(pipe: str, folder: str) -> None:
-    client = await get_tinybird_local_client(os.path.abspath(folder))
-    pipe_tests_path = Path(pipe)
-    pipe_name = pipe
-    if pipe_tests_path.suffix == ".yaml":
-        pipe_name = pipe_tests_path.stem
-    else:
-        pipe_tests_path = Path("tests", f"{pipe}.yaml")
-
-    click.echo(FeedbackManager.gray(message=f"\nUpdating tests expectations for {pipe_name} endpoint..."))
-    pipe_tests_path = Path(folder) / pipe_tests_path
-    pipe_tests_content = yaml.safe_load(pipe_tests_path.read_text())
-    for test in pipe_tests_content:
-        test_params = test["parameters"] if test["parameters"].startswith("?") else f"?{test['parameters']}"
-        response = None
-        try:
-            response = await client._req_raw(f"/v0/pipes/{pipe_name}.ndjson{test_params}")
-        except Exception:
-            continue
-
-        if response.status_code >= 400:
-            test["expected_http_status"] = response.status_code
-            test["expected_result"] = response.json()["error"]
+    try:
+        client = await get_tinybird_local_client(os.path.abspath(folder))
+        pipe_tests_path = Path(pipe)
+        pipe_name = pipe
+        if pipe_tests_path.suffix == ".yaml":
+            pipe_name = pipe_tests_path.stem
         else:
-            if "expected_http_status" in test:
-                del test["expected_http_status"]
+            pipe_tests_path = Path("tests", f"{pipe}.yaml")
 
-            test["expected_result"] = response.text or ""
+        click.echo(FeedbackManager.highlight(message=f"\n» Updating tests expectations for {pipe_name} endpoint..."))
+        pipe_tests_path = Path(folder) / pipe_tests_path
+        pipe_tests_content = yaml.safe_load(pipe_tests_path.read_text())
+        for test in pipe_tests_content:
+            test_params = test["parameters"] if test["parameters"].startswith("?") else f"?{test['parameters']}"
+            response = None
+            try:
+                response = await client._req_raw(f"/v0/pipes/{pipe_name}.ndjson{test_params}")
+            except Exception:
+                continue
+
+            if response.status_code >= 400:
+                test["expected_http_status"] = response.status_code
+                test["expected_result"] = response.json()["error"]
+            else:
+                if "expected_http_status" in test:
+                    del test["expected_http_status"]
 
-    generate_test_file(pipe_name, pipe_tests_content, folder)
-    click.echo(FeedbackManager.info(message=f"✓ /tests/{pipe_name}.yaml"))
-    click.echo(FeedbackManager.success(message="✓ Done!\n"))
+                test["expected_result"] = response.text or ""
+
+        generate_test_file(pipe_name, pipe_tests_content, folder)
+        for test in pipe_tests_content:
+            test_name = test["name"]
+            click.echo(FeedbackManager.info(message=f"✓ {test_name} updated"))
+
+        click.echo(FeedbackManager.success(message="✓ Done!\n"))
+    except Exception as e:
+        click.echo(FeedbackManager.error(message=str(e)))
 
 
 @test.command(
@@ -185,6 +195,7 @@ async def test_update(pipe: str, folder: str) -> None:
 )
 @coro
 async def test_run(name: Tuple[str, ...], folder: str) -> None:
+    click.echo(FeedbackManager.highlight(message="\n» Running tests"))
     client = await get_tinybird_local_client(os.path.abspath(folder))
     paths = [Path(n) for n in name]
     endpoints = [f"./tests/{p.stem}.yaml" for p in paths]
@@ -192,6 +203,7 @@ async def test_run(name: Tuple[str, ...], folder: str) -> None:
 
     async def run_test(test_file):
         test_file_path = Path(test_file)
+        click.echo(FeedbackManager.info(message=f"\n* {test_file_path.stem}{test_file_path.suffix}"))
         test_file_content = yaml.safe_load(test_file_path.read_text())
         for test in test_file_content:
             try:
@@ -219,20 +231,21 @@ async def test_run(name: Tuple[str, ...], folder: str) -> None:
                     raise Exception(
                         f"\nExpected: \n{test['expected_result']}\nGot: \n{expected_result}\nDiff: \n{printable_diff}"
                     )
-                click.echo(FeedbackManager.success(message=f"✓ {test_file_path.name} - {test['name']}"))
+                click.echo(FeedbackManager.info(message=f"✓ {test['name']} passed"))
             except Exception as e:
-                click.echo(FeedbackManager.error(message=f"✗ {test_file_path.name} - {test['name']}"))
-                click.echo(FeedbackManager.error(message=f"Output and expected output are different: \n{e}"))
+                click.echo(FeedbackManager.error(message=f"✗ {test['name']} failed"))
+                click.echo(FeedbackManager.error(message=f"\n** Output and expected output are different: \n{e}"))
                 return False
         return True
 
-    tests_failed = False
+    failed_tests_count = 0
+    test_count = len(file_list)
     for test_file in file_list:
         if not await run_test(test_file):
-            tests_failed = True
+            failed_tests_count += 1
 
-    if tests_failed:
-        click.echo(FeedbackManager.error(message="✗ Some tests failed"))
+    if failed_tests_count:
+        click.echo(FeedbackManager.error(message=f"\n✗ {test_count - failed_tests_count}/{test_count} passed"))
         exit(1)
     else:
-        click.echo(FeedbackManager.success(message="✓ All tests passed"))
+        click.echo(FeedbackManager.success(message=f"\n✓ {test_count}/{test_count} passed"))

{tinybird-0.0.1.dev24 → tinybird-0.0.1.dev26}/tinybird/tb/modules/watch.py

@@ -82,7 +82,7 @@ def watch_files(
 ) -> None:
     # Handle both sync and async process functions
     async def process_wrapper(files: List[str]) -> None:
-        click.echo("⚡ Rebuilding...")
+        click.echo(FeedbackManager.highlight(message="» Rebuilding..."))
         time_start = time.time()
         if asyncio.iscoroutinefunction(process):
             await process(files, watch=True)

{tinybird-0.0.1.dev24 → tinybird-0.0.1.dev26}/tinybird.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: tinybird
-Version: 0.0.1.dev24
+Version: 0.0.1.dev26
 Summary: Tinybird Command Line Tool
 Home-page: https://www.tinybird.co/docs/cli/introduction.html
 Author: Tinybird