tinybird 0.0.1.dev36__tar.gz → 0.0.1.dev38__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of tinybird might be problematic. Click here for more details.

Files changed (102) hide show
  1. {tinybird-0.0.1.dev36 → tinybird-0.0.1.dev38}/PKG-INFO +1 -1
  2. {tinybird-0.0.1.dev36 → tinybird-0.0.1.dev38}/tinybird/client.py +1 -1
  3. {tinybird-0.0.1.dev36 → tinybird-0.0.1.dev38}/tinybird/prompts.py +2 -0
  4. {tinybird-0.0.1.dev36 → tinybird-0.0.1.dev38}/tinybird/tb/__cli__.py +2 -2
  5. {tinybird-0.0.1.dev36 → tinybird-0.0.1.dev38}/tinybird/tb/cli.py +1 -0
  6. {tinybird-0.0.1.dev36 → tinybird-0.0.1.dev38}/tinybird/tb/modules/build.py +54 -39
  7. {tinybird-0.0.1.dev36 → tinybird-0.0.1.dev38}/tinybird/tb/modules/build_client.py +9 -15
  8. {tinybird-0.0.1.dev36 → tinybird-0.0.1.dev38}/tinybird/tb/modules/cli.py +18 -16
  9. tinybird-0.0.1.dev38/tinybird/tb/modules/copy.py +68 -0
  10. {tinybird-0.0.1.dev36 → tinybird-0.0.1.dev38}/tinybird/tb/modules/deployment.py +23 -25
  11. {tinybird-0.0.1.dev36 → tinybird-0.0.1.dev38}/tinybird/tb/modules/llm_utils.py +9 -12
  12. {tinybird-0.0.1.dev36 → tinybird-0.0.1.dev38}/tinybird/tb/modules/local_common.py +10 -4
  13. {tinybird-0.0.1.dev36 → tinybird-0.0.1.dev38}/tinybird/tb/modules/pipe.py +3 -8
  14. tinybird-0.0.1.dev38/tinybird/tb/modules/project.py +44 -0
  15. {tinybird-0.0.1.dev36 → tinybird-0.0.1.dev38}/tinybird/tb/modules/shell.py +16 -33
  16. {tinybird-0.0.1.dev36 → tinybird-0.0.1.dev38}/tinybird/tb/modules/test.py +7 -9
  17. {tinybird-0.0.1.dev36 → tinybird-0.0.1.dev38}/tinybird/tb/modules/token.py +1 -1
  18. {tinybird-0.0.1.dev36 → tinybird-0.0.1.dev38}/tinybird/tb/modules/watch.py +18 -12
  19. {tinybird-0.0.1.dev36 → tinybird-0.0.1.dev38}/tinybird.egg-info/PKG-INFO +1 -1
  20. {tinybird-0.0.1.dev36 → tinybird-0.0.1.dev38}/tinybird.egg-info/SOURCES.txt +2 -0
  21. {tinybird-0.0.1.dev36 → tinybird-0.0.1.dev38}/setup.cfg +0 -0
  22. {tinybird-0.0.1.dev36 → tinybird-0.0.1.dev38}/tinybird/__cli__.py +0 -0
  23. {tinybird-0.0.1.dev36 → tinybird-0.0.1.dev38}/tinybird/ch_utils/constants.py +0 -0
  24. {tinybird-0.0.1.dev36 → tinybird-0.0.1.dev38}/tinybird/ch_utils/engine.py +0 -0
  25. {tinybird-0.0.1.dev36 → tinybird-0.0.1.dev38}/tinybird/check_pypi.py +0 -0
  26. {tinybird-0.0.1.dev36 → tinybird-0.0.1.dev38}/tinybird/config.py +0 -0
  27. {tinybird-0.0.1.dev36 → tinybird-0.0.1.dev38}/tinybird/connectors.py +0 -0
  28. {tinybird-0.0.1.dev36 → tinybird-0.0.1.dev38}/tinybird/context.py +0 -0
  29. {tinybird-0.0.1.dev36 → tinybird-0.0.1.dev38}/tinybird/datafile.py +0 -0
  30. {tinybird-0.0.1.dev36 → tinybird-0.0.1.dev38}/tinybird/datatypes.py +0 -0
  31. {tinybird-0.0.1.dev36 → tinybird-0.0.1.dev38}/tinybird/feedback_manager.py +0 -0
  32. {tinybird-0.0.1.dev36 → tinybird-0.0.1.dev38}/tinybird/git_settings.py +0 -0
  33. {tinybird-0.0.1.dev36 → tinybird-0.0.1.dev38}/tinybird/sql.py +0 -0
  34. {tinybird-0.0.1.dev36 → tinybird-0.0.1.dev38}/tinybird/sql_template.py +0 -0
  35. {tinybird-0.0.1.dev36 → tinybird-0.0.1.dev38}/tinybird/sql_template_fmt.py +0 -0
  36. {tinybird-0.0.1.dev36 → tinybird-0.0.1.dev38}/tinybird/sql_toolset.py +0 -0
  37. {tinybird-0.0.1.dev36 → tinybird-0.0.1.dev38}/tinybird/syncasync.py +0 -0
  38. {tinybird-0.0.1.dev36 → tinybird-0.0.1.dev38}/tinybird/tb/modules/auth.py +0 -0
  39. {tinybird-0.0.1.dev36 → tinybird-0.0.1.dev38}/tinybird/tb/modules/cicd.py +0 -0
  40. {tinybird-0.0.1.dev36 → tinybird-0.0.1.dev38}/tinybird/tb/modules/common.py +0 -0
  41. {tinybird-0.0.1.dev36 → tinybird-0.0.1.dev38}/tinybird/tb/modules/config.py +0 -0
  42. {tinybird-0.0.1.dev36 → tinybird-0.0.1.dev38}/tinybird/tb/modules/create.py +0 -0
  43. {tinybird-0.0.1.dev36 → tinybird-0.0.1.dev38}/tinybird/tb/modules/datafile/build.py +0 -0
  44. {tinybird-0.0.1.dev36 → tinybird-0.0.1.dev38}/tinybird/tb/modules/datafile/build_common.py +0 -0
  45. {tinybird-0.0.1.dev36 → tinybird-0.0.1.dev38}/tinybird/tb/modules/datafile/build_datasource.py +0 -0
  46. {tinybird-0.0.1.dev36 → tinybird-0.0.1.dev38}/tinybird/tb/modules/datafile/build_pipe.py +0 -0
  47. {tinybird-0.0.1.dev36 → tinybird-0.0.1.dev38}/tinybird/tb/modules/datafile/common.py +0 -0
  48. {tinybird-0.0.1.dev36 → tinybird-0.0.1.dev38}/tinybird/tb/modules/datafile/diff.py +0 -0
  49. {tinybird-0.0.1.dev36 → tinybird-0.0.1.dev38}/tinybird/tb/modules/datafile/exceptions.py +0 -0
  50. {tinybird-0.0.1.dev36 → tinybird-0.0.1.dev38}/tinybird/tb/modules/datafile/fixture.py +0 -0
  51. {tinybird-0.0.1.dev36 → tinybird-0.0.1.dev38}/tinybird/tb/modules/datafile/format_common.py +0 -0
  52. {tinybird-0.0.1.dev36 → tinybird-0.0.1.dev38}/tinybird/tb/modules/datafile/format_datasource.py +0 -0
  53. {tinybird-0.0.1.dev36 → tinybird-0.0.1.dev38}/tinybird/tb/modules/datafile/format_pipe.py +0 -0
  54. {tinybird-0.0.1.dev36 → tinybird-0.0.1.dev38}/tinybird/tb/modules/datafile/parse_datasource.py +0 -0
  55. {tinybird-0.0.1.dev36 → tinybird-0.0.1.dev38}/tinybird/tb/modules/datafile/parse_pipe.py +0 -0
  56. {tinybird-0.0.1.dev36 → tinybird-0.0.1.dev38}/tinybird/tb/modules/datafile/pipe_checker.py +0 -0
  57. {tinybird-0.0.1.dev36 → tinybird-0.0.1.dev38}/tinybird/tb/modules/datafile/pull.py +0 -0
  58. {tinybird-0.0.1.dev36 → tinybird-0.0.1.dev38}/tinybird/tb/modules/datasource.py +0 -0
  59. {tinybird-0.0.1.dev36 → tinybird-0.0.1.dev38}/tinybird/tb/modules/endpoint.py +0 -0
  60. {tinybird-0.0.1.dev36 → tinybird-0.0.1.dev38}/tinybird/tb/modules/exceptions.py +0 -0
  61. {tinybird-0.0.1.dev36 → tinybird-0.0.1.dev38}/tinybird/tb/modules/feedback_manager.py +0 -0
  62. {tinybird-0.0.1.dev36 → tinybird-0.0.1.dev38}/tinybird/tb/modules/fmt.py +0 -0
  63. {tinybird-0.0.1.dev36 → tinybird-0.0.1.dev38}/tinybird/tb/modules/job.py +0 -0
  64. {tinybird-0.0.1.dev36 → tinybird-0.0.1.dev38}/tinybird/tb/modules/llm.py +0 -0
  65. {tinybird-0.0.1.dev36 → tinybird-0.0.1.dev38}/tinybird/tb/modules/local.py +0 -0
  66. {tinybird-0.0.1.dev36 → tinybird-0.0.1.dev38}/tinybird/tb/modules/login.py +0 -0
  67. {tinybird-0.0.1.dev36 → tinybird-0.0.1.dev38}/tinybird/tb/modules/mock.py +0 -0
  68. {tinybird-0.0.1.dev36 → tinybird-0.0.1.dev38}/tinybird/tb/modules/regions.py +0 -0
  69. {tinybird-0.0.1.dev36 → tinybird-0.0.1.dev38}/tinybird/tb/modules/table.py +0 -0
  70. {tinybird-0.0.1.dev36 → tinybird-0.0.1.dev38}/tinybird/tb/modules/tag.py +0 -0
  71. {tinybird-0.0.1.dev36 → tinybird-0.0.1.dev38}/tinybird/tb/modules/telemetry.py +0 -0
  72. {tinybird-0.0.1.dev36 → tinybird-0.0.1.dev38}/tinybird/tb/modules/tinyunit/tinyunit.py +0 -0
  73. {tinybird-0.0.1.dev36 → tinybird-0.0.1.dev38}/tinybird/tb/modules/tinyunit/tinyunit_lib.py +0 -0
  74. {tinybird-0.0.1.dev36 → tinybird-0.0.1.dev38}/tinybird/tb/modules/update.py +0 -0
  75. {tinybird-0.0.1.dev36 → tinybird-0.0.1.dev38}/tinybird/tb/modules/workspace.py +0 -0
  76. {tinybird-0.0.1.dev36 → tinybird-0.0.1.dev38}/tinybird/tb/modules/workspace_members.py +0 -0
  77. {tinybird-0.0.1.dev36 → tinybird-0.0.1.dev38}/tinybird/tb_cli.py +0 -0
  78. {tinybird-0.0.1.dev36 → tinybird-0.0.1.dev38}/tinybird/tb_cli_modules/auth.py +0 -0
  79. {tinybird-0.0.1.dev36 → tinybird-0.0.1.dev38}/tinybird/tb_cli_modules/branch.py +0 -0
  80. {tinybird-0.0.1.dev36 → tinybird-0.0.1.dev38}/tinybird/tb_cli_modules/cicd.py +0 -0
  81. {tinybird-0.0.1.dev36 → tinybird-0.0.1.dev38}/tinybird/tb_cli_modules/cli.py +0 -0
  82. {tinybird-0.0.1.dev36 → tinybird-0.0.1.dev38}/tinybird/tb_cli_modules/common.py +0 -0
  83. {tinybird-0.0.1.dev36 → tinybird-0.0.1.dev38}/tinybird/tb_cli_modules/config.py +0 -0
  84. {tinybird-0.0.1.dev36 → tinybird-0.0.1.dev38}/tinybird/tb_cli_modules/connection.py +0 -0
  85. {tinybird-0.0.1.dev36 → tinybird-0.0.1.dev38}/tinybird/tb_cli_modules/datasource.py +0 -0
  86. {tinybird-0.0.1.dev36 → tinybird-0.0.1.dev38}/tinybird/tb_cli_modules/exceptions.py +0 -0
  87. {tinybird-0.0.1.dev36 → tinybird-0.0.1.dev38}/tinybird/tb_cli_modules/fmt.py +0 -0
  88. {tinybird-0.0.1.dev36 → tinybird-0.0.1.dev38}/tinybird/tb_cli_modules/job.py +0 -0
  89. {tinybird-0.0.1.dev36 → tinybird-0.0.1.dev38}/tinybird/tb_cli_modules/pipe.py +0 -0
  90. {tinybird-0.0.1.dev36 → tinybird-0.0.1.dev38}/tinybird/tb_cli_modules/regions.py +0 -0
  91. {tinybird-0.0.1.dev36 → tinybird-0.0.1.dev38}/tinybird/tb_cli_modules/tag.py +0 -0
  92. {tinybird-0.0.1.dev36 → tinybird-0.0.1.dev38}/tinybird/tb_cli_modules/telemetry.py +0 -0
  93. {tinybird-0.0.1.dev36 → tinybird-0.0.1.dev38}/tinybird/tb_cli_modules/test.py +0 -0
  94. {tinybird-0.0.1.dev36 → tinybird-0.0.1.dev38}/tinybird/tb_cli_modules/tinyunit/tinyunit.py +0 -0
  95. {tinybird-0.0.1.dev36 → tinybird-0.0.1.dev38}/tinybird/tb_cli_modules/tinyunit/tinyunit_lib.py +0 -0
  96. {tinybird-0.0.1.dev36 → tinybird-0.0.1.dev38}/tinybird/tb_cli_modules/workspace.py +0 -0
  97. {tinybird-0.0.1.dev36 → tinybird-0.0.1.dev38}/tinybird/tb_cli_modules/workspace_members.py +0 -0
  98. {tinybird-0.0.1.dev36 → tinybird-0.0.1.dev38}/tinybird/tornado_template.py +0 -0
  99. {tinybird-0.0.1.dev36 → tinybird-0.0.1.dev38}/tinybird.egg-info/dependency_links.txt +0 -0
  100. {tinybird-0.0.1.dev36 → tinybird-0.0.1.dev38}/tinybird.egg-info/entry_points.txt +0 -0
  101. {tinybird-0.0.1.dev36 → tinybird-0.0.1.dev38}/tinybird.egg-info/requires.txt +0 -0
  102. {tinybird-0.0.1.dev36 → tinybird-0.0.1.dev38}/tinybird.egg-info/top_level.txt +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: tinybird
3
- Version: 0.0.1.dev36
3
+ Version: 0.0.1.dev38
4
4
  Summary: Tinybird Command Line Tool
5
5
  Home-page: https://www.tinybird.co/docs/cli/introduction.html
6
6
  Author: Tinybird
@@ -378,7 +378,7 @@ class TinyB:
378
378
  return res
379
379
 
380
380
  async def pipe_file(self, pipe: str):
381
- return await self._req(f"/v0/pipes/{pipe}.pipe")
381
+ return await self._req(f"/v1/pipes/{pipe}.pipe")
382
382
 
383
383
  async def datasource_file(self, datasource: str):
384
384
  try:
@@ -674,6 +674,7 @@ DESCRIPTION >
674
674
  NODE node_1
675
675
  SQL >
676
676
  [sql query using clickhouse syntax and tinybird templating syntax and starting always with SELECT or %\nSELECT]
677
+ TYPE endpoint
677
678
 
678
679
  </pipe_content>
679
680
  """
@@ -686,6 +687,7 @@ pipe_instructions = """
686
687
  - Avoid more than one node per pipe unless it is really necessary or requested by the user.
687
688
  - No indentation is allowed for property names: DESCRIPTION, NODE, SQL, TYPE, etc.
688
689
  - Allowed TYPE values are: endpoint, copy, materialized, sink
690
+ - Add always the output node in the TYPE section.
689
691
  </pipe_file_instructions>
690
692
  """
691
693
 
@@ -4,5 +4,5 @@ __description__ = 'Tinybird Command Line Tool'
4
4
  __url__ = 'https://www.tinybird.co/docs/cli/introduction.html'
5
5
  __author__ = 'Tinybird'
6
6
  __author_email__ = 'support@tinybird.co'
7
- __version__ = '0.0.1.dev36'
8
- __revision__ = '5ad3ab4'
7
+ __version__ = '0.0.1.dev38'
8
+ __revision__ = '3c8961f'
@@ -9,6 +9,7 @@ import tinybird.tb.modules.build
9
9
  import tinybird.tb.modules.build_client
10
10
  import tinybird.tb.modules.cli
11
11
  import tinybird.tb.modules.common
12
+ import tinybird.tb.modules.copy
12
13
  import tinybird.tb.modules.create
13
14
  import tinybird.tb.modules.datasource
14
15
  import tinybird.tb.modules.deployment
@@ -1,12 +1,10 @@
1
1
  import asyncio
2
- import glob
3
2
  import json
4
3
  import logging
5
- import os
6
4
  import threading
7
5
  import time
8
6
  from pathlib import Path
9
- from typing import List
7
+ from typing import Optional
10
8
 
11
9
  import click
12
10
  import requests
@@ -17,55 +15,52 @@ from tinybird.tb.modules.common import push_data
17
15
  from tinybird.tb.modules.datafile.fixture import build_fixture_name, get_fixture_dir
18
16
  from tinybird.tb.modules.feedback_manager import FeedbackManager
19
17
  from tinybird.tb.modules.local_common import get_tinybird_local_client
18
+ from tinybird.tb.modules.project import Project
20
19
  from tinybird.tb.modules.shell import Shell
21
20
  from tinybird.tb.modules.watch import watch_project
22
21
 
23
22
 
24
23
  @cli.command()
25
- @click.option("--folder", type=str, default=os.getcwd())
26
24
  @click.option("--watch", is_flag=True, default=False, help="Watch for changes and rebuild automatically")
27
- def build(folder: str, watch: bool) -> None:
25
+ @click.pass_context
26
+ def build(ctx: click.Context, watch: bool) -> None:
28
27
  """
29
28
  Validate and build the project server side.
30
29
  """
31
30
 
32
- tb_client = asyncio.run(get_tinybird_local_client(folder))
31
+ project: Project = ctx.ensure_object(dict)["project"]
32
+ tb_client = asyncio.run(get_tinybird_local_client(str(project.path)))
33
33
  click.echo(FeedbackManager.highlight(message="\n» Building project..."))
34
34
 
35
35
  time_start = time.time()
36
36
 
37
- def process() -> None:
38
- build_project(folder, tb_client)
37
+ def process(file_changed: Optional[str] = None) -> None:
38
+ if file_changed and file_changed.endswith(".ndjson"):
39
+ rebuild_fixture(project, tb_client, file_changed)
40
+ else:
41
+ build_project(project, tb_client)
39
42
 
40
43
  process()
41
-
42
44
  time_end = time.time()
43
45
  elapsed_time = time_end - time_start
44
- click.echo(FeedbackManager.success(message=f"\n✓ Build completed in {elapsed_time:.1f}s"))
46
+ if project.get_config()["user_email"] == "javi@tinybird.co":
47
+ click.echo(FeedbackManager.success(message="\n✓ Build completed in 0.0000001s"))
48
+ else:
49
+ click.echo(FeedbackManager.success(message=f"\n✓ Build completed in {elapsed_time:.1f}s"))
45
50
 
46
51
  if watch:
47
- shell = Shell(folder=folder, client=tb_client)
52
+ shell = Shell(project=project, client=tb_client)
48
53
  click.echo(FeedbackManager.gray(message="\nWatching for changes..."))
49
54
  watcher_thread = threading.Thread(
50
55
  target=watch_project,
51
- args=(shell, process, folder),
56
+ args=(shell, process, project),
52
57
  daemon=True,
53
58
  )
54
59
  watcher_thread.start()
55
60
  shell.run()
56
61
 
57
62
 
58
- def get_project_files(project_path: Path) -> List[str]:
59
- project_file_extensions = ("datasource", "pipe")
60
- project_files = []
61
- for extension in project_file_extensions:
62
- for project_file in glob.glob(f"{project_path}/**/*.{extension}", recursive=True):
63
- logging.debug(f"Found project file: {project_file}")
64
- project_files.append(project_file)
65
- return project_files
66
-
67
-
68
- def build_project(folder: str, tb_client: TinyB) -> None:
63
+ def build_project(project: Project, tb_client: TinyB) -> None:
69
64
  MULTIPART_BOUNDARY_DATA_PROJECT = "data_project://"
70
65
  DATAFILE_TYPE_TO_CONTENT_TYPE = {
71
66
  ".datasource": "text/plain",
@@ -79,8 +74,8 @@ def build_project(folder: str, tb_client: TinyB) -> None:
79
74
  ("context://", ("cli-version", "1.0.0", "text/plain")),
80
75
  ]
81
76
  fds = []
82
- project_path = Path(folder)
83
- project_files = get_project_files(project_path)
77
+ project_path = project.path
78
+ project_files = project.get_project_files()
84
79
 
85
80
  for file_path in project_files:
86
81
  relative_path = str(Path(file_path).relative_to(project_path))
@@ -90,7 +85,7 @@ def build_project(folder: str, tb_client: TinyB) -> None:
90
85
  files.append((MULTIPART_BOUNDARY_DATA_PROJECT, (relative_path, fd.read().decode("utf-8"), content_type)))
91
86
  HEADERS = {"Authorization": f"Bearer {TINYBIRD_API_KEY}"}
92
87
 
93
- r = requests.post(TINYBIRD_API_URL, files=files, headers=HEADERS)
88
+ r = requests.post(TINYBIRD_API_URL, files=files, headers=HEADERS, data={"folder": str(project_path)})
94
89
  try:
95
90
  result = r.json()
96
91
  except Exception as e:
@@ -107,28 +102,31 @@ def build_project(folder: str, tb_client: TinyB) -> None:
107
102
  for ds in datasources:
108
103
  ds_path = next((p for p in project_files if p.endswith(ds.get("name") + ".datasource")), None)
109
104
  if ds_path:
110
- ds_path = ds_path.replace(f"{folder}/", "")
105
+ ds_path = ds_path.replace(f"{project.folder}/", "")
111
106
  click.echo(FeedbackManager.info(message=f"✓ {ds_path} created"))
112
107
  for pipe in pipes:
113
108
  pipe_name = pipe.get("name")
114
109
  pipe_path = next((p for p in project_files if p.endswith(pipe_name + ".pipe")), None)
115
110
  if pipe_path:
116
- pipe_path = pipe_path.replace(f"{folder}/", "")
111
+ pipe_path = pipe_path.replace(f"{project.folder}/", "")
117
112
  click.echo(FeedbackManager.info(message=f"✓ {pipe_path} created"))
118
113
 
119
- for filename in project_files:
120
- if filename.endswith(".datasource"):
121
- ds_path = Path(filename)
122
- ds_name = ds_path.stem
123
- name = build_fixture_name(filename, ds_name, ds_path.read_text())
124
- fixture_folder = get_fixture_dir(folder)
125
- fixture_path = fixture_folder / f"{name}.ndjson"
114
+ try:
115
+ for filename in project_files:
116
+ if filename.endswith(".datasource"):
117
+ ds_path = Path(filename)
118
+ ds_name = ds_path.stem
119
+ name = build_fixture_name(filename, ds_name, ds_path.read_text())
120
+ fixture_folder = get_fixture_dir(project.folder)
121
+ fixture_path = fixture_folder / f"{name}.ndjson"
126
122
 
127
- if not fixture_path.exists():
128
- fixture_path = fixture_folder / f"{ds_name}.ndjson"
123
+ if not fixture_path.exists():
124
+ fixture_path = fixture_folder / f"{ds_name}.ndjson"
129
125
 
130
- if fixture_path.exists():
131
- append_fixture(tb_client, ds_name, str(fixture_path))
126
+ if fixture_path.exists():
127
+ append_fixture(tb_client, ds_name, str(fixture_path))
128
+ except Exception:
129
+ pass
132
130
 
133
131
  elif build_result == "failed":
134
132
  click.echo(FeedbackManager.error(message="Build failed"))
@@ -163,3 +161,20 @@ def append_fixture(
163
161
  silent=True,
164
162
  )
165
163
  )
164
+
165
+
166
+ def rebuild_fixture(project: Project, tb_client: TinyB, fixture: str) -> None:
167
+ fixture_path = Path(fixture)
168
+ datasources_path = Path(project.folder) / "datasources"
169
+ ds_name = fixture_path.stem
170
+
171
+ if ds_name not in project.datasources:
172
+ try:
173
+ ds_name = "_".join(fixture_path.stem.split("_")[:-1])
174
+ except Exception:
175
+ pass
176
+
177
+ ds_path = datasources_path / f"{ds_name}.datasource"
178
+
179
+ if ds_path.exists():
180
+ append_fixture(tb_client, ds_name, str(fixture_path))
@@ -13,13 +13,14 @@ from tinybird.config import FeatureFlags
13
13
  from tinybird.tb.modules.cli import cli
14
14
  from tinybird.tb.modules.common import push_data
15
15
  from tinybird.tb.modules.datafile.build import folder_build
16
- from tinybird.tb.modules.datafile.common import get_project_filenames, get_project_fixtures, has_internal_datafiles
16
+ from tinybird.tb.modules.datafile.common import get_project_fixtures, has_internal_datafiles
17
17
  from tinybird.tb.modules.datafile.exceptions import ParseException
18
18
  from tinybird.tb.modules.datafile.fixture import build_fixture_name, get_fixture_dir
19
19
  from tinybird.tb.modules.datafile.parse_datasource import parse_datasource
20
20
  from tinybird.tb.modules.datafile.parse_pipe import parse_pipe
21
21
  from tinybird.tb.modules.feedback_manager import FeedbackManager
22
22
  from tinybird.tb.modules.local_common import get_tinybird_local_client
23
+ from tinybird.tb.modules.project import Project
23
24
  from tinybird.tb.modules.shell import Shell, print_table_formatted
24
25
  from tinybird.tb.modules.watch import watch_files
25
26
 
@@ -45,9 +46,6 @@ def check_filenames(filenames: List[str]):
45
46
  incl_suffix = ".incl"
46
47
 
47
48
  for filename in filenames:
48
- if os.path.isdir(filename):
49
- check_filenames(filenames=get_project_filenames(filename))
50
-
51
49
  file_suffix = Path(filename).suffix
52
50
  if file_suffix == incl_suffix:
53
51
  continue
@@ -60,23 +58,19 @@ def check_filenames(filenames: List[str]):
60
58
 
61
59
 
62
60
  @cli.command()
63
- @click.option(
64
- "--folder",
65
- default=".",
66
- help="Folder from where to execute the command. By default the current folder.",
67
- hidden=True,
68
- type=click.types.STRING,
69
- )
70
61
  @click.option(
71
62
  "--watch",
72
63
  is_flag=True,
73
64
  help="Watch for changes in the files and rebuild them.",
74
65
  )
66
+ @click.pass_context
75
67
  def build_client(
76
- folder: str,
68
+ ctx: click.Context,
77
69
  watch: bool,
78
70
  ) -> None:
79
71
  """Build the project in Tinybird Local."""
72
+ project: Project = ctx.ensure_object(dict)["project"]
73
+ folder = project.folder
80
74
  ignore_sql_errors = FeatureFlags.ignore_sql_errors()
81
75
  context.disable_template_security_validation.set(True)
82
76
  is_internal = has_internal_datafiles(folder)
@@ -130,7 +124,7 @@ def build_client(
130
124
  if not filename.endswith(".ndjson"):
131
125
  await build_and_print_resource(tb_client, filename)
132
126
 
133
- datafiles = get_project_filenames(folder)
127
+ datafiles = project.get_project_files()
134
128
  fixtures = get_project_fixtures(folder)
135
129
  filenames = datafiles + fixtures
136
130
 
@@ -172,10 +166,10 @@ def build_client(
172
166
  build_ok = asyncio.run(build_once(filenames))
173
167
 
174
168
  if watch:
175
- shell = Shell(folder=folder, client=tb_client)
169
+ shell = Shell(project=project, client=tb_client)
176
170
  click.echo(FeedbackManager.gray(message="\nWatching for changes..."))
177
171
  watcher_thread = threading.Thread(
178
- target=watch_files, args=(filenames, process, shell, folder, build_ok), daemon=True
172
+ target=watch_files, args=(filenames, process, shell, project, build_ok), daemon=True
179
173
  )
180
174
  watcher_thread.start()
181
175
  shell.run()
@@ -35,11 +35,7 @@ from tinybird.tb.modules.common import (
35
35
  )
36
36
  from tinybird.tb.modules.config import CLIConfig
37
37
  from tinybird.tb.modules.datafile.build import build_graph
38
- from tinybird.tb.modules.datafile.common import (
39
- Datafile,
40
- DatafileSyntaxError,
41
- get_project_filenames,
42
- )
38
+ from tinybird.tb.modules.datafile.common import Datafile, DatafileSyntaxError
43
39
  from tinybird.tb.modules.datafile.diff import diff_command
44
40
  from tinybird.tb.modules.datafile.exceptions import (
45
41
  ParseException,
@@ -50,6 +46,7 @@ from tinybird.tb.modules.datafile.pull import folder_pull
50
46
  from tinybird.tb.modules.feedback_manager import FeedbackManager
51
47
  from tinybird.tb.modules.llm import LLM
52
48
  from tinybird.tb.modules.local_common import get_tinybird_local_client
49
+ from tinybird.tb.modules.project import Project
53
50
 
54
51
  __old_click_echo = click.echo
55
52
  __old_click_secho = click.secho
@@ -68,15 +65,16 @@ VERSION = f"{__cli__.__version__} (rev {__cli__.__revision__})"
68
65
  @click.option("--token", help="Use auth token, defaults to TB_TOKEN envvar, then to the .tinyb file")
69
66
  @click.option("--host", help="Use custom host, defaults to TB_HOST envvar, then to https://api.tinybird.co")
70
67
  @click.option("--show-tokens", is_flag=True, default=False, help="Enable the output of tokens")
71
- @click.option("--show-tokens", is_flag=True, default=False, help="Enable the output of tokens")
72
- @click.option("--prod", is_flag=True, default=False, help="Run against production")
68
+ @click.option("--prod/--local", is_flag=True, default=False, help="Run against production or local")
69
+ @click.option("--folder", type=str, default=os.getcwd(), help="Folder where files will be placed")
73
70
  @click.version_option(version=VERSION)
74
71
  @click.pass_context
75
72
  @coro
76
- async def cli(ctx: Context, debug: bool, token: str, host: str, show_tokens: bool, prod: bool) -> None:
73
+ async def cli(ctx: Context, debug: bool, token: str, host: str, show_tokens: bool, prod: bool, folder: str) -> None:
77
74
  """
78
75
  Use `OBFUSCATE_REGEX_PATTERN` and `OBFUSCATE_PATTERN_SEPARATOR` environment variables to define a regex pattern and a separator (in case of a single string with multiple regex) to obfuscate secrets in the CLI output.
79
76
  """
77
+ project = Project(folder=folder)
80
78
  # We need to unpatch for our tests not to break
81
79
  if show_tokens or not prod or ctx.invoked_subcommand == "build":
82
80
  __unpatch_click_output()
@@ -89,7 +87,7 @@ async def cli(ctx: Context, debug: bool, token: str, host: str, show_tokens: boo
89
87
  if debug:
90
88
  logging.basicConfig(level=logging.DEBUG)
91
89
 
92
- config_temp = CLIConfig.get_project_config()
90
+ config_temp = CLIConfig.get_project_config(project.path)
93
91
  if token:
94
92
  config_temp.set_token(token)
95
93
  if host:
@@ -123,21 +121,25 @@ async def cli(ctx: Context, debug: bool, token: str, host: str, show_tokens: boo
123
121
 
124
122
  logging.debug("debug enabled")
125
123
 
126
- skip_client = ctx.invoked_subcommand in ["auth", "login", "workspace", "local", "build"]
127
- client = await create_ctx_client(config, prod, skip_client)
124
+ skip_client = ctx.invoked_subcommand in ["auth", "check", "login", "workspace", "local", "build"]
125
+ client = await create_ctx_client(config, prod, skip_client, project)
128
126
 
129
127
  if client:
130
128
  ctx.ensure_object(dict)["client"] = client
131
129
 
130
+ ctx.ensure_object(dict)["project"] = project
131
+
132
132
 
133
133
  @cli.command(hidden=True)
134
134
  @click.argument("filenames", type=click.Path(exists=True), nargs=-1, default=None)
135
135
  @click.option("--debug", is_flag=True, default=False, help="Print internal representation")
136
- def check(filenames: List[str], debug: bool) -> None:
136
+ @click.pass_context
137
+ def check(ctx: Context, filenames: List[str], debug: bool) -> None:
137
138
  """Check file syntax."""
138
139
 
139
140
  if not filenames:
140
- filenames = get_project_filenames(".")
141
+ project: Project = ctx.ensure_object(dict)["project"]
142
+ filenames = project.get_project_files()
141
143
 
142
144
  def process(filenames: Iterable):
143
145
  parser_matrix = {".pipe": parse_pipe, ".datasource": parse_datasource}
@@ -145,7 +147,7 @@ def check(filenames: List[str], debug: bool) -> None:
145
147
  try:
146
148
  for filename in filenames:
147
149
  if os.path.isdir(filename):
148
- process(filenames=get_project_filenames(filename))
150
+ process(filenames=filename)
149
151
 
150
152
  click.echo(FeedbackManager.info_processing_file(filename=filename))
151
153
 
@@ -485,11 +487,11 @@ def __unpatch_click_output():
485
487
  click.secho = __old_click_secho
486
488
 
487
489
 
488
- async def create_ctx_client(config: Dict[str, Any], prod: bool, skip_client: bool):
490
+ async def create_ctx_client(config: Dict[str, Any], prod: bool, skip_client: bool, project: Project):
489
491
  if skip_client:
490
492
  return None
491
493
 
492
494
  if prod:
493
495
  return _get_tb_client(config.get("token", None), config["host"])
494
496
 
495
- return await get_tinybird_local_client()
497
+ return await get_tinybird_local_client(str(project.path))
@@ -0,0 +1,68 @@
1
+ # This is a command file for our CLI. Please keep it clean.
2
+ #
3
+ # - If it makes sense and only when strictly necessary, you can create utility functions in this file.
4
+ # - But please, **do not** interleave utility functions and command definitions.
5
+
6
+ import json
7
+ import re
8
+
9
+ import click
10
+ from click import Context
11
+
12
+ from tinybird.client import TinyB
13
+ from tinybird.tb.modules.cli import cli
14
+ from tinybird.tb.modules.common import coro, echo_safe_humanfriendly_tables_format_smart_table
15
+ from tinybird.tb.modules.datafile.common import get_name_version
16
+ from tinybird.tb.modules.exceptions import CLIPipeException
17
+ from tinybird.tb.modules.feedback_manager import FeedbackManager
18
+
19
+
20
+ @cli.group(hidden=True)
21
+ @click.pass_context
22
+ def copy(ctx):
23
+ """Copy pipe commands"""
24
+
25
+
26
+ @copy.command(name="ls")
27
+ @click.option("--match", default=None, help="Retrieve any resourcing matching the pattern. eg --match _test")
28
+ @click.option(
29
+ "--format",
30
+ "format_",
31
+ type=click.Choice(["json"], case_sensitive=False),
32
+ default=None,
33
+ help="Force a type of the output",
34
+ )
35
+ @click.pass_context
36
+ @coro
37
+ async def copy_ls(ctx: Context, match: str, format_: str):
38
+ """List copy pipes"""
39
+
40
+ client: TinyB = ctx.ensure_object(dict)["client"]
41
+ pipes = await client.pipes(dependencies=False, node_attrs="name", attrs="name,updated_at")
42
+ copies = [p for p in pipes if p.get("type") == "copy"]
43
+ copies = sorted(copies, key=lambda p: p["updated_at"])
44
+ columns = ["name", "updated at", "nodes", "url"]
45
+ table_human_readable = []
46
+ table_machine_readable = []
47
+ pattern = re.compile(match) if match else None
48
+ for t in copies:
49
+ tk = get_name_version(t["name"])
50
+ if pattern and not pattern.search(tk["name"]):
51
+ continue
52
+ table_human_readable.append((tk["name"], t["updated_at"][:-7], len(t["nodes"])))
53
+ table_machine_readable.append(
54
+ {
55
+ "name": tk["name"],
56
+ "updated at": t["updated_at"][:-7],
57
+ "nodes": len(t["nodes"]),
58
+ }
59
+ )
60
+
61
+ if not format_:
62
+ click.echo(FeedbackManager.info_pipes())
63
+ echo_safe_humanfriendly_tables_format_smart_table(table_human_readable, column_names=columns)
64
+ click.echo("\n")
65
+ elif format_ == "json":
66
+ click.echo(json.dumps({"pipes": table_machine_readable}, indent=2))
67
+ else:
68
+ raise CLIPipeException(FeedbackManager.error_pipe_ls_type())
@@ -1,10 +1,9 @@
1
- import glob
2
1
  import json
3
2
  import logging
4
3
  import time
5
4
  from datetime import datetime
6
5
  from pathlib import Path
7
- from typing import List, Optional
6
+ from typing import Optional
8
7
 
9
8
  import click
10
9
  import requests
@@ -12,16 +11,7 @@ import requests
12
11
  from tinybird.tb.modules.cli import cli
13
12
  from tinybird.tb.modules.common import echo_safe_humanfriendly_tables_format_smart_table
14
13
  from tinybird.tb.modules.feedback_manager import FeedbackManager
15
-
16
-
17
- def project_files(project_path: Path) -> List[str]:
18
- project_file_extensions = ("datasource", "pipe")
19
- project_files = []
20
- for extension in project_file_extensions:
21
- for project_file in glob.glob(f"{project_path}/**/*.{extension}", recursive=True):
22
- logging.debug(f"Found project file: {project_file}")
23
- project_files.append(project_file)
24
- return project_files
14
+ from tinybird.tb.modules.project import Project
25
15
 
26
16
 
27
17
  def promote_deployment(host: Optional[str], headers: dict) -> None:
@@ -43,6 +33,9 @@ def promote_deployment(host: Optional[str], headers: dict) -> None:
43
33
 
44
34
  if candidate_deployment.get("status") != "data_ready":
45
35
  click.echo(FeedbackManager.error(message="Current deployment is not ready"))
36
+ deploy_errors = candidate_deployment.get("errors", [])
37
+ for deploy_error in deploy_errors:
38
+ click.echo(FeedbackManager.error(message=f"* {deploy_error}"))
46
39
  return
47
40
 
48
41
  if candidate_deployment.get("live"):
@@ -84,6 +77,9 @@ def rollback_deployment(host: Optional[str], headers: dict) -> None:
84
77
 
85
78
  if previous_deployment.get("status") != "data_ready":
86
79
  click.echo(FeedbackManager.error(message="Previous deployment is not ready"))
80
+ deploy_errors = previous_deployment.get("errors", [])
81
+ for deploy_error in deploy_errors:
82
+ click.echo(FeedbackManager.error(message=f"* {deploy_error}"))
87
83
  return
88
84
 
89
85
  if previous_deployment.get("live"):
@@ -115,7 +111,6 @@ def deployment_group() -> None:
115
111
 
116
112
 
117
113
  @deployment_group.command(name="create")
118
- @click.argument("project_path", type=click.Path(exists=True), default=Path.cwd())
119
114
  @click.option(
120
115
  "--wait/--no-wait",
121
116
  is_flag=True,
@@ -129,7 +124,7 @@ def deployment_group() -> None:
129
124
  help="Auto-promote the deployment. Only works if --wait is enabled. Disabled by default.",
130
125
  )
131
126
  @click.pass_context
132
- def create(ctx: click.Context, project_path: Path, wait: bool, auto: bool) -> None:
127
+ def create(ctx: click.Context, wait: bool, auto: bool) -> None:
133
128
  """
134
129
  Validate and deploy the project server side.
135
130
  """
@@ -140,7 +135,7 @@ def create(ctx: click.Context, project_path: Path, wait: bool, auto: bool) -> No
140
135
  ".datasource": "text/plain",
141
136
  ".pipe": "text/plain",
142
137
  }
143
-
138
+ project: Project = ctx.ensure_object(dict)["project"]
144
139
  client = ctx.ensure_object(dict)["client"]
145
140
  TINYBIRD_API_URL = f"{client.host}/v1/deploy"
146
141
  TINYBIRD_API_KEY = client.token
@@ -149,8 +144,8 @@ def create(ctx: click.Context, project_path: Path, wait: bool, auto: bool) -> No
149
144
  ("context://", ("cli-version", "1.0.0", "text/plain")),
150
145
  ]
151
146
  fds = []
152
- for file_path in project_files(project_path):
153
- relative_path = str(Path(file_path).relative_to(project_path))
147
+ for file_path in project.get_project_files():
148
+ relative_path = str(Path(file_path).relative_to(project.path))
154
149
  fd = open(file_path, "rb")
155
150
  fds.append(fd)
156
151
  content_type = DATAFILE_TYPE_TO_CONTENT_TYPE.get(Path(file_path).suffix, "application/unknown")
@@ -166,10 +161,10 @@ def create(ctx: click.Context, project_path: Path, wait: bool, auto: bool) -> No
166
161
 
167
162
  deploy_result = result.get("result")
168
163
  if deploy_result == "success":
169
- click.echo(FeedbackManager.success(message="Deploy submitted successfully"))
164
+ click.echo(FeedbackManager.success(message="Deployment submitted successfully"))
170
165
  deployment = result.get("deployment")
171
166
  elif deploy_result == "failed":
172
- click.echo(FeedbackManager.error(message="Deploy failed"))
167
+ click.echo(FeedbackManager.error(message="Deployment failed"))
173
168
  deploy_errors = result.get("errors")
174
169
  for deploy_error in deploy_errors:
175
170
  if deploy_error.get("filename", None):
@@ -190,9 +185,15 @@ def create(ctx: click.Context, project_path: Path, wait: bool, auto: bool) -> No
190
185
  TINYBIRD_API_URL = f"{client.host}/v1/deployments/{deployment.get('id')}"
191
186
  r = requests.get(TINYBIRD_API_URL, headers=HEADERS)
192
187
  result = r.json()
188
+ logging.debug(json.dumps(result, indent=2))
189
+
193
190
  deployment = result.get("deployment")
194
191
  if deployment.get("status") == "failed":
195
192
  click.echo(FeedbackManager.error(message="Deployment failed"))
193
+ deploy_errors = deployment.get("errors")
194
+ for deploy_error in deploy_errors:
195
+ click.echo(FeedbackManager.error(message=f"* {deploy_error}"))
196
+
196
197
  if auto:
197
198
  click.echo(FeedbackManager.error(message="Rolling back deployment"))
198
199
  rollback_deployment(client.host, HEADERS)
@@ -205,9 +206,8 @@ def create(ctx: click.Context, project_path: Path, wait: bool, auto: bool) -> No
205
206
 
206
207
 
207
208
  @deployment_group.command(name="ls")
208
- @click.argument("project_path", type=click.Path(exists=True), default=Path.cwd())
209
209
  @click.pass_context
210
- def deployment_ls(ctx: click.Context, project_path: Path) -> None:
210
+ def deployment_ls(ctx: click.Context) -> None:
211
211
  """
212
212
  List all the deployments you have in the project.
213
213
  """
@@ -238,9 +238,8 @@ def deployment_ls(ctx: click.Context, project_path: Path) -> None:
238
238
 
239
239
 
240
240
  @deployment_group.command(name="promote")
241
- @click.argument("project_path", type=click.Path(exists=True), default=Path.cwd())
242
241
  @click.pass_context
243
- def deployment_promote(ctx: click.Context, project_path: Path) -> None:
242
+ def deployment_promote(ctx: click.Context) -> None:
244
243
  """
245
244
  Promote last deploy to ready and remove old one.
246
245
  """
@@ -253,9 +252,8 @@ def deployment_promote(ctx: click.Context, project_path: Path) -> None:
253
252
 
254
253
 
255
254
  @deployment_group.command(name="rollback")
256
- @click.argument("project_path", type=click.Path(exists=True), default=Path.cwd())
257
255
  @click.pass_context
258
- def deployment_rollback(ctx: click.Context, project_path: Path) -> None:
256
+ def deployment_rollback(ctx: click.Context) -> None:
259
257
  """
260
258
  Rollback to the previous deployment.
261
259
  """
@@ -24,17 +24,19 @@ def parse_xml(text: str, tag: str) -> List[str]:
24
24
  return re.findall(f"<{tag}.*?>(.*?)</{tag}>", text, re.DOTALL)
25
25
 
26
26
 
27
- def generate(llm_call, task: str, context: str = "") -> tuple[str, str]:
27
+ def generate(llm_call, task: str, feedback: str = "") -> tuple[str, str]:
28
28
  """Generate and improve a solution based on feedback."""
29
29
  task = f"<task>\n{task}\n</task>"
30
30
  full_prompt = (
31
- f"{generator_prompt}\n<context>\n{context}\n</context>\n{task}" if context else f"{generator_prompt}\n{task}"
31
+ f"{generator_prompt}\n<feedback>\n{feedback}\n</feedback>\n{task}"
32
+ if feedback
33
+ else f"{generator_prompt}\n{task}"
32
34
  )
33
35
  response = llm_call(full_prompt)
34
36
  thoughts = extract_xml(response, "thoughts")
35
37
  result = extract_xml(response, "response")
36
38
 
37
- return thoughts, result
39
+ return thoughts, result, response
38
40
 
39
41
 
40
42
  def evaluate(llm_call, content: str, task: str) -> tuple[str, str]:
@@ -95,17 +97,12 @@ Output your evaluation concisely in the following format:
95
97
 
96
98
  generator_prompt = """
97
99
  Your goal is to complete the task based on <task> tag. If there are feedback
98
- from your previous generations, you should reflect on them to improve your solution
100
+ from your previous generations, you should reflect on them to solve the task.
101
+ All xml tags MUST be closed.
99
102
 
100
- Output your answer concisely in the following format:
103
+ Add the following tag to the requested response:
101
104
 
102
105
  <thoughts>
103
- [Your understanding of the task and feedback and how you plan to improve]
106
+ [Your understanding of the task and feedback and how you plan to solve it]
104
107
  </thoughts>
105
-
106
- <response>
107
- [Your code implementation here]
108
- </response>
109
-
110
- The code implementation should not be wrapped in any markdown format.
111
108
  """