tinybird 0.0.1.dev42__py3-none-any.whl → 0.0.1.dev44__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of tinybird might be problematic. Click here for more details.
- tinybird/client.py +1 -1
- tinybird/connectors.py +3 -3
- tinybird/feedback_manager.py +1 -1
- tinybird/prompts.py +31 -3
- tinybird/sql.py +1 -1
- tinybird/sql_template_fmt.py +1 -1
- tinybird/tb/__cli__.py +2 -2
- tinybird/tb/cli.py +1 -1
- tinybird/tb/modules/build.py +39 -21
- tinybird/tb/modules/cicd.py +2 -2
- tinybird/tb/modules/cli.py +8 -61
- tinybird/tb/modules/common.py +2 -1
- tinybird/tb/modules/copy.py +96 -5
- tinybird/tb/modules/create.py +105 -46
- tinybird/tb/modules/datafile/build.py +64 -247
- tinybird/tb/modules/datasource.py +1 -1
- tinybird/tb/modules/deployment.py +86 -61
- tinybird/tb/modules/endpoint.py +90 -3
- tinybird/tb/modules/llm_utils.py +2 -2
- tinybird/tb/modules/materialization.py +146 -0
- tinybird/tb/modules/mock.py +56 -16
- tinybird/tb/modules/pipe.py +2 -411
- tinybird/tb/modules/project.py +31 -1
- tinybird/tb/modules/test.py +72 -37
- tinybird/tb/modules/update.py +1 -1
- tinybird/tb/modules/watch.py +54 -5
- tinybird/tb_cli_modules/common.py +1 -1
- tinybird/tornado_template.py +2 -2
- {tinybird-0.0.1.dev42.dist-info → tinybird-0.0.1.dev44.dist-info}/METADATA +1 -1
- {tinybird-0.0.1.dev42.dist-info → tinybird-0.0.1.dev44.dist-info}/RECORD +33 -33
- tinybird/tb/modules/build_client.py +0 -199
- {tinybird-0.0.1.dev42.dist-info → tinybird-0.0.1.dev44.dist-info}/WHEEL +0 -0
- {tinybird-0.0.1.dev42.dist-info → tinybird-0.0.1.dev44.dist-info}/entry_points.txt +0 -0
- {tinybird-0.0.1.dev42.dist-info → tinybird-0.0.1.dev44.dist-info}/top_level.txt +0 -0
tinybird/tb/modules/watch.py
CHANGED
|
@@ -15,6 +15,7 @@ from watchdog.events import (
|
|
|
15
15
|
)
|
|
16
16
|
from watchdog.observers import Observer
|
|
17
17
|
|
|
18
|
+
from tinybird.tb.modules.datafile.common import Datafile, DatafileKind
|
|
18
19
|
from tinybird.tb.modules.feedback_manager import FeedbackManager
|
|
19
20
|
from tinybird.tb.modules.project import Project
|
|
20
21
|
from tinybird.tb.modules.shell import Shell
|
|
@@ -108,7 +109,7 @@ def watch_files(
|
|
|
108
109
|
event_handler = FileChangeHandler(filenames, lambda f: asyncio.run(process_wrapper(f)), build_ok)
|
|
109
110
|
observer = Observer()
|
|
110
111
|
|
|
111
|
-
observer.schedule(event_handler, path=project.path, recursive=True)
|
|
112
|
+
observer.schedule(event_handler, path=str(project.path), recursive=True)
|
|
112
113
|
|
|
113
114
|
observer.start()
|
|
114
115
|
|
|
@@ -122,10 +123,16 @@ def watch_files(
|
|
|
122
123
|
|
|
123
124
|
|
|
124
125
|
class WatchProjectHandler(PatternMatchingEventHandler):
|
|
125
|
-
def __init__(
|
|
126
|
+
def __init__(
|
|
127
|
+
self,
|
|
128
|
+
shell: Shell,
|
|
129
|
+
project: Project,
|
|
130
|
+
process: Callable[[Optional[str], Optional[str]], None],
|
|
131
|
+
):
|
|
126
132
|
self.shell = shell
|
|
127
133
|
self.project = project
|
|
128
134
|
self.process = process
|
|
135
|
+
self.datafiles = project.get_project_datafiles()
|
|
129
136
|
super().__init__(
|
|
130
137
|
patterns=[
|
|
131
138
|
f"{project.path}/**/*.datasource",
|
|
@@ -150,7 +157,7 @@ class WatchProjectHandler(PatternMatchingEventHandler):
|
|
|
150
157
|
def _process(self, path: Optional[str] = None) -> None:
|
|
151
158
|
click.echo(FeedbackManager.highlight(message="» Rebuilding project..."))
|
|
152
159
|
time_start = time.time()
|
|
153
|
-
self.process(path)
|
|
160
|
+
self.process(path, self.diff(path))
|
|
154
161
|
time_end = time.time()
|
|
155
162
|
elapsed_time = time_end - time_start
|
|
156
163
|
click.echo(
|
|
@@ -159,6 +166,48 @@ class WatchProjectHandler(PatternMatchingEventHandler):
|
|
|
159
166
|
)
|
|
160
167
|
self.shell.reprint_prompt()
|
|
161
168
|
|
|
169
|
+
def diff(self, path: Optional[str] = None) -> Optional[str]:
|
|
170
|
+
if not path:
|
|
171
|
+
return None
|
|
172
|
+
|
|
173
|
+
current_datafile = self.datafiles.get(path, None)
|
|
174
|
+
new_datafile = self.project.get_datafile(path)
|
|
175
|
+
table_name = None
|
|
176
|
+
if current_datafile and new_datafile:
|
|
177
|
+
if current_datafile.kind == DatafileKind.datasource:
|
|
178
|
+
table_name = self.datasource_diff(current_datafile, new_datafile)
|
|
179
|
+
elif current_datafile.kind == DatafileKind.pipe:
|
|
180
|
+
table_name = self.pipe_diff(current_datafile, new_datafile)
|
|
181
|
+
|
|
182
|
+
self.refresh_datafiles()
|
|
183
|
+
return table_name
|
|
184
|
+
|
|
185
|
+
def refresh_datafiles(self) -> None:
|
|
186
|
+
self.datafiles = self.project.get_project_datafiles()
|
|
187
|
+
|
|
188
|
+
def datasource_diff(self, current_datafile: Datafile, new_datafile: Datafile) -> Optional[str]:
|
|
189
|
+
current_schema = current_datafile.nodes[0].get("schema")
|
|
190
|
+
new_schema = new_datafile.nodes[0].get("schema")
|
|
191
|
+
if current_schema != new_schema:
|
|
192
|
+
return current_datafile.nodes[0].get("name")
|
|
193
|
+
return None
|
|
194
|
+
|
|
195
|
+
def pipe_diff(self, current_datafile: Datafile, new_datafile: Datafile) -> Optional[str]:
|
|
196
|
+
current_nodes = current_datafile.nodes
|
|
197
|
+
current_sql_dict = {node.get("name"): node.get("sql") for node in current_nodes}
|
|
198
|
+
new_nodes = new_datafile.nodes
|
|
199
|
+
new_sql_dict = {node.get("name"): node.get("sql") for node in new_nodes}
|
|
200
|
+
for node in new_sql_dict.keys():
|
|
201
|
+
if node and node not in current_sql_dict:
|
|
202
|
+
return node
|
|
203
|
+
|
|
204
|
+
for node_name, sql in new_sql_dict.items():
|
|
205
|
+
current_sql = current_sql_dict.get(node_name)
|
|
206
|
+
if current_sql and current_sql != sql:
|
|
207
|
+
return node_name
|
|
208
|
+
|
|
209
|
+
return None
|
|
210
|
+
|
|
162
211
|
def on_modified(self, event: Any) -> None:
|
|
163
212
|
if path := self.should_process(event):
|
|
164
213
|
filename = Path(path).name
|
|
@@ -176,12 +225,12 @@ class WatchProjectHandler(PatternMatchingEventHandler):
|
|
|
176
225
|
|
|
177
226
|
def watch_project(
|
|
178
227
|
shell: Shell,
|
|
179
|
-
process: Callable[[Optional[str]], None],
|
|
228
|
+
process: Callable[[Optional[str], Optional[str]], None],
|
|
180
229
|
project: Project,
|
|
181
230
|
) -> None:
|
|
182
231
|
event_handler = WatchProjectHandler(shell=shell, project=project, process=process)
|
|
183
232
|
observer = Observer()
|
|
184
|
-
observer.schedule(event_handler, path=project.path, recursive=True)
|
|
233
|
+
observer.schedule(event_handler, path=str(project.path), recursive=True)
|
|
185
234
|
observer.start()
|
|
186
235
|
|
|
187
236
|
try:
|
|
@@ -1590,7 +1590,7 @@ async def try_update_config_with_remote(
|
|
|
1590
1590
|
def ask_for_admin_token_interactively(ui_host: str, default_token: Optional[str]) -> str:
|
|
1591
1591
|
return (
|
|
1592
1592
|
click.prompt(
|
|
1593
|
-
f
|
|
1593
|
+
f'\nCopy the "admin your@email" token from {ui_host}/tokens and paste it here {"OR press enter to use the token from .tinyb file" if default_token else ""}',
|
|
1594
1594
|
hide_input=True,
|
|
1595
1595
|
show_default=False,
|
|
1596
1596
|
default=default_token,
|
tinybird/tornado_template.py
CHANGED
|
@@ -370,7 +370,7 @@ class Template:
|
|
|
370
370
|
for chunk in self.file.body.chunks:
|
|
371
371
|
if isinstance(chunk, _ExtendsBlock):
|
|
372
372
|
if not loader:
|
|
373
|
-
raise ParseError("{% extends %} block found, but no
|
|
373
|
+
raise ParseError("{% extends %} block found, but no template loader")
|
|
374
374
|
template = loader.load(chunk.name, self.name)
|
|
375
375
|
ancestors.extend(template._get_ancestors(loader))
|
|
376
376
|
return ancestors
|
|
@@ -633,7 +633,7 @@ class _Expression(_Node):
|
|
|
633
633
|
|
|
634
634
|
def generate(self, writer):
|
|
635
635
|
writer.write_line("_tt_tmp = %s" % self.expression, self.line)
|
|
636
|
-
writer.write_line("if isinstance(_tt_tmp, _tt_string_types):
|
|
636
|
+
writer.write_line("if isinstance(_tt_tmp, _tt_string_types): _tt_tmp = _tt_utf8(_tt_tmp)", self.line)
|
|
637
637
|
writer.write_line("else: _tt_tmp = _tt_utf8(str(_tt_tmp))", self.line)
|
|
638
638
|
if not self.raw and writer.current_template.autoescape is not None:
|
|
639
639
|
# In python3 functions like xhtml_escape return unicode,
|
|
@@ -1,58 +1,58 @@
|
|
|
1
1
|
tinybird/__cli__.py,sha256=esPl5QDTzuQgHe5FuxWLm-fURFigGGwjnYLh9GuWUw4,232
|
|
2
|
-
tinybird/client.py,sha256=
|
|
2
|
+
tinybird/client.py,sha256=p3tpPC0QbObihnwGdmw1bRjKmQS2kOebJ9Me3ioM-Ww,51517
|
|
3
3
|
tinybird/config.py,sha256=ENRNyEMXHj_P882o31iFz0hTveziLabVRrxiWE5RRBE,6233
|
|
4
|
-
tinybird/connectors.py,sha256=
|
|
4
|
+
tinybird/connectors.py,sha256=7Gjms7b5MAaBFGi3xytsJurCylprONpFcYrzp4Fw2Rc,15241
|
|
5
5
|
tinybird/context.py,sha256=A3GBApac9xO6hrAMJ1s9dMrI_ou9aKF84CdEjtPddMk,1417
|
|
6
6
|
tinybird/datatypes.py,sha256=XNypumfqNjsvLJ5iNXnbVHRvAJe0aQwI3lS6Cxox-e0,10979
|
|
7
|
-
tinybird/feedback_manager.py,sha256=
|
|
7
|
+
tinybird/feedback_manager.py,sha256=g1r9NcFfKXdk_13soaiTZLvdoUGleVfawl6Yfj3zmRw,67823
|
|
8
8
|
tinybird/git_settings.py,sha256=Sw_8rGmribEFJ4Z_6idrVytxpFYk7ez8ei0qHULzs3E,3934
|
|
9
|
-
tinybird/prompts.py,sha256=
|
|
10
|
-
tinybird/sql.py,sha256=
|
|
9
|
+
tinybird/prompts.py,sha256=pgG-Gwtx8ZQf6XWjMibYVTmW4doykP4gnJda1qA54NY,26746
|
|
10
|
+
tinybird/sql.py,sha256=igHaRIeEREN5XYowwpIGYG8gDB5kn5p2cDNL1t8uc40,46168
|
|
11
11
|
tinybird/sql_template.py,sha256=GmMLAI10MTqjQo9qztuQHLRWs67teozsWDxUBdvkAn4,93668
|
|
12
|
-
tinybird/sql_template_fmt.py,sha256=
|
|
12
|
+
tinybird/sql_template_fmt.py,sha256=KUHdj5rYCYm_rKKdXYSJAE9vIyXUQLB0YSZnUXHeBlY,10196
|
|
13
13
|
tinybird/sql_toolset.py,sha256=NEUj8Ro5x9XlfVLlGr6nWt9o7OLWVxlqs6TIpgumUNs,14678
|
|
14
14
|
tinybird/syncasync.py,sha256=IPnOx6lMbf9SNddN1eBtssg8vCLHMt76SuZ6YNYm-Yk,27761
|
|
15
|
-
tinybird/tornado_template.py,sha256=
|
|
15
|
+
tinybird/tornado_template.py,sha256=FL85SMPq2dH4JqKovmSbaolGdEzwOO91NqOzqXo2Qr0,41863
|
|
16
16
|
tinybird/ch_utils/constants.py,sha256=aYvg2C_WxYWsnqPdZB1ZFoIr8ZY-XjUXYyHKE9Ansj0,3890
|
|
17
17
|
tinybird/ch_utils/engine.py,sha256=OXkBhlzGjZotjD0vaT-rFIbSGV4tpiHxE8qO_ip0SyQ,40454
|
|
18
|
-
tinybird/tb/__cli__.py,sha256=
|
|
19
|
-
tinybird/tb/cli.py,sha256=
|
|
18
|
+
tinybird/tb/__cli__.py,sha256=Hf3ON8UnNJYBmDMYOxmtXUDe3zHYepJYO2kbuaSwKvc,251
|
|
19
|
+
tinybird/tb/cli.py,sha256=X6zLCVI10wOxcu-USGfbp_jjz1cojH5X-9dL7D8c8aI,960
|
|
20
20
|
tinybird/tb/modules/auth.py,sha256=EzRWFmwRkXNhUmRaruEVFLdkbUg8xMSix0cAWl5D4Jg,9029
|
|
21
|
-
tinybird/tb/modules/build.py,sha256=
|
|
22
|
-
tinybird/tb/modules/
|
|
23
|
-
tinybird/tb/modules/
|
|
24
|
-
tinybird/tb/modules/
|
|
25
|
-
tinybird/tb/modules/common.py,sha256=e4U7AT0dUBG6O-7Iq2CVN1UHPd6-ZCFucyW0L5gBi4g,70592
|
|
21
|
+
tinybird/tb/modules/build.py,sha256=sE9SYshNS10kFJor1oLES_ivG6Z573dnrk2W_7EvGmg,7973
|
|
22
|
+
tinybird/tb/modules/cicd.py,sha256=xxXwy-QekJcG14kkJeGNl7LkHduhZXfvBZE8WrU6-t4,5351
|
|
23
|
+
tinybird/tb/modules/cli.py,sha256=O-kmEanr1W1Uxun8N8CM9aD6dlTLsuccoWgdQYpKqco,17271
|
|
24
|
+
tinybird/tb/modules/common.py,sha256=4eaiTsjbjg_UtW52PrhdcBdjiBQs8G5Iih42wW8eTi8,70613
|
|
26
25
|
tinybird/tb/modules/config.py,sha256=mie3oMVTf5YOUFEiLs88P16U4LkJafJjSpjwyAkFHog,10979
|
|
27
|
-
tinybird/tb/modules/copy.py,sha256=
|
|
28
|
-
tinybird/tb/modules/create.py,sha256=
|
|
29
|
-
tinybird/tb/modules/datasource.py,sha256
|
|
30
|
-
tinybird/tb/modules/deployment.py,sha256=
|
|
31
|
-
tinybird/tb/modules/endpoint.py,sha256=
|
|
26
|
+
tinybird/tb/modules/copy.py,sha256=wxyxZg8BPiWDgbW5HXJKYQp7_EumBXmAilo3McbCQOo,5916
|
|
27
|
+
tinybird/tb/modules/create.py,sha256=AJz-oFJGHzKuH7_Dy-shYjTI9KSf9ZaE0Ely4rgvXPw,13646
|
|
28
|
+
tinybird/tb/modules/datasource.py,sha256=hHt-rn4VWrQ4EWG797wnLqRhRP-3C4chAdk_vXxCmVI,13712
|
|
29
|
+
tinybird/tb/modules/deployment.py,sha256=0e00lLMaNoVnONYdA51Zy9Sa_-4jWqePoYeFyHUn5PU,10241
|
|
30
|
+
tinybird/tb/modules/endpoint.py,sha256=9arqN1JQCMb0Nd3-EJ7lukOYkGHHCpQmiiZpp5FqPhc,9432
|
|
32
31
|
tinybird/tb/modules/exceptions.py,sha256=4A2sSjCEqKUMqpP3WI00zouCWW4uLaghXXLZBSw04mY,3363
|
|
33
32
|
tinybird/tb/modules/feedback_manager.py,sha256=e8tqehRR0Buhs8O0n8N2Sg2vnnBVb1NLtnZqkPrYD_A,68379
|
|
34
33
|
tinybird/tb/modules/fmt.py,sha256=poh6_cwVGSf-sBu6LKWuO2TANL_J8Sgm25sPpwxa3Aw,3558
|
|
35
34
|
tinybird/tb/modules/job.py,sha256=956Pj8BEEsiD2GZsV9RKKVM3I_CveOLgS82lykO5ukk,2963
|
|
36
35
|
tinybird/tb/modules/llm.py,sha256=AC0VSphTOM2t-v1_3NLvNN_FIbgMo4dTyMqIv5nniPo,835
|
|
37
|
-
tinybird/tb/modules/llm_utils.py,sha256=
|
|
36
|
+
tinybird/tb/modules/llm_utils.py,sha256=nS9r4FAElJw8yXtmdYrx-rtI2zXR8qXfi1QqUDCfxvg,3469
|
|
38
37
|
tinybird/tb/modules/local.py,sha256=x4xuCGVkoa8KLYGZEJnFUP8HUkKX05Frp_djRVjVjTs,5669
|
|
39
38
|
tinybird/tb/modules/local_common.py,sha256=afPW6bSc-YI7Q4ngvBV53sM7QxnPGBMe5N91iETM2yE,2783
|
|
40
39
|
tinybird/tb/modules/login.py,sha256=0cS-f3MsQFHc6xjw8FRWJm4EJBH9C7Ri68EcO_tiwes,6508
|
|
41
|
-
tinybird/tb/modules/
|
|
42
|
-
tinybird/tb/modules/
|
|
43
|
-
tinybird/tb/modules/
|
|
40
|
+
tinybird/tb/modules/materialization.py,sha256=HQKRTH6lkcYiDQJihbFqF_in58ezXG4ggZ_7Ywp_nUM,5738
|
|
41
|
+
tinybird/tb/modules/mock.py,sha256=maaPY91mQx8m9x-1-2r9svljRO8VO8-uqD-71VfXBCU,5271
|
|
42
|
+
tinybird/tb/modules/pipe.py,sha256=NdzqLsyU2WZ4S5JXGc2CozvGh0WaGL2rsoZ8iQERjm8,2359
|
|
43
|
+
tinybird/tb/modules/project.py,sha256=YH-pwioIBIYxSmy6awM6Ya6WoB7MSM2fte-x9Gbr6lc,2604
|
|
44
44
|
tinybird/tb/modules/regions.py,sha256=QjsL5H6Kg-qr0aYVLrvb1STeJ5Sx_sjvbOYO0LrEGMk,166
|
|
45
45
|
tinybird/tb/modules/shell.py,sha256=iCX9HsvNhVDecop-s4zDPjXfL9GQ56-QODqwHsN5oT8,12994
|
|
46
46
|
tinybird/tb/modules/table.py,sha256=4XrtjM-N0zfNtxVkbvLDQQazno1EPXnxTyo7llivfXk,11035
|
|
47
47
|
tinybird/tb/modules/tag.py,sha256=anPmMUBc-TbFovlpFi8GPkKA18y7Y0GczMsMms5TZsU,3502
|
|
48
48
|
tinybird/tb/modules/telemetry.py,sha256=iEGnMuCuNhvF6ln__j6X9MSTwL_0Hm-GgFHHHvhfknk,10466
|
|
49
|
-
tinybird/tb/modules/test.py,sha256=
|
|
49
|
+
tinybird/tb/modules/test.py,sha256=_I3xufediPNrAi1L2JkFHNwZEWebwVPzSF0iWJCzTsc,13088
|
|
50
50
|
tinybird/tb/modules/token.py,sha256=sPdJoBE-6dd3Sd6W-prst7VOoJ0NbvP0uTaB6dXHs5s,12711
|
|
51
|
-
tinybird/tb/modules/update.py,sha256=
|
|
52
|
-
tinybird/tb/modules/watch.py,sha256=
|
|
51
|
+
tinybird/tb/modules/update.py,sha256=GWYfbKnB0yf4ywAUp8jItaZY0ltBnaIMcLSiPpwtbMc,6832
|
|
52
|
+
tinybird/tb/modules/watch.py,sha256=Kredt5C7OOiI6YOivuR5QBdiDY4J_xLiwqHOROnfcsU,8591
|
|
53
53
|
tinybird/tb/modules/workspace.py,sha256=M0RtXCaw7RdZ3c_fqtmjbVb7HqlV742Drn4OiyZlp3M,6345
|
|
54
54
|
tinybird/tb/modules/workspace_members.py,sha256=Ai6iCOzXX1zQ8q9iXIFSFHsBJlT-8Q28DaG5Ie-UweY,8726
|
|
55
|
-
tinybird/tb/modules/datafile/build.py,sha256=
|
|
55
|
+
tinybird/tb/modules/datafile/build.py,sha256=VgQ0k-dBNE0hXdnxB12NW5I0XpxEosN2vaszboj3LZY,51045
|
|
56
56
|
tinybird/tb/modules/datafile/build_common.py,sha256=IXl-Z51zUi1dypV7meNenX0iu2UmowNeqgG6WHyMHlk,4562
|
|
57
57
|
tinybird/tb/modules/datafile/build_datasource.py,sha256=4aP8_DYCRGghXntZSeWDNJxjps1QRVa7WHoYCzQwQts,17355
|
|
58
58
|
tinybird/tb/modules/datafile/build_pipe.py,sha256=Jgv3YKIvMfjPiSIdw1k2mpaoDdAWMiMRaSHwRgyI97E,28258
|
|
@@ -70,13 +70,13 @@ tinybird/tb/modules/datafile/pull.py,sha256=vcjMUbjnZ9XQMGmL33J3ElpbXBTat8Yzp-ha
|
|
|
70
70
|
tinybird/tb/modules/tinyunit/tinyunit.py,sha256=3EBqKzNCfyDuZiO4H61ihanFBRLFUGeuXf3nDXnYFcU,11727
|
|
71
71
|
tinybird/tb/modules/tinyunit/tinyunit_lib.py,sha256=hGh1ZaXC1af7rKnX7222urkj0QJMhMWclqMy59dOqwE,1922
|
|
72
72
|
tinybird/tb_cli_modules/cicd.py,sha256=0lMkb6CVOFZl5HOwgY8mK4T4mgI7O8335UngLXtCc-c,13851
|
|
73
|
-
tinybird/tb_cli_modules/common.py,sha256=
|
|
73
|
+
tinybird/tb_cli_modules/common.py,sha256=SnC_PLCJHwaFaNg7LsUtc1s5jgiD_vkglbva0D3w8RA,78833
|
|
74
74
|
tinybird/tb_cli_modules/config.py,sha256=6u6B5QCdiQLbJkCkwtnKGs9H3nP-KXXhC75mF7B-1DQ,11464
|
|
75
75
|
tinybird/tb_cli_modules/exceptions.py,sha256=pmucP4kTF4irIt7dXiG-FcnI-o3mvDusPmch1L8RCWk,3367
|
|
76
76
|
tinybird/tb_cli_modules/regions.py,sha256=QjsL5H6Kg-qr0aYVLrvb1STeJ5Sx_sjvbOYO0LrEGMk,166
|
|
77
77
|
tinybird/tb_cli_modules/telemetry.py,sha256=iEGnMuCuNhvF6ln__j6X9MSTwL_0Hm-GgFHHHvhfknk,10466
|
|
78
|
-
tinybird-0.0.1.
|
|
79
|
-
tinybird-0.0.1.
|
|
80
|
-
tinybird-0.0.1.
|
|
81
|
-
tinybird-0.0.1.
|
|
82
|
-
tinybird-0.0.1.
|
|
78
|
+
tinybird-0.0.1.dev44.dist-info/METADATA,sha256=gJMWTcb2L9oHDngWSBCwCqgelvbVB_8fNyFZC5OBPDk,2482
|
|
79
|
+
tinybird-0.0.1.dev44.dist-info/WHEEL,sha256=G16H4A3IeoQmnOrYV4ueZGKSjhipXx8zc8nu9FGlvMA,92
|
|
80
|
+
tinybird-0.0.1.dev44.dist-info/entry_points.txt,sha256=LwdHU6TfKx4Qs7BqqtaczEZbImgU7Abe9Lp920zb_fo,43
|
|
81
|
+
tinybird-0.0.1.dev44.dist-info/top_level.txt,sha256=pgw6AzERHBcW3YTi2PW4arjxLkulk2msOz_SomfOEuc,45
|
|
82
|
+
tinybird-0.0.1.dev44.dist-info/RECORD,,
|
|
@@ -1,199 +0,0 @@
|
|
|
1
|
-
import asyncio
|
|
2
|
-
import os
|
|
3
|
-
import threading
|
|
4
|
-
import time
|
|
5
|
-
from pathlib import Path
|
|
6
|
-
from typing import List
|
|
7
|
-
|
|
8
|
-
import click
|
|
9
|
-
|
|
10
|
-
import tinybird.context as context
|
|
11
|
-
from tinybird.client import TinyB
|
|
12
|
-
from tinybird.config import FeatureFlags
|
|
13
|
-
from tinybird.tb.modules.cli import cli
|
|
14
|
-
from tinybird.tb.modules.common import push_data
|
|
15
|
-
from tinybird.tb.modules.datafile.build import folder_build
|
|
16
|
-
from tinybird.tb.modules.datafile.common import get_project_fixtures, has_internal_datafiles
|
|
17
|
-
from tinybird.tb.modules.datafile.exceptions import ParseException
|
|
18
|
-
from tinybird.tb.modules.datafile.fixture import build_fixture_name, get_fixture_dir
|
|
19
|
-
from tinybird.tb.modules.datafile.parse_datasource import parse_datasource
|
|
20
|
-
from tinybird.tb.modules.datafile.parse_pipe import parse_pipe
|
|
21
|
-
from tinybird.tb.modules.feedback_manager import FeedbackManager
|
|
22
|
-
from tinybird.tb.modules.local_common import get_tinybird_local_client
|
|
23
|
-
from tinybird.tb.modules.project import Project
|
|
24
|
-
from tinybird.tb.modules.shell import Shell, print_table_formatted
|
|
25
|
-
from tinybird.tb.modules.watch import watch_files
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
def is_vendor(f: Path) -> bool:
|
|
29
|
-
return f.parts[0] == "vendor"
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
def get_vendor_workspace(f: Path) -> str:
|
|
33
|
-
return f.parts[1]
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
def is_endpoint(f: Path) -> bool:
|
|
37
|
-
return f.suffix == ".pipe" and not is_vendor(f) and f.parts[0] == "endpoints"
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
def is_pipe(f: Path) -> bool:
|
|
41
|
-
return f.suffix == ".pipe" and not is_vendor(f)
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
def check_filenames(filenames: List[str]):
|
|
45
|
-
parser_matrix = {".pipe": parse_pipe, ".datasource": parse_datasource}
|
|
46
|
-
incl_suffix = ".incl"
|
|
47
|
-
|
|
48
|
-
for filename in filenames:
|
|
49
|
-
file_suffix = Path(filename).suffix
|
|
50
|
-
if file_suffix == incl_suffix:
|
|
51
|
-
continue
|
|
52
|
-
|
|
53
|
-
parser = parser_matrix.get(file_suffix)
|
|
54
|
-
if not parser:
|
|
55
|
-
raise ParseException(FeedbackManager.error_unsupported_datafile(extension=file_suffix))
|
|
56
|
-
|
|
57
|
-
parser(filename)
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
@cli.command()
|
|
61
|
-
@click.option(
|
|
62
|
-
"--watch",
|
|
63
|
-
is_flag=True,
|
|
64
|
-
help="Watch for changes in the files and rebuild them.",
|
|
65
|
-
)
|
|
66
|
-
@click.pass_context
|
|
67
|
-
def build_client(
|
|
68
|
-
ctx: click.Context,
|
|
69
|
-
watch: bool,
|
|
70
|
-
) -> None:
|
|
71
|
-
"""Build the project in Tinybird Local."""
|
|
72
|
-
project: Project = ctx.ensure_object(dict)["project"]
|
|
73
|
-
folder = project.folder
|
|
74
|
-
ignore_sql_errors = FeatureFlags.ignore_sql_errors()
|
|
75
|
-
context.disable_template_security_validation.set(True)
|
|
76
|
-
is_internal = has_internal_datafiles(folder)
|
|
77
|
-
folder_path = os.path.abspath(folder)
|
|
78
|
-
tb_client = asyncio.run(get_tinybird_local_client(folder_path))
|
|
79
|
-
|
|
80
|
-
async def process(filenames: List[str], watch: bool = False):
|
|
81
|
-
datafiles = [f for f in filenames if f.endswith(".datasource") or f.endswith(".pipe")]
|
|
82
|
-
if len(datafiles) > 0:
|
|
83
|
-
check_filenames(filenames=datafiles)
|
|
84
|
-
await folder_build(
|
|
85
|
-
tb_client,
|
|
86
|
-
filenames=datafiles,
|
|
87
|
-
ignore_sql_errors=ignore_sql_errors,
|
|
88
|
-
is_internal=is_internal,
|
|
89
|
-
watch=watch,
|
|
90
|
-
folder=folder,
|
|
91
|
-
)
|
|
92
|
-
if len(filenames) > 0:
|
|
93
|
-
filename = filenames[0]
|
|
94
|
-
if filename.endswith(".ndjson"):
|
|
95
|
-
fixture_path = Path(filename)
|
|
96
|
-
datasources_path = Path(folder) / "datasources"
|
|
97
|
-
ds_name = fixture_path.stem
|
|
98
|
-
ds_path = datasources_path / f"{ds_name}.datasource"
|
|
99
|
-
|
|
100
|
-
if not ds_path.exists():
|
|
101
|
-
try:
|
|
102
|
-
ds_name = "_".join(fixture_path.stem.split("_")[:-1])
|
|
103
|
-
ds_path = datasources_path / f"{ds_name}.datasource"
|
|
104
|
-
except Exception:
|
|
105
|
-
pass
|
|
106
|
-
|
|
107
|
-
if ds_path.exists():
|
|
108
|
-
await append_datasource(tb_client, ds_name, str(fixture_path))
|
|
109
|
-
|
|
110
|
-
if watch:
|
|
111
|
-
if filename.endswith(".datasource"):
|
|
112
|
-
ds_path = Path(filename)
|
|
113
|
-
ds_name = ds_path.stem
|
|
114
|
-
name = build_fixture_name(filename, ds_name, ds_path.read_text())
|
|
115
|
-
fixture_folder = get_fixture_dir(folder)
|
|
116
|
-
fixture_path = fixture_folder / f"{name}.ndjson"
|
|
117
|
-
|
|
118
|
-
if not fixture_path.exists():
|
|
119
|
-
fixture_path = fixture_folder / f"{ds_name}.ndjson"
|
|
120
|
-
|
|
121
|
-
if fixture_path.exists():
|
|
122
|
-
await append_datasource(tb_client, ds_name, str(fixture_path))
|
|
123
|
-
|
|
124
|
-
if not filename.endswith(".ndjson"):
|
|
125
|
-
await build_and_print_resource(tb_client, filename)
|
|
126
|
-
|
|
127
|
-
datafiles = project.get_project_files()
|
|
128
|
-
fixtures = get_project_fixtures(folder)
|
|
129
|
-
filenames = datafiles + fixtures
|
|
130
|
-
|
|
131
|
-
async def build_once(filenames: List[str]):
|
|
132
|
-
ok = False
|
|
133
|
-
try:
|
|
134
|
-
click.echo(FeedbackManager.highlight(message="» Building project...\n"))
|
|
135
|
-
time_start = time.time()
|
|
136
|
-
await process(filenames=filenames, watch=False)
|
|
137
|
-
time_end = time.time()
|
|
138
|
-
elapsed_time = time_end - time_start
|
|
139
|
-
for filename in filenames:
|
|
140
|
-
if filename.endswith(".datasource"):
|
|
141
|
-
ds_path = Path(filename)
|
|
142
|
-
ds_name = ds_path.stem
|
|
143
|
-
name = build_fixture_name(filename, ds_name, ds_path.read_text())
|
|
144
|
-
fixture_folder = get_fixture_dir(folder)
|
|
145
|
-
fixture_path = fixture_folder / f"{name}.ndjson"
|
|
146
|
-
|
|
147
|
-
if not fixture_path.exists():
|
|
148
|
-
fixture_path = fixture_folder / f"{ds_name}.ndjson"
|
|
149
|
-
|
|
150
|
-
if fixture_path.exists():
|
|
151
|
-
await append_datasource(tb_client, ds_name, str(fixture_path))
|
|
152
|
-
click.echo(FeedbackManager.success(message=f"\n✓ Build completed in {elapsed_time:.1f}s"))
|
|
153
|
-
ok = True
|
|
154
|
-
except Exception as e:
|
|
155
|
-
error_path = Path(".tb_error.txt")
|
|
156
|
-
if error_path.exists():
|
|
157
|
-
content = error_path.read_text()
|
|
158
|
-
content += f"\n\n{str(e)}"
|
|
159
|
-
error_path.write_text(content)
|
|
160
|
-
else:
|
|
161
|
-
error_path.write_text(str(e))
|
|
162
|
-
click.echo(FeedbackManager.error_exception(error=e))
|
|
163
|
-
ok = False
|
|
164
|
-
return ok
|
|
165
|
-
|
|
166
|
-
build_ok = asyncio.run(build_once(filenames))
|
|
167
|
-
|
|
168
|
-
if watch:
|
|
169
|
-
shell = Shell(project=project, client=tb_client)
|
|
170
|
-
click.echo(FeedbackManager.gray(message="\nWatching for changes..."))
|
|
171
|
-
watcher_thread = threading.Thread(
|
|
172
|
-
target=watch_files, args=(filenames, process, shell, project, build_ok), daemon=True
|
|
173
|
-
)
|
|
174
|
-
watcher_thread.start()
|
|
175
|
-
shell.run()
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
async def build_and_print_resource(tb_client: TinyB, filename: str):
|
|
179
|
-
resource_path = Path(filename)
|
|
180
|
-
name = resource_path.stem
|
|
181
|
-
pipeline = name if filename.endswith(".pipe") else None
|
|
182
|
-
res = await tb_client.query(f"SELECT * FROM {name} FORMAT JSON", pipeline=pipeline)
|
|
183
|
-
print_table_formatted(res, name)
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
async def append_datasource(
|
|
187
|
-
tb_client: TinyB,
|
|
188
|
-
datasource_name: str,
|
|
189
|
-
url: str,
|
|
190
|
-
):
|
|
191
|
-
await tb_client.datasource_truncate(datasource_name)
|
|
192
|
-
await push_data(
|
|
193
|
-
tb_client,
|
|
194
|
-
datasource_name,
|
|
195
|
-
url,
|
|
196
|
-
mode="append",
|
|
197
|
-
concurrency=1,
|
|
198
|
-
silent=True,
|
|
199
|
-
)
|
|
File without changes
|
|
File without changes
|
|
File without changes
|