tinybird 0.0.1.dev19__py3-none-any.whl → 0.0.1.dev20__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of tinybird might be problematic. Click here for more details.

Files changed (49) hide show
  1. tinybird/client.py +2 -2
  2. tinybird/config.py +1 -1
  3. tinybird/feedback_manager.py +8 -30
  4. tinybird/tb/__cli__.py +2 -2
  5. tinybird/tb/modules/auth.py +1 -1
  6. tinybird/tb/modules/build.py +10 -78
  7. tinybird/tb/modules/cicd.py +1 -1
  8. tinybird/tb/modules/cli.py +9 -837
  9. tinybird/tb/modules/common.py +1 -55
  10. tinybird/tb/modules/connection.py +1 -1
  11. tinybird/tb/modules/create.py +18 -7
  12. tinybird/tb/modules/datafile/build.py +142 -971
  13. tinybird/tb/modules/datafile/build_common.py +1 -1
  14. tinybird/tb/modules/datafile/build_datasource.py +1 -1
  15. tinybird/tb/modules/datafile/build_pipe.py +1 -1
  16. tinybird/tb/modules/datafile/common.py +11 -11
  17. tinybird/tb/modules/datafile/diff.py +1 -1
  18. tinybird/tb/modules/datafile/fixture.py +1 -1
  19. tinybird/tb/modules/datafile/format_common.py +0 -7
  20. tinybird/tb/modules/datafile/format_datasource.py +0 -2
  21. tinybird/tb/modules/datafile/format_pipe.py +0 -2
  22. tinybird/tb/modules/datafile/parse_datasource.py +1 -1
  23. tinybird/tb/modules/datafile/parse_pipe.py +1 -1
  24. tinybird/tb/modules/datafile/pull.py +1 -1
  25. tinybird/tb/modules/datasource.py +4 -75
  26. tinybird/tb/modules/feedback_manager.py +1048 -0
  27. tinybird/tb/modules/fmt.py +1 -1
  28. tinybird/tb/modules/job.py +1 -1
  29. tinybird/tb/modules/llm.py +6 -4
  30. tinybird/tb/modules/local.py +26 -21
  31. tinybird/tb/modules/local_common.py +2 -1
  32. tinybird/tb/modules/login.py +8 -8
  33. tinybird/tb/modules/mock.py +18 -7
  34. tinybird/tb/modules/pipe.py +4 -126
  35. tinybird/tb/modules/{build_shell.py → shell.py} +9 -21
  36. tinybird/tb/modules/table.py +88 -5
  37. tinybird/tb/modules/tag.py +2 -2
  38. tinybird/tb/modules/test.py +8 -11
  39. tinybird/tb/modules/tinyunit/tinyunit.py +1 -1
  40. tinybird/tb/modules/token.py +2 -2
  41. tinybird/tb/modules/watch.py +72 -0
  42. tinybird/tb/modules/workspace.py +1 -1
  43. tinybird/tb/modules/workspace_members.py +1 -1
  44. {tinybird-0.0.1.dev19.dist-info → tinybird-0.0.1.dev20.dist-info}/METADATA +1 -1
  45. tinybird-0.0.1.dev20.dist-info/RECORD +76 -0
  46. tinybird-0.0.1.dev19.dist-info/RECORD +0 -74
  47. {tinybird-0.0.1.dev19.dist-info → tinybird-0.0.1.dev20.dist-info}/WHEEL +0 -0
  48. {tinybird-0.0.1.dev19.dist-info → tinybird-0.0.1.dev20.dist-info}/entry_points.txt +0 -0
  49. {tinybird-0.0.1.dev19.dist-info → tinybird-0.0.1.dev20.dist-info}/top_level.txt +0 -0
@@ -7,64 +7,48 @@ import json
7
7
  import logging
8
8
  import os
9
9
  import pprint
10
- import re
11
- import shutil
12
- import sys
13
10
  from os import getcwd
14
11
  from pathlib import Path
15
- from typing import Any, Callable, Dict, Iterable, List, Optional, Tuple, Union
12
+ from typing import Any, Callable, Iterable, List, Optional, Tuple, Union
16
13
 
17
14
  import click
18
15
  import humanfriendly
19
16
  from click import Context
20
- from packaging import version
21
17
 
22
- import tinybird.context as context
23
18
  from tinybird.client import (
24
19
  AuthException,
25
20
  AuthNoTokenException,
26
- DoesNotExistException,
27
21
  TinyB,
28
22
  )
29
- from tinybird.config import SUPPORTED_CONNECTORS, FeatureFlags, get_config
30
- from tinybird.feedback_manager import FeedbackManager
23
+ from tinybird.config import SUPPORTED_CONNECTORS, get_config
31
24
  from tinybird.tb import __cli__
32
25
  from tinybird.tb.modules.common import (
33
- OLDEST_ROLLBACK,
34
26
  CatchAuthExceptions,
35
27
  CLIException,
36
28
  _get_tb_client,
37
29
  coro,
38
- create_tb_client,
39
30
  echo_safe_format_table,
40
31
  get_current_main_workspace,
41
32
  getenv_bool,
42
- is_major_semver,
43
- is_post_semver,
44
33
  load_connector_config,
45
- remove_release,
46
34
  try_update_config_with_remote,
47
- wait_job,
48
35
  )
49
36
  from tinybird.tb.modules.config import CLIConfig
50
- from tinybird.tb.modules.datafile.build import build_graph, folder_push
37
+ from tinybird.tb.modules.datafile.build import build_graph
51
38
  from tinybird.tb.modules.datafile.common import (
52
39
  Datafile,
53
40
  DatafileSyntaxError,
54
41
  get_project_filenames,
55
- get_resource_versions,
56
- has_internal_datafiles,
57
42
  )
58
43
  from tinybird.tb.modules.datafile.diff import diff_command
59
44
  from tinybird.tb.modules.datafile.exceptions import (
60
- AlreadyExistsException,
61
45
  ParseException,
62
46
  )
63
47
  from tinybird.tb.modules.datafile.parse_datasource import parse_datasource
64
48
  from tinybird.tb.modules.datafile.parse_pipe import parse_pipe
65
49
  from tinybird.tb.modules.datafile.pull import folder_pull
50
+ from tinybird.tb.modules.feedback_manager import FeedbackManager
66
51
  from tinybird.tb.modules.local_common import get_tinybird_local_client
67
- from tinybird.tb.modules.telemetry import add_telemetry_event
68
52
 
69
53
  __old_click_echo = click.echo
70
54
  __old_click_secho = click.secho
@@ -234,7 +218,7 @@ async def cli(
234
218
  load_connector_config(ctx, connector, debug, check_uninstalled=True)
235
219
 
236
220
 
237
- @cli.command()
221
+ @cli.command(hidden=True)
238
222
  @click.argument("filenames", type=click.Path(exists=True), nargs=-1, default=None)
239
223
  @click.option("--debug", is_flag=True, default=False, help="Print internal representation")
240
224
  def check(filenames: List[str], debug: bool) -> None:
@@ -273,208 +257,14 @@ def check(filenames: List[str], debug: bool) -> None:
273
257
 
274
258
  except DatafileSyntaxError as e:
275
259
  # TODO(eclbg): add the filename to the error message
276
- raise CLIException(e)
260
+ raise CLIException(str(e))
277
261
  except ParseException as e:
278
262
  raise CLIException(FeedbackManager.error_exception(error=e))
279
263
 
280
264
  process(filenames=filenames)
281
265
 
282
266
 
283
- @cli.command()
284
- @click.option(
285
- "--dry-run",
286
- is_flag=True,
287
- default=False,
288
- help="Run the command without creating resources on the Tinybird account or any side effect",
289
- )
290
- @click.option(
291
- "--check/--no-check", is_flag=True, default=True, help="Enable/Disable output checking, enabled by default"
292
- )
293
- @click.option("--push-deps", is_flag=True, default=False, help="Push dependencies, disabled by default")
294
- @click.option(
295
- "--only-changes",
296
- is_flag=True,
297
- default=False,
298
- help="Push only the resources that have changed compared to the destination workspace",
299
- )
300
- @click.option(
301
- "--debug",
302
- is_flag=True,
303
- default=False,
304
- help="Prints internal representation, can be combined with any command to get more information.",
305
- )
306
- @click.option("-f", "--force", is_flag=True, default=False, help="Override pipes when they already exist")
307
- @click.option(
308
- "--override-datasource",
309
- is_flag=True,
310
- default=False,
311
- help="When pushing a pipe with a Materialized node if the target Data Source exists it will try to override it.",
312
- )
313
- @click.option("--populate", is_flag=True, default=False, help="Populate materialized nodes when pushing them")
314
- @click.option(
315
- "--subset",
316
- type=float,
317
- default=None,
318
- help="Populate with a subset percent of the data (limited to a maximum of 2M rows), this is useful to quickly test a materialized node with some data. The subset must be greater than 0 and lower than 0.1. A subset of 0.1 means a 10 percent of the data in the source Data Source will be used to populate the materialized view. Use it together with --populate, it has precedence over --sql-condition",
319
- )
320
- @click.option(
321
- "--sql-condition",
322
- type=str,
323
- default=None,
324
- help="Populate with a SQL condition to be applied to the trigger Data Source of the Materialized View. For instance, `--sql-condition='date == toYYYYMM(now())'` it'll populate taking all the rows from the trigger Data Source which `date` is the current month. Use it together with --populate. --sql-condition is not taken into account if the --subset param is present. Including in the ``sql_condition`` any column present in the Data Source ``engine_sorting_key`` will make the populate job process less data.",
325
- )
326
- @click.option(
327
- "--unlink-on-populate-error",
328
- is_flag=True,
329
- default=False,
330
- help="If the populate job fails the Materialized View is unlinked and new data won't be ingested in the Materialized View. First time a populate job fails, the Materialized View is always unlinked.",
331
- )
332
- @click.option("--fixtures", is_flag=True, default=False, help="Append fixtures to data sources")
333
- @click.option(
334
- "--wait",
335
- is_flag=True,
336
- default=False,
337
- help="To be used along with --populate command. Waits for populate jobs to finish, showing a progress bar. Disabled by default.",
338
- )
339
- @click.option("--yes", is_flag=True, default=False, help="Do not ask for confirmation")
340
- @click.option(
341
- "--only-response-times", is_flag=True, default=False, help="Checks only response times, when --force push a pipe"
342
- )
343
- @click.argument("filenames", type=click.Path(exists=True), nargs=-1, default=None)
344
- @click.option("--workspace_map", nargs=2, type=str, multiple=True)
345
- @click.option(
346
- "--workspace",
347
- nargs=2,
348
- type=str,
349
- multiple=True,
350
- help="add a workspace path to the list of external workspaces, usage: --workspace name path/to/folder",
351
- )
352
- @click.option(
353
- "--no-versions",
354
- is_flag=True,
355
- default=False,
356
- help="when set, resource dependency versions are not used, it pushes the dependencies as-is",
357
- )
358
- @click.option(
359
- "-l", "--limit", type=click.IntRange(0, 100), default=0, required=False, help="Number of requests to validate"
360
- )
361
- @click.option(
362
- "--sample-by-params",
363
- type=click.IntRange(1, 100),
364
- default=1,
365
- required=False,
366
- help="When set, we will aggregate the pipe_stats_rt requests by extractURLParameterNames(assumeNotNull(url)) and for each combination we will take a sample of N requests",
367
- )
368
- @click.option(
369
- "-ff", "--failfast", is_flag=True, default=False, help="When set, the checker will exit as soon one test fails"
370
- )
371
- @click.option(
372
- "--ignore-order", is_flag=True, default=False, help="When set, the checker will ignore the order of list properties"
373
- )
374
- @click.option(
375
- "--validate-processed-bytes",
376
- is_flag=True,
377
- default=False,
378
- help="When set, the checker will validate that the new version doesn't process more than 25% than the current version",
379
- )
380
- @click.option(
381
- "--check-requests-from-main",
382
- is_flag=True,
383
- default=False,
384
- help="When set, the checker will get Main Workspace requests",
385
- hidden=True,
386
- )
387
- @click.option(
388
- "--folder",
389
- default=".",
390
- help="Folder from where to execute the command. By default the current folder",
391
- hidden=True,
392
- type=click.types.STRING,
393
- )
394
- @click.option(
395
- "--user_token",
396
- is_flag=False,
397
- default=None,
398
- help="The user token is required for sharing a datasource that contains the SHARED_WITH entry.",
399
- type=click.types.STRING,
400
- )
401
- @click.pass_context
402
- @coro
403
- async def push(
404
- ctx: Context,
405
- filenames: Optional[List[str]],
406
- dry_run: bool,
407
- check: bool,
408
- push_deps: bool,
409
- only_changes: bool,
410
- debug: bool,
411
- force: bool,
412
- override_datasource: bool,
413
- populate: bool,
414
- subset: Optional[float],
415
- sql_condition: Optional[str],
416
- unlink_on_populate_error: bool,
417
- fixtures: bool,
418
- wait: bool,
419
- yes: bool,
420
- only_response_times: bool,
421
- workspace_map,
422
- workspace,
423
- no_versions: bool,
424
- limit: int,
425
- sample_by_params: int,
426
- failfast: bool,
427
- ignore_order: bool,
428
- validate_processed_bytes: bool,
429
- check_requests_from_main: bool,
430
- folder: str,
431
- user_token: Optional[str],
432
- ) -> None:
433
- """Push files to Tinybird."""
434
-
435
- ignore_sql_errors = FeatureFlags.ignore_sql_errors()
436
- context.disable_template_security_validation.set(True)
437
-
438
- is_internal = has_internal_datafiles(folder)
439
-
440
- await folder_push(
441
- create_tb_client(ctx),
442
- filenames,
443
- dry_run,
444
- check,
445
- push_deps,
446
- only_changes,
447
- debug=debug,
448
- force=force,
449
- override_datasource=override_datasource,
450
- populate=populate,
451
- populate_subset=subset,
452
- populate_condition=sql_condition,
453
- unlink_on_populate_error=unlink_on_populate_error,
454
- upload_fixtures=fixtures,
455
- wait=wait,
456
- ignore_sql_errors=ignore_sql_errors,
457
- skip_confirmation=yes,
458
- only_response_times=only_response_times,
459
- workspace_map=dict(workspace_map),
460
- workspace_lib_paths=workspace,
461
- no_versions=no_versions,
462
- run_tests=False,
463
- tests_to_run=limit,
464
- tests_sample_by_params=sample_by_params,
465
- tests_failfast=failfast,
466
- tests_ignore_order=ignore_order,
467
- tests_validate_processed_bytes=validate_processed_bytes,
468
- tests_check_requests_from_branch=check_requests_from_main,
469
- folder=folder,
470
- config=CLIConfig.get_project_config(),
471
- user_token=user_token,
472
- is_internal=is_internal,
473
- )
474
- return
475
-
476
-
477
- @cli.command()
267
+ @cli.command(hidden=True)
478
268
  @click.option(
479
269
  "--folder", default=None, type=click.Path(exists=True, file_okay=False), help="Folder where files will be placed"
480
270
  )
@@ -491,7 +281,7 @@ async def pull(ctx: Context, folder: str, force: bool, fmt: bool) -> None:
491
281
  return await folder_pull(client, folder, force, fmt=fmt)
492
282
 
493
283
 
494
- @cli.command()
284
+ @cli.command(hidden=True)
495
285
  @click.option("--no-deps", is_flag=True, default=False, help="Print only data sources with no pipes using them")
496
286
  @click.option("--match", default=None, help="Retrieve any resource matching the pattern")
497
287
  @click.option("--pipe", default=None, help="Retrieve any resource used by pipe")
@@ -643,11 +433,6 @@ async def sql(
643
433
  f"SELECT * FROM ({query}) LIMIT {rows_limit} FORMAT {req_format}", pipeline=pipeline
644
434
  )
645
435
  elif pipe and node:
646
- datasources: List[Dict[str, Any]] = await client.datasources()
647
- pipes: List[Dict[str, Any]] = await client.pipes()
648
-
649
- existing_resources: List[str] = [x["name"] for x in datasources] + [x["name"] for x in pipes]
650
- resource_versions = get_resource_versions(existing_resources)
651
436
  filenames = [pipe]
652
437
 
653
438
  # build graph to get new versions for all the files involved in the query
@@ -660,22 +445,6 @@ async def sql(
660
445
  skip_connectors=True,
661
446
  )
662
447
 
663
- # update existing versions
664
- latest_datasource_versions = resource_versions.copy()
665
-
666
- for dep in dependencies_graph.to_run.values():
667
- ds = dep["resource_name"]
668
- if dep["version"] is not None:
669
- latest_datasource_versions[ds] = dep["version"]
670
-
671
- # build the graph again with the rigth version
672
- dependencies_graph = await build_graph(
673
- filenames,
674
- client,
675
- dir_path=".",
676
- resource_versions=latest_datasource_versions,
677
- verbose=False,
678
- )
679
448
  query = ""
680
449
  for _, elem in dependencies_graph.to_run.items():
681
450
  for _node in elem["nodes"]:
@@ -717,291 +486,6 @@ async def sql(
717
486
  click.echo(FeedbackManager.info_no_rows())
718
487
 
719
488
 
720
- @cli.command(
721
- name="materialize",
722
- short_help="Given a local Pipe datafile (.pipe) and a node name it generates the target Data Source and materialized Pipe ready to be pushed and guides you through the process to create the materialized view",
723
- )
724
- @click.argument("filename", type=click.Path(exists=True))
725
- @click.argument("target_datasource", default=None, required=False)
726
- @click.option("--push-deps", is_flag=True, default=False, help="Push dependencies, disabled by default")
727
- @click.option("--workspace_map", nargs=2, type=str, multiple=True, hidden=True)
728
- @click.option(
729
- "--workspace",
730
- nargs=2,
731
- type=str,
732
- multiple=True,
733
- help="add a workspace path to the list of external workspaces, usage: --workspace name path/to/folder",
734
- )
735
- @click.option(
736
- "--no-versions",
737
- is_flag=True,
738
- default=False,
739
- help="when set, resource dependency versions are not used, it pushes the dependencies as-is",
740
- )
741
- @click.option("--verbose", is_flag=True, default=False, help="Prints more log")
742
- @click.option("--force-populate", default=None, required=False, help="subset or full", hidden=True)
743
- @click.option(
744
- "--unlink-on-populate-error",
745
- is_flag=True,
746
- default=False,
747
- help="If the populate job fails the Materialized View is unlinked and new data won't be ingested in the Materialized View. First time a populate job fails, the Materialized View is always unlinked.",
748
- )
749
- @click.option("--override-pipe", is_flag=True, default=False, help="Override pipe if exists or prompt", hidden=True)
750
- @click.option(
751
- "--override-datasource", is_flag=True, default=False, help="Override data source if exists or prompt", hidden=True
752
- )
753
- @click.pass_context
754
- @coro
755
- async def materialize(
756
- ctx: Context,
757
- filename: str,
758
- push_deps: bool,
759
- workspace_map: List[str],
760
- workspace: List[str],
761
- no_versions: bool,
762
- verbose: bool,
763
- force_populate: Optional[str],
764
- unlink_on_populate_error: bool,
765
- override_pipe: bool,
766
- override_datasource: bool,
767
- target_datasource: Optional[str] = None,
768
- ) -> None:
769
- deprecation_notice = FeedbackManager.warning_deprecated(
770
- warning="'tb materialize' is deprecated. To create a Materialized View in a guided way you can use the UI: `Create Pipe` and use the `Create Materialized View` wizard. Finally download the resulting `.pipe` and `.datasource` files."
771
- )
772
- click.echo(deprecation_notice)
773
- cl = create_tb_client(ctx)
774
-
775
- async def _try_push_pipe_to_analyze(pipe_name):
776
- try:
777
- to_run = await folder_push(
778
- cl,
779
- filenames=[filename],
780
- dry_run=False,
781
- check=False,
782
- push_deps=push_deps,
783
- debug=False,
784
- force=False,
785
- workspace_map=dict(workspace_map),
786
- workspace_lib_paths=workspace,
787
- no_versions=no_versions,
788
- run_tests=False,
789
- as_standard=True,
790
- raise_on_exists=True,
791
- verbose=verbose,
792
- )
793
- except AlreadyExistsException as e:
794
- if "Datasource" in str(e):
795
- click.echo(str(e))
796
- return
797
- if override_pipe or click.confirm(FeedbackManager.info_pipe_exists(name=pipe_name)):
798
- to_run = await folder_push(
799
- cl,
800
- filenames=[filename],
801
- dry_run=False,
802
- check=False,
803
- push_deps=push_deps,
804
- debug=False,
805
- force=True,
806
- workspace_map=dict(workspace_map),
807
- workspace_lib_paths=workspace,
808
- no_versions=no_versions,
809
- run_tests=False,
810
- as_standard=True,
811
- verbose=verbose,
812
- )
813
- else:
814
- return
815
- except click.ClickException as ex:
816
- # HACK: By now, datafile raises click.ClickException instead of
817
- # CLIException to avoid circular imports. Thats we need to trace
818
- # the error here.
819
- #
820
- # Once we do a big refactor in datafile, we can get rid of this
821
- # snippet.
822
- msg: str = str(ex)
823
- add_telemetry_event("datafile_error", error=msg)
824
- click.echo(msg)
825
-
826
- return to_run
827
-
828
- def _choose_node_name(pipe):
829
- node = pipe["nodes"][0]
830
- materialized_nodes = [node for node in pipe["nodes"] if node["type"].lower() == "materialized"]
831
-
832
- if len(materialized_nodes) == 1:
833
- node = materialized_nodes[0]
834
-
835
- if len(pipe["nodes"]) > 1 and len(materialized_nodes) != 1:
836
- for index, node in enumerate(pipe["nodes"], start=1):
837
- click.echo(f" [{index}] Materialize node with name => {node['name']}")
838
- option = click.prompt(FeedbackManager.prompt_choose_node(), default=len(pipe["nodes"]))
839
- node = pipe["nodes"][option - 1]
840
- node_name = node["name"]
841
- return node, node_name
842
-
843
- def _choose_target_datasource_name(pipe, node, node_name):
844
- return target_datasource or node.get("datasource", None) or f'mv_{pipe["resource_name"]}_{node_name}'
845
-
846
- def _save_local_backup_pipe(pipe):
847
- pipe_bak = f"{filename}_bak"
848
- shutil.copyfile(filename, pipe_bak)
849
- pipe_file_name = f"{pipe['resource_name']}.pipe"
850
- click.echo(FeedbackManager.info_pipe_backup_created(name=pipe_bak))
851
- return pipe_file_name
852
-
853
- def _save_local_datasource(datasource_name, ds_datafile):
854
- base = Path("datasources")
855
- if not base.exists():
856
- base = Path()
857
- file_name = f"{datasource_name}.datasource"
858
- f = base / file_name
859
- with open(f"{f}", "w") as file:
860
- file.write(ds_datafile)
861
-
862
- click.echo(FeedbackManager.success_generated_local_file(file=f))
863
- return f
864
-
865
- async def _try_push_datasource(datasource_name, f):
866
- exists = False
867
- try:
868
- exists = await cl.get_datasource(datasource_name)
869
- except Exception:
870
- pass
871
-
872
- if exists:
873
- click.echo(FeedbackManager.info_materialize_push_datasource_exists(name=f.name))
874
- if override_datasource or click.confirm(FeedbackManager.info_materialize_push_datasource_override(name=f)):
875
- try:
876
- await cl.datasource_delete(datasource_name, force=True)
877
- except DoesNotExistException:
878
- pass
879
-
880
- filename = str(f.absolute())
881
- to_run = await folder_push(
882
- cl,
883
- filenames=[filename],
884
- push_deps=push_deps,
885
- workspace_map=dict(workspace_map),
886
- workspace_lib_paths=workspace,
887
- no_versions=no_versions,
888
- verbose=verbose,
889
- )
890
- return to_run
891
-
892
- def _save_local_pipe(pipe_file_name, pipe_datafile, pipe):
893
- base = Path("pipes")
894
- if not base.exists():
895
- base = Path()
896
- f_pipe = base / pipe_file_name
897
-
898
- with open(f"{f_pipe}", "w") as file:
899
- if pipe["version"] is not None and pipe["version"] >= 0:
900
- pipe_datafile = f"VERSION {pipe['version']} \n {pipe_datafile}"
901
- matches = re.findall(r"([^\s\.]*__v\d+)", pipe_datafile)
902
- for match in set(matches):
903
- m = match.split("__v")[0]
904
- if m in pipe_datafile:
905
- pipe_datafile = pipe_datafile.replace(match, m)
906
- file.write(pipe_datafile)
907
-
908
- click.echo(FeedbackManager.success_generated_local_file(file=f_pipe))
909
- return f_pipe
910
-
911
- async def _try_push_pipe(f_pipe):
912
- if override_pipe:
913
- option = 2
914
- else:
915
- click.echo(FeedbackManager.info_materialize_push_pipe_skip(name=f_pipe.name))
916
- click.echo(FeedbackManager.info_materialize_push_pipe_override(name=f_pipe.name))
917
- option = click.prompt(FeedbackManager.prompt_choose(), default=1)
918
- force = True
919
- check = option == 1
920
-
921
- filename = str(f_pipe.absolute())
922
- to_run = await folder_push(
923
- cl,
924
- filenames=[filename],
925
- dry_run=False,
926
- check=check,
927
- push_deps=push_deps,
928
- debug=False,
929
- force=force,
930
- unlink_on_populate_error=unlink_on_populate_error,
931
- workspace_map=dict(workspace_map),
932
- workspace_lib_paths=workspace,
933
- no_versions=no_versions,
934
- run_tests=False,
935
- verbose=verbose,
936
- )
937
- return to_run
938
-
939
- async def _populate(pipe, node_name, f_pipe):
940
- if force_populate or click.confirm(FeedbackManager.prompt_populate(file=f_pipe)):
941
- if not force_populate:
942
- click.echo(FeedbackManager.info_materialize_populate_partial())
943
- click.echo(FeedbackManager.info_materialize_populate_full())
944
- option = click.prompt(FeedbackManager.prompt_choose(), default=1)
945
- else:
946
- option = 1 if force_populate == "subset" else 2
947
- populate = False
948
- populate_subset = False
949
- if option == 1:
950
- populate_subset = 0.1
951
- populate = True
952
- elif option == 2:
953
- populate = True
954
-
955
- if populate:
956
- response = await cl.populate_node(
957
- pipe["name"],
958
- node_name,
959
- populate_subset=populate_subset,
960
- unlink_on_populate_error=unlink_on_populate_error,
961
- )
962
- if "job" not in response:
963
- raise CLIException(response)
964
-
965
- job_url = response.get("job", {}).get("job_url", None)
966
- job_id = response.get("job", {}).get("job_id", None)
967
- click.echo(FeedbackManager.info_populate_job_url(url=job_url))
968
- wait_populate = True
969
- if wait_populate:
970
- await wait_job(cl, job_id, job_url, "Populating")
971
-
972
- click.echo(FeedbackManager.warning_beta_tester())
973
- pipe_name = os.path.basename(filename).rsplit(".", 1)[0]
974
- click.echo(FeedbackManager.info_before_push_materialize(name=filename))
975
- try:
976
- # extracted the materialize logic to local functions so the workflow is more readable
977
- to_run = await _try_push_pipe_to_analyze(pipe_name)
978
-
979
- if to_run is None:
980
- return
981
-
982
- pipe = to_run[pipe_name.split("/")[-1]]
983
- node, node_name = _choose_node_name(pipe)
984
- datasource_name = _choose_target_datasource_name(pipe, node, node_name)
985
-
986
- click.echo(FeedbackManager.info_before_materialize(name=pipe["name"]))
987
- analysis = await cl.analyze_pipe_node(pipe["name"], node, datasource_name=datasource_name)
988
- ds_datafile = analysis["analysis"]["datasource"]["datafile"]
989
- pipe_datafile = analysis["analysis"]["pipe"]["datafile"]
990
-
991
- pipe_file_name = _save_local_backup_pipe(pipe)
992
- f = _save_local_datasource(datasource_name, ds_datafile)
993
- await _try_push_datasource(datasource_name, f)
994
-
995
- f_pipe = _save_local_pipe(pipe_file_name, pipe_datafile, pipe)
996
- await _try_push_pipe(f_pipe)
997
- await _populate(pipe, node_name, f_pipe)
998
- click.echo(FeedbackManager.success_created_matview(name=datasource_name))
999
- except AuthNoTokenException:
1000
- raise
1001
- except Exception as e:
1002
- raise CLIException(FeedbackManager.error_exception(error=str(e)))
1003
-
1004
-
1005
489
  def __patch_click_output():
1006
490
  import re
1007
491
 
@@ -1036,316 +520,4 @@ def __patch_click_output():
1036
520
 
1037
521
  def __unpatch_click_output():
1038
522
  click.echo = __old_click_echo
1039
- click.secho = __old_click_echo
1040
-
1041
-
1042
- @cli.command(short_help="Learn how to include info about the CLI in your shell PROMPT")
1043
- @click.pass_context
1044
- @coro
1045
- async def prompt(_ctx: Context) -> None:
1046
- click.secho("Follow these instructions => https://www.tinybird.co/docs/cli.html#configure-the-shell-prompt")
1047
-
1048
-
1049
- @cli.command()
1050
- @click.option(
1051
- "--dry-run",
1052
- is_flag=True,
1053
- default=False,
1054
- help="Run the command without creating resources on the Tinybird account or any side effect",
1055
- )
1056
- @click.option(
1057
- "--debug",
1058
- is_flag=True,
1059
- default=False,
1060
- help="Prints internal representation, can be combined with any command to get more information.",
1061
- )
1062
- @click.option(
1063
- "--override-datasource",
1064
- is_flag=True,
1065
- default=False,
1066
- help="When pushing a pipe with a Materialized node if the target Data Source exists it will try to override it.",
1067
- )
1068
- @click.option("--populate", is_flag=True, default=False, help="Populate materialized nodes when pushing them")
1069
- @click.option(
1070
- "--subset",
1071
- type=float,
1072
- default=None,
1073
- help="Populate with a subset percent of the data (limited to a maximum of 2M rows), this is useful to quickly test a materialized node with some data. The subset must be greater than 0 and lower than 0.1. A subset of 0.1 means a 10 percent of the data in the source Data Source will be used to populate the materialized view. Use it together with --populate, it has precedence over --sql-condition",
1074
- )
1075
- @click.option(
1076
- "--sql-condition",
1077
- type=str,
1078
- default=None,
1079
- help="Populate with a SQL condition to be applied to the trigger Data Source of the Materialized View. For instance, `--sql-condition='date == toYYYYMM(now())'` it'll populate taking all the rows from the trigger Data Source which `date` is the current month. Use it together with --populate. --sql-condition is not taken into account if the --subset param is present. Including in the ``sql_condition`` any column present in the Data Source ``engine_sorting_key`` will make the populate job process less data.",
1080
- )
1081
- @click.option(
1082
- "--unlink-on-populate-error",
1083
- is_flag=True,
1084
- default=False,
1085
- help="If the populate job fails the Materialized View is unlinked and new data won't be ingested in the Materialized View. First time a populate job fails, the Materialized View is always unlinked.",
1086
- )
1087
- @click.option(
1088
- "--wait",
1089
- is_flag=True,
1090
- default=False,
1091
- help="To be used along with --populate command. Waits for populate jobs to finish, showing a progress bar. Disabled by default.",
1092
- )
1093
- @click.option("--yes", is_flag=True, default=False, help="Do not ask for confirmation")
1094
- @click.option("--workspace_map", nargs=2, type=str, multiple=True)
1095
- @click.option(
1096
- "--workspace",
1097
- nargs=2,
1098
- type=str,
1099
- multiple=True,
1100
- help="add a workspace path to the list of external workspaces, usage: --workspace name path/to/folder",
1101
- )
1102
- @click.option(
1103
- "--folder",
1104
- default=".",
1105
- help="Folder from where to execute the command. By default the current folder",
1106
- hidden=True,
1107
- type=click.types.STRING,
1108
- )
1109
- @click.option(
1110
- "--user_token",
1111
- is_flag=False,
1112
- default=None,
1113
- help="The user token is required for sharing a datasource that contains the SHARED_WITH entry.",
1114
- type=click.types.STRING,
1115
- )
1116
- @click.option(
1117
- "--fork-downstream",
1118
- is_flag=True,
1119
- default=False,
1120
- help="Creates new versions of the dependent downstream resources to the changed resources.",
1121
- hidden=True,
1122
- )
1123
- @click.option(
1124
- "--fork",
1125
- is_flag=True,
1126
- default=False,
1127
- help="Creates new versions of the changed resources. Use --fork-downstream to fork also the downstream dependencies of the changed resources.",
1128
- hidden=True,
1129
- )
1130
- # this is added to use tb deploy in dry run mode. It's temprary => https://gitlab.com/tinybird/analytics/-/issues/12551
1131
- @click.option(
1132
- "--use-main",
1133
- is_flag=True,
1134
- default=False,
1135
- help="Use main commit instead of release commit",
1136
- hidden=True,
1137
- )
1138
- @click.option(
1139
- "--skip-head-outdated",
1140
- is_flag=True,
1141
- default=False,
1142
- help="Allows to deploy any commit without checking if the branch is rebased to the workspace commit",
1143
- hidden=True,
1144
- )
1145
- @click.pass_context
1146
- @coro
1147
- async def deploy(
1148
- ctx: Context,
1149
- dry_run: bool,
1150
- debug: bool,
1151
- override_datasource: bool,
1152
- populate: bool,
1153
- subset: Optional[float],
1154
- sql_condition: Optional[str],
1155
- unlink_on_populate_error: bool,
1156
- wait: bool,
1157
- yes: bool,
1158
- workspace_map,
1159
- workspace,
1160
- folder: str,
1161
- user_token: Optional[str],
1162
- fork_downstream: bool,
1163
- fork: bool,
1164
- use_main: bool,
1165
- skip_head_outdated: bool,
1166
- ) -> None:
1167
- """Deploy in Tinybird pushing resources changed from last git commit deployed.
1168
-
1169
- Usage: tb deploy
1170
- """
1171
- try:
1172
- ignore_sql_errors = FeatureFlags.ignore_sql_errors()
1173
- context.disable_template_security_validation.set(True)
1174
-
1175
- is_internal = has_internal_datafiles(folder)
1176
-
1177
- client: TinyB = ctx.ensure_object(dict)["client"]
1178
- config = CLIConfig.get_project_config()
1179
- workspaces: List[Dict[str, Any]] = (await client.user_workspaces_and_branches()).get("workspaces", [])
1180
- current_ws: Dict[str, Any] = next(
1181
- (workspace for workspace in workspaces if config and workspace.get("id", ".") == config.get("id", "..")), {}
1182
- )
1183
-
1184
- semver = config.get("semver")
1185
- auto_promote = getenv_bool("TB_AUTO_PROMOTE", True)
1186
- release = current_ws.get("release", {})
1187
- current_semver: Optional[str] = None
1188
- if release and isinstance(release, dict):
1189
- current_semver = release.get("semver")
1190
-
1191
- if not current_semver:
1192
- click.echo(FeedbackManager.error_init_release(workspace=current_ws.get("name")))
1193
- sys.exit(1)
1194
-
1195
- release_created = False
1196
- new_release = False
1197
- check_backfill_required = False
1198
-
1199
- # semver is now optional, if not sent force the bump
1200
- if not semver:
1201
- semver = current_semver
1202
- else:
1203
- click.echo(FeedbackManager.warning_deprecated_releases())
1204
-
1205
- if semver and current_semver:
1206
- new_version = version.parse(semver.split("-snapshot")[0])
1207
- current_version = version.parse(current_semver.split("-snapshot")[0])
1208
- show_feedback = new_version != current_version
1209
- if show_feedback:
1210
- if dry_run:
1211
- click.echo(
1212
- FeedbackManager.info_dry_releases_detected(current_semver=current_version, semver=new_version)
1213
- )
1214
- else:
1215
- click.echo(
1216
- FeedbackManager.info_releases_detected(current_semver=current_version, semver=new_version)
1217
- )
1218
-
1219
- if new_version == current_version:
1220
- if current_version.post is None:
1221
- semver = str(current_version) + "-1"
1222
- new_version = version.Version(semver)
1223
- else:
1224
- semver = f"{current_version.major}.{current_version.minor}.{current_version.micro}-{current_version.post + 1}"
1225
- new_version = version.Version(semver)
1226
-
1227
- if is_post_semver(new_version, current_version):
1228
- if show_feedback:
1229
- if dry_run:
1230
- click.echo(FeedbackManager.info_dry_local_release(version=new_version))
1231
- else:
1232
- click.echo(FeedbackManager.info_local_release(version=new_version))
1233
- yes = True
1234
- auto_promote = False
1235
- new_release = False
1236
- elif is_major_semver(new_version, current_version):
1237
- if show_feedback:
1238
- if dry_run:
1239
- click.echo(FeedbackManager.info_dry_major_release(version=new_version))
1240
- else:
1241
- click.echo(FeedbackManager.info_major_release(version=new_version))
1242
- auto_promote = False
1243
- new_release = True
1244
- else:
1245
- # allows TB_AUTO_PROMOTE=0 so release is left in preview
1246
- auto_promote = getenv_bool("TB_AUTO_PROMOTE", True)
1247
- if auto_promote:
1248
- if show_feedback:
1249
- if dry_run:
1250
- click.echo(
1251
- FeedbackManager.info_dry_minor_patch_release_with_autopromote(version=new_version)
1252
- )
1253
- else:
1254
- click.echo(FeedbackManager.info_minor_patch_release_with_autopromote(version=new_version))
1255
- else:
1256
- if show_feedback:
1257
- if dry_run:
1258
- click.echo(FeedbackManager.info_dry_minor_patch_release_no_autopromote(version=new_version))
1259
- else:
1260
- click.echo(FeedbackManager.info_minor_patch_release_no_autopromote(version=new_version))
1261
- new_release = True
1262
-
1263
- if new_release:
1264
- release_created = True
1265
- fork_downstream = True
1266
- # allows TB_CHECK_BACKFILL_REQUIRED=0 so it is not checked
1267
- check_backfill_required = getenv_bool("TB_CHECK_BACKFILL_REQUIRED", True)
1268
- try:
1269
- tb_client = create_tb_client(ctx)
1270
- await folder_push(
1271
- tb_client=tb_client,
1272
- dry_run=dry_run,
1273
- check=False,
1274
- push_deps=True,
1275
- debug=debug,
1276
- force=True,
1277
- git_release=True,
1278
- override_datasource=override_datasource,
1279
- populate=populate,
1280
- populate_subset=subset,
1281
- populate_condition=sql_condition,
1282
- unlink_on_populate_error=unlink_on_populate_error,
1283
- upload_fixtures=False,
1284
- wait=wait,
1285
- ignore_sql_errors=ignore_sql_errors,
1286
- skip_confirmation=yes,
1287
- workspace_map=dict(workspace_map),
1288
- workspace_lib_paths=workspace,
1289
- run_tests=False,
1290
- folder=folder,
1291
- config=config,
1292
- user_token=user_token,
1293
- fork_downstream=fork_downstream,
1294
- fork=fork,
1295
- is_internal=is_internal,
1296
- release_created=release_created,
1297
- auto_promote=auto_promote,
1298
- check_backfill_required=check_backfill_required,
1299
- use_main=use_main,
1300
- check_outdated=not skip_head_outdated,
1301
- )
1302
- except Exception as e:
1303
- if release_created and not dry_run:
1304
- await client.release_failed(config["id"], semver)
1305
- raise e
1306
-
1307
- if release_created:
1308
- try:
1309
- if not dry_run:
1310
- await client.release_preview(config["id"], semver)
1311
- click.echo(FeedbackManager.success_release_preview(semver=semver))
1312
- else:
1313
- click.echo(FeedbackManager.success_dry_release_preview(semver=semver))
1314
- if auto_promote:
1315
- try:
1316
- force_remove = getenv_bool("TB_FORCE_REMOVE_OLDEST_ROLLBACK", False)
1317
- if dry_run:
1318
- click.echo(FeedbackManager.warning_dry_remove_oldest_rollback(semver=semver))
1319
- else:
1320
- click.echo(FeedbackManager.warning_remove_oldest_rollback(semver=semver))
1321
- try:
1322
- await remove_release(dry_run, config, OLDEST_ROLLBACK, client, force=force_remove)
1323
- except Exception as e:
1324
- click.echo(FeedbackManager.error_remove_oldest_rollback(error=str(e), semver=semver))
1325
- sys.exit(1)
1326
-
1327
- if not dry_run:
1328
- release = await client.release_promote(config["id"], semver)
1329
- click.echo(FeedbackManager.success_release_promote(semver=semver))
1330
- click.echo(FeedbackManager.success_git_release(release_commit=release["commit"]))
1331
- else:
1332
- click.echo(FeedbackManager.success_dry_release_promote(semver=semver))
1333
- click.echo(FeedbackManager.success_dry_git_release(release_commit=release["commit"]))
1334
- except Exception as e:
1335
- raise CLIException(FeedbackManager.error_exception(error=str(e)))
1336
- except Exception as e:
1337
- raise CLIException(FeedbackManager.error_exception(error=str(e)))
1338
-
1339
- if not new_release:
1340
- if dry_run:
1341
- if show_feedback:
1342
- click.echo(FeedbackManager.success_dry_release_update(semver=current_semver, new_semver=semver))
1343
- else:
1344
- client.semver = None
1345
- await client.update_release_semver(config["id"], current_semver, semver)
1346
- if show_feedback:
1347
- click.echo(FeedbackManager.success_release_update(semver=current_semver, new_semver=semver))
1348
- except AuthNoTokenException:
1349
- raise
1350
- except Exception as e:
1351
- raise CLIException(str(e))
523
+ click.secho = __old_click_secho