tinybird 0.0.1.dev95__tar.gz → 0.0.1.dev96__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of tinybird might be problematic. Click here for more details.

Files changed (108) hide show
  1. {tinybird-0.0.1.dev95 → tinybird-0.0.1.dev96}/PKG-INFO +1 -1
  2. {tinybird-0.0.1.dev95 → tinybird-0.0.1.dev96}/tinybird/prompts.py +29 -1
  3. {tinybird-0.0.1.dev95 → tinybird-0.0.1.dev96}/tinybird/sql_template.py +84 -0
  4. {tinybird-0.0.1.dev95 → tinybird-0.0.1.dev96}/tinybird/tb/__cli__.py +2 -2
  5. {tinybird-0.0.1.dev95 → tinybird-0.0.1.dev96}/tinybird/tb/modules/build.py +11 -0
  6. {tinybird-0.0.1.dev95 → tinybird-0.0.1.dev96}/tinybird/tb/modules/cicd.py +7 -0
  7. {tinybird-0.0.1.dev95 → tinybird-0.0.1.dev96}/tinybird/tb/modules/cli.py +42 -3
  8. {tinybird-0.0.1.dev95 → tinybird-0.0.1.dev96}/tinybird/tb/modules/create.py +25 -5
  9. {tinybird-0.0.1.dev95 → tinybird-0.0.1.dev96}/tinybird/tb/modules/datafile/common.py +2 -0
  10. {tinybird-0.0.1.dev95 → tinybird-0.0.1.dev96}/tinybird/tb/modules/datafile/parse_datasource.py +2 -1
  11. {tinybird-0.0.1.dev95 → tinybird-0.0.1.dev96}/tinybird/tb/modules/datafile/parse_pipe.py +2 -1
  12. {tinybird-0.0.1.dev95 → tinybird-0.0.1.dev96}/tinybird/tb/modules/infra.py +37 -3
  13. {tinybird-0.0.1.dev95 → tinybird-0.0.1.dev96}/tinybird/tb/modules/mock.py +4 -4
  14. {tinybird-0.0.1.dev95 → tinybird-0.0.1.dev96}/tinybird/tb/modules/project.py +4 -0
  15. {tinybird-0.0.1.dev95 → tinybird-0.0.1.dev96}/tinybird/tb/modules/watch.py +1 -1
  16. {tinybird-0.0.1.dev95 → tinybird-0.0.1.dev96}/tinybird.egg-info/PKG-INFO +1 -1
  17. {tinybird-0.0.1.dev95 → tinybird-0.0.1.dev96}/setup.cfg +0 -0
  18. {tinybird-0.0.1.dev95 → tinybird-0.0.1.dev96}/tinybird/__cli__.py +0 -0
  19. {tinybird-0.0.1.dev95 → tinybird-0.0.1.dev96}/tinybird/ch_utils/constants.py +0 -0
  20. {tinybird-0.0.1.dev95 → tinybird-0.0.1.dev96}/tinybird/ch_utils/engine.py +0 -0
  21. {tinybird-0.0.1.dev95 → tinybird-0.0.1.dev96}/tinybird/check_pypi.py +0 -0
  22. {tinybird-0.0.1.dev95 → tinybird-0.0.1.dev96}/tinybird/client.py +0 -0
  23. {tinybird-0.0.1.dev95 → tinybird-0.0.1.dev96}/tinybird/config.py +0 -0
  24. {tinybird-0.0.1.dev95 → tinybird-0.0.1.dev96}/tinybird/connectors.py +0 -0
  25. {tinybird-0.0.1.dev95 → tinybird-0.0.1.dev96}/tinybird/context.py +0 -0
  26. {tinybird-0.0.1.dev95 → tinybird-0.0.1.dev96}/tinybird/datafile.py +0 -0
  27. {tinybird-0.0.1.dev95 → tinybird-0.0.1.dev96}/tinybird/datatypes.py +0 -0
  28. {tinybird-0.0.1.dev95 → tinybird-0.0.1.dev96}/tinybird/feedback_manager.py +0 -0
  29. {tinybird-0.0.1.dev95 → tinybird-0.0.1.dev96}/tinybird/git_settings.py +0 -0
  30. {tinybird-0.0.1.dev95 → tinybird-0.0.1.dev96}/tinybird/sql.py +0 -0
  31. {tinybird-0.0.1.dev95 → tinybird-0.0.1.dev96}/tinybird/sql_template_fmt.py +0 -0
  32. {tinybird-0.0.1.dev95 → tinybird-0.0.1.dev96}/tinybird/sql_toolset.py +0 -0
  33. {tinybird-0.0.1.dev95 → tinybird-0.0.1.dev96}/tinybird/syncasync.py +0 -0
  34. {tinybird-0.0.1.dev95 → tinybird-0.0.1.dev96}/tinybird/tb/cli.py +0 -0
  35. {tinybird-0.0.1.dev95 → tinybird-0.0.1.dev96}/tinybird/tb/modules/auth.py +0 -0
  36. {tinybird-0.0.1.dev95 → tinybird-0.0.1.dev96}/tinybird/tb/modules/common.py +0 -0
  37. {tinybird-0.0.1.dev95 → tinybird-0.0.1.dev96}/tinybird/tb/modules/config.py +0 -0
  38. {tinybird-0.0.1.dev95 → tinybird-0.0.1.dev96}/tinybird/tb/modules/connection.py +0 -0
  39. {tinybird-0.0.1.dev95 → tinybird-0.0.1.dev96}/tinybird/tb/modules/copy.py +0 -0
  40. {tinybird-0.0.1.dev95 → tinybird-0.0.1.dev96}/tinybird/tb/modules/datafile/build.py +0 -0
  41. {tinybird-0.0.1.dev95 → tinybird-0.0.1.dev96}/tinybird/tb/modules/datafile/build_common.py +0 -0
  42. {tinybird-0.0.1.dev95 → tinybird-0.0.1.dev96}/tinybird/tb/modules/datafile/build_datasource.py +0 -0
  43. {tinybird-0.0.1.dev95 → tinybird-0.0.1.dev96}/tinybird/tb/modules/datafile/build_pipe.py +0 -0
  44. {tinybird-0.0.1.dev95 → tinybird-0.0.1.dev96}/tinybird/tb/modules/datafile/diff.py +0 -0
  45. {tinybird-0.0.1.dev95 → tinybird-0.0.1.dev96}/tinybird/tb/modules/datafile/exceptions.py +0 -0
  46. {tinybird-0.0.1.dev95 → tinybird-0.0.1.dev96}/tinybird/tb/modules/datafile/fixture.py +0 -0
  47. {tinybird-0.0.1.dev95 → tinybird-0.0.1.dev96}/tinybird/tb/modules/datafile/format_common.py +0 -0
  48. {tinybird-0.0.1.dev95 → tinybird-0.0.1.dev96}/tinybird/tb/modules/datafile/format_datasource.py +0 -0
  49. {tinybird-0.0.1.dev95 → tinybird-0.0.1.dev96}/tinybird/tb/modules/datafile/format_pipe.py +0 -0
  50. {tinybird-0.0.1.dev95 → tinybird-0.0.1.dev96}/tinybird/tb/modules/datafile/pipe_checker.py +0 -0
  51. {tinybird-0.0.1.dev95 → tinybird-0.0.1.dev96}/tinybird/tb/modules/datafile/playground.py +0 -0
  52. {tinybird-0.0.1.dev95 → tinybird-0.0.1.dev96}/tinybird/tb/modules/datafile/pull.py +0 -0
  53. {tinybird-0.0.1.dev95 → tinybird-0.0.1.dev96}/tinybird/tb/modules/datasource.py +0 -0
  54. {tinybird-0.0.1.dev95 → tinybird-0.0.1.dev96}/tinybird/tb/modules/deployment.py +0 -0
  55. {tinybird-0.0.1.dev95 → tinybird-0.0.1.dev96}/tinybird/tb/modules/endpoint.py +0 -0
  56. {tinybird-0.0.1.dev95 → tinybird-0.0.1.dev96}/tinybird/tb/modules/exceptions.py +0 -0
  57. {tinybird-0.0.1.dev95 → tinybird-0.0.1.dev96}/tinybird/tb/modules/feedback_manager.py +0 -0
  58. {tinybird-0.0.1.dev95 → tinybird-0.0.1.dev96}/tinybird/tb/modules/fmt.py +0 -0
  59. {tinybird-0.0.1.dev95 → tinybird-0.0.1.dev96}/tinybird/tb/modules/job.py +0 -0
  60. {tinybird-0.0.1.dev95 → tinybird-0.0.1.dev96}/tinybird/tb/modules/llm.py +0 -0
  61. {tinybird-0.0.1.dev95 → tinybird-0.0.1.dev96}/tinybird/tb/modules/llm_utils.py +0 -0
  62. {tinybird-0.0.1.dev95 → tinybird-0.0.1.dev96}/tinybird/tb/modules/local.py +0 -0
  63. {tinybird-0.0.1.dev95 → tinybird-0.0.1.dev96}/tinybird/tb/modules/local_common.py +0 -0
  64. {tinybird-0.0.1.dev95 → tinybird-0.0.1.dev96}/tinybird/tb/modules/login.py +0 -0
  65. {tinybird-0.0.1.dev95 → tinybird-0.0.1.dev96}/tinybird/tb/modules/logout.py +0 -0
  66. {tinybird-0.0.1.dev95 → tinybird-0.0.1.dev96}/tinybird/tb/modules/materialization.py +0 -0
  67. {tinybird-0.0.1.dev95 → tinybird-0.0.1.dev96}/tinybird/tb/modules/open.py +0 -0
  68. {tinybird-0.0.1.dev95 → tinybird-0.0.1.dev96}/tinybird/tb/modules/pipe.py +0 -0
  69. {tinybird-0.0.1.dev95 → tinybird-0.0.1.dev96}/tinybird/tb/modules/playground.py +0 -0
  70. {tinybird-0.0.1.dev95 → tinybird-0.0.1.dev96}/tinybird/tb/modules/regions.py +0 -0
  71. {tinybird-0.0.1.dev95 → tinybird-0.0.1.dev96}/tinybird/tb/modules/secret.py +0 -0
  72. {tinybird-0.0.1.dev95 → tinybird-0.0.1.dev96}/tinybird/tb/modules/shell.py +0 -0
  73. {tinybird-0.0.1.dev95 → tinybird-0.0.1.dev96}/tinybird/tb/modules/table.py +0 -0
  74. {tinybird-0.0.1.dev95 → tinybird-0.0.1.dev96}/tinybird/tb/modules/tag.py +0 -0
  75. {tinybird-0.0.1.dev95 → tinybird-0.0.1.dev96}/tinybird/tb/modules/telemetry.py +0 -0
  76. {tinybird-0.0.1.dev95 → tinybird-0.0.1.dev96}/tinybird/tb/modules/test.py +0 -0
  77. {tinybird-0.0.1.dev95 → tinybird-0.0.1.dev96}/tinybird/tb/modules/tinyunit/tinyunit.py +0 -0
  78. {tinybird-0.0.1.dev95 → tinybird-0.0.1.dev96}/tinybird/tb/modules/tinyunit/tinyunit_lib.py +0 -0
  79. {tinybird-0.0.1.dev95 → tinybird-0.0.1.dev96}/tinybird/tb/modules/token.py +0 -0
  80. {tinybird-0.0.1.dev95 → tinybird-0.0.1.dev96}/tinybird/tb/modules/workspace.py +0 -0
  81. {tinybird-0.0.1.dev95 → tinybird-0.0.1.dev96}/tinybird/tb/modules/workspace_members.py +0 -0
  82. {tinybird-0.0.1.dev95 → tinybird-0.0.1.dev96}/tinybird/tb_cli.py +0 -0
  83. {tinybird-0.0.1.dev95 → tinybird-0.0.1.dev96}/tinybird/tb_cli_modules/auth.py +0 -0
  84. {tinybird-0.0.1.dev95 → tinybird-0.0.1.dev96}/tinybird/tb_cli_modules/branch.py +0 -0
  85. {tinybird-0.0.1.dev95 → tinybird-0.0.1.dev96}/tinybird/tb_cli_modules/cicd.py +0 -0
  86. {tinybird-0.0.1.dev95 → tinybird-0.0.1.dev96}/tinybird/tb_cli_modules/cli.py +0 -0
  87. {tinybird-0.0.1.dev95 → tinybird-0.0.1.dev96}/tinybird/tb_cli_modules/common.py +0 -0
  88. {tinybird-0.0.1.dev95 → tinybird-0.0.1.dev96}/tinybird/tb_cli_modules/config.py +0 -0
  89. {tinybird-0.0.1.dev95 → tinybird-0.0.1.dev96}/tinybird/tb_cli_modules/connection.py +0 -0
  90. {tinybird-0.0.1.dev95 → tinybird-0.0.1.dev96}/tinybird/tb_cli_modules/datasource.py +0 -0
  91. {tinybird-0.0.1.dev95 → tinybird-0.0.1.dev96}/tinybird/tb_cli_modules/exceptions.py +0 -0
  92. {tinybird-0.0.1.dev95 → tinybird-0.0.1.dev96}/tinybird/tb_cli_modules/fmt.py +0 -0
  93. {tinybird-0.0.1.dev95 → tinybird-0.0.1.dev96}/tinybird/tb_cli_modules/job.py +0 -0
  94. {tinybird-0.0.1.dev95 → tinybird-0.0.1.dev96}/tinybird/tb_cli_modules/pipe.py +0 -0
  95. {tinybird-0.0.1.dev95 → tinybird-0.0.1.dev96}/tinybird/tb_cli_modules/regions.py +0 -0
  96. {tinybird-0.0.1.dev95 → tinybird-0.0.1.dev96}/tinybird/tb_cli_modules/tag.py +0 -0
  97. {tinybird-0.0.1.dev95 → tinybird-0.0.1.dev96}/tinybird/tb_cli_modules/telemetry.py +0 -0
  98. {tinybird-0.0.1.dev95 → tinybird-0.0.1.dev96}/tinybird/tb_cli_modules/test.py +0 -0
  99. {tinybird-0.0.1.dev95 → tinybird-0.0.1.dev96}/tinybird/tb_cli_modules/tinyunit/tinyunit.py +0 -0
  100. {tinybird-0.0.1.dev95 → tinybird-0.0.1.dev96}/tinybird/tb_cli_modules/tinyunit/tinyunit_lib.py +0 -0
  101. {tinybird-0.0.1.dev95 → tinybird-0.0.1.dev96}/tinybird/tb_cli_modules/workspace.py +0 -0
  102. {tinybird-0.0.1.dev95 → tinybird-0.0.1.dev96}/tinybird/tb_cli_modules/workspace_members.py +0 -0
  103. {tinybird-0.0.1.dev95 → tinybird-0.0.1.dev96}/tinybird/tornado_template.py +0 -0
  104. {tinybird-0.0.1.dev95 → tinybird-0.0.1.dev96}/tinybird.egg-info/SOURCES.txt +0 -0
  105. {tinybird-0.0.1.dev95 → tinybird-0.0.1.dev96}/tinybird.egg-info/dependency_links.txt +0 -0
  106. {tinybird-0.0.1.dev95 → tinybird-0.0.1.dev96}/tinybird.egg-info/entry_points.txt +0 -0
  107. {tinybird-0.0.1.dev95 → tinybird-0.0.1.dev96}/tinybird.egg-info/requires.txt +0 -0
  108. {tinybird-0.0.1.dev95 → tinybird-0.0.1.dev96}/tinybird.egg-info/top_level.txt +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.2
2
2
  Name: tinybird
3
- Version: 0.0.1.dev95
3
+ Version: 0.0.1.dev96
4
4
  Summary: Tinybird Command Line Tool
5
5
  Home-page: https://www.tinybird.co/docs/cli/introduction.html
6
6
  Author: Tinybird
@@ -419,13 +419,15 @@ You are a Tinybird expert. You will be given a prompt to generate new or update
419
419
  {pipe_example}
420
420
  {copy_pipe_instructions}
421
421
  {materialized_pipe_instructions}
422
+ {connection_instructions}
423
+ {connection_example}
422
424
 
423
425
  {feedback_history}
424
426
 
425
427
  Use the following format to generate the response and do not wrap it in any other text, including the <response> tag.
426
428
  <response>
427
429
  <resource>
428
- <type>[datasource or pipe]</type>
430
+ <type>[datasource or pipe or connection]</type>
429
431
  <name>[resource name here]</name>
430
432
  <content>[resource content here]</content>
431
433
  </resource>
@@ -440,6 +442,8 @@ Use the following format to generate the response and do not wrap it in any othe
440
442
  pipe_example=pipe_example,
441
443
  copy_pipe_instructions=copy_pipe_instructions,
442
444
  materialized_pipe_instructions=materialized_pipe_instructions,
445
+ connection_instructions=connection_instructions,
446
+ connection_example=connection_example,
443
447
  feedback_history=feedback_history,
444
448
  )
445
449
 
@@ -672,6 +676,25 @@ ENGINE_SORTING_KEY "date, dimension_1, dimension_2, ..."
672
676
  </target_datasource_content>
673
677
  """
674
678
 
679
+ connection_instructions = """
680
+ <connection_file_instructions>
681
+ - Content cannot be empty.
682
+ - The connection names must be unique.
683
+ - No indentation is allowed for property names
684
+ - We only support kafka connections for now
685
+ </connection_file_instructions>
686
+ """
687
+
688
+ connection_example = """
689
+ <connection_content>
690
+ TYPE kafka
691
+ KAFKA_BOOTSTRAP_SERVERS {{ tb_secret("PRODUCTION_KAFKA_SERVERS", "localhost:9092") }}
692
+ KAFKA_SECURITY_PROTOCOL SASL_SSL
693
+ KAFKA_SASL_MECHANISM PLAIN
694
+ KAFKA_KEY {{ tb_secret("PRODUCTION_KAFKA_USERNAME", "") }}
695
+ KAFKA_SECRET {{ tb_secret("PRODUCTION_KAFKA_PASSWORD", "") }}
696
+ </connection_content>
697
+ """
675
698
 
676
699
  datasource_instructions = """
677
700
  <datasource_file_instructions>
@@ -805,6 +828,7 @@ When you need to work with resources or data in cloud, add always the --cloud fl
805
828
  </command_calling>
806
829
  <development_instructions>
807
830
  - When asking to create a tinybird data project, if the needed folders are not already created, use the following structure:
831
+ ├── connections
808
832
  ├── copies
809
833
  ├── datasources
810
834
  ├── endpoints
@@ -838,6 +862,8 @@ Follow these instructions when creating or updating .pipe files:
838
862
  {pipe_example}
839
863
  {copy_pipe_instructions}
840
864
  {materialized_pipe_instructions}
865
+ {connection_instructions}
866
+ {connection_example}
841
867
  </pipe_file_instructions>
842
868
  <test_file_instructions>
843
869
  Follow these instructions when creating or updating .yaml files for tests:
@@ -858,6 +884,8 @@ Follow these instructions when evolving a datasource schema:
858
884
  materialized_pipe_instructions=materialized_pipe_instructions,
859
885
  test_instructions=test_instructions,
860
886
  deployment_instructions=deployment_instructions,
887
+ connection_instructions=connection_instructions,
888
+ connection_example=connection_example,
861
889
  )
862
890
 
863
891
 
@@ -2399,3 +2399,87 @@ def extract_variables_from_sql(sql: str, params: List[Dict[str, Any]]) -> Dict[s
2399
2399
  return {}
2400
2400
 
2401
2401
  return defaults
2402
+
2403
+
2404
+ def render_template_with_secrets(name: str, content: str, secrets: Optional[Dict[str, str]] = None) -> str:
2405
+ """Renders a template with secrets, allowing for default values.
2406
+
2407
+ Args:
2408
+ name: The name of the template
2409
+ content: The template content
2410
+ secrets: A dictionary mapping secret names to their values
2411
+
2412
+ Returns:
2413
+ The rendered template
2414
+
2415
+ Examples:
2416
+ >>> render_template_with_secrets(
2417
+ ... "my_kafka_connection",
2418
+ ... "KAFKA_BOOTSTRAP_SERVERS {{ tb_secret('PRODUCTION_KAFKA_SERVERS', 'localhost:9092') }}",
2419
+ ... secrets = {'PRODUCTION_KAFKA_SERVERS': 'server1:9092,server2:9092'}
2420
+ ... )
2421
+ 'KAFKA_BOOTSTRAP_SERVERS server1:9092,server2:9092'
2422
+
2423
+ >>> render_template_with_secrets(
2424
+ ... "my_kafka_connection",
2425
+ ... "KAFKA_BOOTSTRAP_SERVERS {{ tb_secret('MISSING_SECRET', 'localhost:9092') }}",
2426
+ ... secrets = {}
2427
+ ... )
2428
+ 'KAFKA_BOOTSTRAP_SERVERS localhost:9092'
2429
+
2430
+ >>> render_template_with_secrets(
2431
+ ... "my_kafka_connection",
2432
+ ... "KAFKA_BOOTSTRAP_SERVERS {{ tb_secret('MISSING_SECRET') }}",
2433
+ ... secrets = {}
2434
+ ... )
2435
+ Traceback (most recent call last):
2436
+ ...
2437
+ tinybird.sql_template.SQLTemplateException: Template Syntax Error: Cannot access secret 'MISSING_SECRET'. Check the secret exists in the Workspace and the token has the required scope.
2438
+ """
2439
+ if not secrets:
2440
+ secrets = {}
2441
+
2442
+ def tb_secret(secret_name: str, default: Optional[str] = None) -> str:
2443
+ """Get a secret value with an optional default.
2444
+
2445
+ Args:
2446
+ secret_name: The name of the secret to retrieve
2447
+ default: The default value to use if the secret is not found
2448
+
2449
+ Returns:
2450
+ The secret value or default
2451
+
2452
+ Raises:
2453
+ SQLTemplateException: If the secret is not found and no default is provided
2454
+ """
2455
+ if secret_name in secrets:
2456
+ return secrets[secret_name]
2457
+ elif default is not None:
2458
+ return default
2459
+ else:
2460
+ raise SQLTemplateException(
2461
+ f"Cannot access secret '{secret_name}'. Check the secret exists in the Workspace and the token has the required scope."
2462
+ )
2463
+
2464
+ # Create the template
2465
+ t = Template(content, name=name)
2466
+
2467
+ try:
2468
+ # Create namespace with our tb_secret function
2469
+ namespace = {"tb_secret": tb_secret}
2470
+
2471
+ # Generate the template without all the extra processing
2472
+ # This directly uses the underlying _generate method of the Template class
2473
+ result = t.generate(**namespace)
2474
+
2475
+ # Convert the result to string
2476
+ if isinstance(result, bytes):
2477
+ return result.decode("utf-8")
2478
+
2479
+ return str(result)
2480
+ except SQLTemplateCustomError as e:
2481
+ raise e
2482
+ except SQLTemplateException as e:
2483
+ raise e
2484
+ except Exception as e:
2485
+ raise SQLTemplateException(f"Error rendering template with secrets: {str(e)}")
@@ -4,5 +4,5 @@ __description__ = 'Tinybird Command Line Tool'
4
4
  __url__ = 'https://www.tinybird.co/docs/cli/introduction.html'
5
5
  __author__ = 'Tinybird'
6
6
  __author_email__ = 'support@tinybird.co'
7
- __version__ = '0.0.1.dev95'
8
- __revision__ = 'dc3767a'
7
+ __version__ = '0.0.1.dev96'
8
+ __revision__ = 'f1cd799'
@@ -97,6 +97,8 @@ def build_project(project: Project, tb_client: TinyB, file_changed: Optional[str
97
97
  build = result.get("build")
98
98
  datasources = build.get("new_datasource_names", [])
99
99
  pipes = build.get("new_pipe_names", [])
100
+ connections = build.get("new_data_connector_names", [])
101
+
100
102
  if not file_changed:
101
103
  for ds in datasources:
102
104
  ds_path_str: Optional[str] = next(
@@ -115,6 +117,15 @@ def build_project(project: Project, tb_client: TinyB, file_changed: Optional[str
115
117
  pipe_path_str = pipe_path_str.replace(f"{project.folder}/", "")
116
118
  click.echo(FeedbackManager.info(message=f"✓ {pipe_path_str} created"))
117
119
 
120
+ for connection in connections:
121
+ connection_name = connection
122
+ connection_path_str: Optional[str] = next(
123
+ (p for p in project_files if p.endswith(connection_name + ".connection")), None
124
+ )
125
+ if connection_path_str:
126
+ connection_path_str = connection_path_str.replace(f"{project.folder}/", "")
127
+ click.echo(FeedbackManager.info(message=f"✓ {connection_path_str} created"))
128
+
118
129
  try:
119
130
  for filename in project_files:
120
131
  if filename.endswith(".datasource"):
@@ -31,6 +31,10 @@ on:
31
31
 
32
32
  concurrency: ${{! github.workflow }}-${{! github.event.pull_request.number }}
33
33
 
34
+ env:
35
+ TINYBIRD_HOST: ${{ secrets.TINYBIRD_HOST }}
36
+ TINYBIRD_TOKEN: ${{ secrets.TINYBIRD_TOKEN }}
37
+
34
38
  jobs:
35
39
  ci:
36
40
  runs-on: ubuntu-latest
@@ -50,6 +54,8 @@ jobs:
50
54
  run: tb build
51
55
  - name: Test project
52
56
  run: tb test run
57
+ - name: Deployment check
58
+ run: tb --cloud --host ${{ TINYBIRD_HOST }} --token ${{ TINYBIRD_TOKEN }} deploy --check
53
59
  """
54
60
 
55
61
 
@@ -82,6 +88,7 @@ tinybird_ci_workflow:
82
88
  - cd $CI_PROJECT_DIR/{{ data_project_dir }}
83
89
  - tb build
84
90
  - tb test run
91
+ - tb --cloud --host ${{ TINYBIRD_HOST }} --token ${{ TINYBIRD_TOKEN }} deploy --check
85
92
  services:
86
93
  - name: tinybirdco/tinybird-local:beta
87
94
  alias: tinybird-local
@@ -7,8 +7,11 @@ import json
7
7
  import logging
8
8
  import os
9
9
  import sys
10
+ from os import getcwd
11
+ from pathlib import Path
10
12
  from typing import Any, Callable, Dict, List, Optional, Tuple, Union
11
13
 
14
+ import aiofiles
12
15
  import click
13
16
  import humanfriendly
14
17
  from click import Context
@@ -18,7 +21,6 @@ from tinybird.client import (
18
21
  AuthNoTokenException,
19
22
  TinyB,
20
23
  )
21
- from tinybird.config import get_config
22
24
  from tinybird.tb import __cli__
23
25
  from tinybird.tb.modules.common import (
24
26
  CatchAuthExceptions,
@@ -53,6 +55,7 @@ VERSION = f"{__cli__.__version__} (rev {__cli__.__revision__})"
53
55
  help="Prints internal representation, can be combined with any command to get more information.",
54
56
  )
55
57
  @click.option("--token", help="Use auth token, defaults to TB_TOKEN envvar, then to the .tinyb file.")
58
+ @click.option("--user-token", help="Use user token, defaults to TB_USER_TOKEN envvar, then to the .tinyb file.")
56
59
  @click.option("--host", help="Use custom host, defaults to TB_HOST envvar, then to https://api.tinybird.co")
57
60
  @click.option("--show-tokens", is_flag=True, default=False, help="Enable the output of tokens.")
58
61
  @click.option("--cloud/--local", is_flag=True, default=False, help="Run against cloud or local.")
@@ -65,6 +68,7 @@ async def cli(
65
68
  ctx: Context,
66
69
  debug: bool,
67
70
  token: str,
71
+ user_token: str,
68
72
  host: str,
69
73
  show_tokens: bool,
70
74
  cloud: bool,
@@ -92,7 +96,9 @@ async def cli(
92
96
  config_temp.set_token(token)
93
97
  if host:
94
98
  config_temp.set_host(host)
95
- if token or host:
99
+ if user_token:
100
+ config_temp.set_user_token(user_token)
101
+ if token or host or user_token:
96
102
  await try_update_config_with_remote(config_temp, auto_persist=False, raise_on_errors=False)
97
103
 
98
104
  # Overwrite token and host with env vars manually, without resorting to click.
@@ -104,8 +110,10 @@ async def cli(
104
110
  token = os.environ.get("TB_TOKEN", "")
105
111
  if not host and "TB_HOST" in os.environ:
106
112
  host = os.environ.get("TB_HOST", "")
113
+ if not user_token and "TB_USER_TOKEN" in os.environ:
114
+ user_token = os.environ.get("TB_USER_TOKEN", "")
107
115
 
108
- config = await get_config(host, token, config_file=config_temp._path)
116
+ config = await get_config(host, token, user_token=user_token, config_file=config_temp._path)
109
117
  client = _get_tb_client(config.get("token", None), config["host"])
110
118
  folder = os.path.join(config_temp._path.replace(".tinyb", ""), config.get("cwd", os.getcwd()))
111
119
  project = Project(folder=folder)
@@ -422,3 +430,34 @@ def get_target_env(cloud: bool, build: bool) -> str:
422
430
  if build:
423
431
  return "build"
424
432
  return "local"
433
+
434
+
435
+ async def get_config(
436
+ host: str,
437
+ token: Optional[str],
438
+ user_token: Optional[str],
439
+ semver: Optional[str] = None,
440
+ config_file: Optional[str] = None,
441
+ ) -> Dict[str, Any]:
442
+ if host:
443
+ host = host.rstrip("/")
444
+
445
+ config = {}
446
+ try:
447
+ async with aiofiles.open(config_file or Path(getcwd()) / ".tinyb") as file:
448
+ res = await file.read()
449
+ config = json.loads(res)
450
+ except OSError:
451
+ pass
452
+ except json.decoder.JSONDecodeError:
453
+ click.echo(FeedbackManager.error_load_file_config(config_file=config_file))
454
+ return config
455
+
456
+ config["token_passed"] = token
457
+ config["token"] = token or config.get("token", None)
458
+ config["user_token"] = user_token or config.get("user_token", None)
459
+ config["semver"] = semver or config.get("semver", None)
460
+ config["host"] = host or config.get("host", "https://api.europe-west2.gcp.tinybird.co")
461
+ config["workspaces"] = config.get("workspaces", [])
462
+ config["cwd"] = config.get("cwd", getcwd())
463
+ return config
@@ -9,7 +9,7 @@ from tinybird.client import TinyB
9
9
  from tinybird.prompts import create_prompt, mock_prompt, rules_prompt
10
10
  from tinybird.tb.modules.cicd import init_cicd
11
11
  from tinybird.tb.modules.cli import cli
12
- from tinybird.tb.modules.common import _generate_datafile, check_user_token_with_client, coro, generate_datafile
12
+ from tinybird.tb.modules.common import _generate_datafile, coro, generate_datafile
13
13
  from tinybird.tb.modules.config import CLIConfig
14
14
  from tinybird.tb.modules.datafile.fixture import persist_fixture
15
15
  from tinybird.tb.modules.exceptions import CLIException
@@ -44,6 +44,7 @@ async def create(
44
44
  local_client: TinyB = ctx.ensure_object(dict)["client"]
45
45
  project: Project = ctx.ensure_object(dict)["project"]
46
46
  config = CLIConfig.get_project_config()
47
+ ctx_config = ctx.ensure_object(dict)["config"]
47
48
 
48
49
  # If folder is provided, rewrite the config and project folder
49
50
  if folder:
@@ -65,15 +66,14 @@ async def create(
65
66
  folder_path.mkdir()
66
67
 
67
68
  try:
68
- tb_client = config.get_client()
69
+ tb_client = config.get_client(token=ctx_config.get("token"), host=ctx_config.get("host"))
69
70
  user_token: Optional[str] = None
70
71
  created_something = False
71
72
  if prompt:
72
73
  try:
73
- user_token = config.get_user_token()
74
+ user_token = ctx_config.get("user_token")
74
75
  if not user_token:
75
76
  raise CLIException("No user token found")
76
- await check_user_token_with_client(tb_client, token=user_token)
77
77
  except Exception as e:
78
78
  click.echo(
79
79
  FeedbackManager.error(
@@ -144,7 +144,7 @@ async def create(
144
144
  click.echo(FeedbackManager.error(message=f"Error: {str(e)}"))
145
145
 
146
146
 
147
- PROJECT_PATHS = ("datasources", "endpoints", "materializations", "copies", "pipes", "fixtures", "tests")
147
+ PROJECT_PATHS = ("datasources", "endpoints", "materializations", "copies", "pipes", "fixtures", "tests", "connections")
148
148
 
149
149
 
150
150
  def validate_project_structure(folder: str) -> bool:
@@ -248,6 +248,7 @@ TYPE ENDPOINT
248
248
  resources = parse_xml(result, "resource")
249
249
  datasources = []
250
250
  pipes = []
251
+ connections = []
251
252
  for resource_xml in resources:
252
253
  resource_type = extract_xml(resource_xml, "type")
253
254
  name = extract_xml(resource_xml, "name")
@@ -260,6 +261,8 @@ TYPE ENDPOINT
260
261
  datasources.append(resource)
261
262
  elif resource_type.lower() == "pipe":
262
263
  pipes.append(resource)
264
+ elif resource_type.lower() == "connection":
265
+ connections.append(resource)
263
266
 
264
267
  for ds in datasources:
265
268
  content = ds["content"].replace("```", "")
@@ -279,6 +282,12 @@ TYPE ENDPOINT
279
282
  generate_pipe_file(pipe["name"], content, folder)
280
283
  created_any_resource = True
281
284
 
285
+ for conn in connections:
286
+ content = conn["content"].replace("```", "")
287
+ filename = f"{conn['name']}.connection"
288
+ generate_connection_file(conn["name"], content, folder)
289
+ created_any_resource = True
290
+
282
291
  return result, created_any_resource
283
292
 
284
293
 
@@ -333,6 +342,17 @@ def generate_pipe_file(name: str, content: str, folder: str) -> Path:
333
342
  return f.relative_to(folder)
334
343
 
335
344
 
345
+ def generate_connection_file(name: str, content: str, folder: str) -> Path:
346
+ base = Path(folder) / "connections"
347
+ if not base.exists():
348
+ base.mkdir()
349
+ f = base / (f"{name}.connection")
350
+ with open(f"{f}", "w") as file:
351
+ file.write(content)
352
+ click.echo(FeedbackManager.info_file_created(file=f.relative_to(folder)))
353
+ return f.relative_to(folder)
354
+
355
+
336
356
  def create_rules(folder: str, source: str, agent: str):
337
357
  if agent == "cursor":
338
358
  extension = ".cursorrules"
@@ -115,6 +115,7 @@ class DataFileExtensions:
115
115
  PIPE = ".pipe"
116
116
  DATASOURCE = ".datasource"
117
117
  INCL = ".incl"
118
+ CONNECTION = ".connection"
118
119
 
119
120
 
120
121
  class CopyModes:
@@ -1700,6 +1701,7 @@ def get_project_filenames(folder: str, with_vendor=False) -> List[str]:
1700
1701
  f"{folder}/sinks/*.pipe",
1701
1702
  f"{folder}/copies/*.pipe",
1702
1703
  f"{folder}/playgrounds/*.pipe",
1704
+ f"{folder}/connections/*.connection",
1703
1705
  ]
1704
1706
  if with_vendor:
1705
1707
  folders.append(f"{folder}/vendor/**/**/*.datasource")
@@ -1,5 +1,5 @@
1
1
  import os
2
- from typing import Optional
2
+ from typing import List, Optional
3
3
 
4
4
  import click
5
5
 
@@ -21,6 +21,7 @@ def parse_datasource(
21
21
  skip_eval: bool = False,
22
22
  hide_folders: bool = False,
23
23
  add_context_to_datafile_syntax_errors: bool = True,
24
+ secrets: Optional[List[str]] = None,
24
25
  ) -> Datafile:
25
26
  basepath = ""
26
27
  if not content:
@@ -1,5 +1,5 @@
1
1
  import os
2
- from typing import Optional
2
+ from typing import List, Optional
3
3
 
4
4
  import click
5
5
 
@@ -23,6 +23,7 @@ def parse_pipe(
23
23
  skip_eval: bool = False,
24
24
  hide_folders: bool = False,
25
25
  add_context_to_datafile_syntax_errors: bool = True,
26
+ secrets: Optional[List[str]] = None,
26
27
  ) -> Datafile:
27
28
  basepath = ""
28
29
  if not content:
@@ -155,13 +155,13 @@ output "tinybird_dns" {
155
155
  """
156
156
 
157
157
 
158
- @cli.group(context_settings=CONTEXT_SETTINGS)
158
+ @cli.group(context_settings=CONTEXT_SETTINGS, hidden=True)
159
159
  @click.pass_context
160
160
  def infra(ctx: Context) -> None:
161
161
  """Infra commands."""
162
162
 
163
163
 
164
- @infra.command(name="ls")
164
+ @infra.command(name="ls", hidden=True)
165
165
  @click.pass_context
166
166
  async def infra_ls(ctx: Context) -> None:
167
167
  """List infra"""
@@ -455,6 +455,7 @@ def infra_init(
455
455
  click.echo("Failed to create DNS record:")
456
456
  click.echo(apply_result.stderr)
457
457
  else:
458
+ click.echo(apply_result.stdout)
458
459
  click.echo("DNS record created successfully!")
459
460
 
460
461
  # Get the DNS name from Terraform output
@@ -466,7 +467,40 @@ def infra_init(
466
467
 
467
468
  if dns_output.returncode == 0:
468
469
  dns_name = dns_output.stdout.strip().replace('"', "")
469
- click.echo(f"\nYour Tinybird instance is now available at: https://{dns_name}")
470
+ click.echo("\nDNS record created successfully!")
471
+ click.echo(
472
+ "\nWaiting up to 5 minutes for HTTPS endpoint to become available..."
473
+ )
474
+
475
+ import time
476
+
477
+ import requests
478
+
479
+ max_attempts = 30 # 30 attempts * 10 seconds = 5 minutes
480
+ attempt = 0
481
+ while attempt < max_attempts:
482
+ attempt += 1
483
+ try:
484
+ response = requests.get(
485
+ f"https://{dns_name}", allow_redirects=False, timeout=5
486
+ )
487
+ response.raise_for_status()
488
+ click.echo("\n✅ HTTPS endpoint is now accessible!")
489
+ break
490
+ except requests.RequestException:
491
+ if attempt == max_attempts:
492
+ click.echo("\n⚠️ HTTPS endpoint not accessible after 5 minutes")
493
+ click.echo(
494
+ " This might be due to DNS propagation or the Load Balancer provisioning delays"
495
+ )
496
+ click.echo(
497
+ " Please try accessing the URL manually in a few minutes"
498
+ )
499
+ else:
500
+ click.echo(
501
+ f"Attempt {attempt}/{max_attempts}: Not ready yet, waiting 10 seconds..."
502
+ )
503
+ time.sleep(10)
470
504
  else:
471
505
  click.echo(
472
506
  f"\nYour Tinybird instance should be available at: https://tinybird.{domain}"
@@ -6,7 +6,7 @@ import click
6
6
  from tinybird.client import TinyB
7
7
  from tinybird.prompts import mock_prompt
8
8
  from tinybird.tb.modules.cli import cli
9
- from tinybird.tb.modules.common import CLIException, check_user_token_with_client, coro, push_data
9
+ from tinybird.tb.modules.common import CLIException, coro, push_data
10
10
  from tinybird.tb.modules.config import CLIConfig
11
11
  from tinybird.tb.modules.datafile.fixture import persist_fixture, persist_fixture_sql
12
12
  from tinybird.tb.modules.feedback_manager import FeedbackManager
@@ -39,6 +39,7 @@ async def mock(ctx: click.Context, datasource: str, rows: int, prompt: str) -> N
39
39
  try:
40
40
  tb_client: TinyB = ctx.ensure_object(dict)["client"]
41
41
  project: Project = ctx.ensure_object(dict)["project"]
42
+ ctx_config = ctx.ensure_object(dict)["config"]
42
43
  env = ctx.ensure_object(dict)["env"]
43
44
  datasource_path = Path(datasource)
44
45
  datasource_name = datasource
@@ -56,13 +57,12 @@ async def mock(ctx: click.Context, datasource: str, rows: int, prompt: str) -> N
56
57
 
57
58
  datasource_content = datasource_path.read_text()
58
59
  config = CLIConfig.get_project_config()
59
- user_client = config.get_client()
60
- user_token = config.get_user_token()
60
+ user_client = config.get_client(token=ctx_config.get("token"), host=ctx_config.get("host"))
61
+ user_token = ctx_config.get("user_token")
61
62
 
62
63
  try:
63
64
  if not user_token:
64
65
  raise CLIException("No user token found")
65
- await check_user_token_with_client(user_client, token=user_token)
66
66
  except Exception:
67
67
  click.echo(FeedbackManager.error(message="This action requires authentication. Run 'tb login' first."))
68
68
  return
@@ -51,6 +51,10 @@ class Project:
51
51
  def pipes(self) -> List[str]:
52
52
  return sorted([Path(f).stem for f in glob.glob(f"{self.path}/**/*.pipe", recursive=False)])
53
53
 
54
+ @property
55
+ def connections(self) -> List[str]:
56
+ return sorted([Path(f).stem for f in glob.glob(f"{self.path}/**/*.connection", recursive=False)])
57
+
54
58
  def get_pipe_datafile(self, filename: str) -> Optional[Datafile]:
55
59
  try:
56
60
  return parse_pipe(filename)
@@ -22,7 +22,7 @@ from tinybird.tb.modules.shell import Shell
22
22
 
23
23
 
24
24
  class WatchProjectHandler(PatternMatchingEventHandler):
25
- valid_extensions = [".datasource", ".pipe", ".ndjson", ".sql"]
25
+ valid_extensions = [".datasource", ".pipe", "connection", ".ndjson", ".sql"]
26
26
 
27
27
  def __init__(self, shell: Shell, project: Project, process: Callable):
28
28
  self.shell = shell
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.2
2
2
  Name: tinybird
3
- Version: 0.0.1.dev95
3
+ Version: 0.0.1.dev96
4
4
  Summary: Tinybird Command Line Tool
5
5
  Home-page: https://www.tinybird.co/docs/cli/introduction.html
6
6
  Author: Tinybird
File without changes