tinybird 4.6.21.dev0__tar.gz → 4.6.22__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (130) hide show
  1. {tinybird-4.6.21.dev0 → tinybird-4.6.22}/PKG-INFO +16 -1
  2. {tinybird-4.6.21.dev0 → tinybird-4.6.22}/pyproject.toml +41 -26
  3. {tinybird-4.6.21.dev0 → tinybird-4.6.22}/tinybird/client.py +4 -8
  4. {tinybird-4.6.21.dev0 → tinybird-4.6.22}/tinybird/datafile/common.py +9 -1
  5. {tinybird-4.6.21.dev0 → tinybird-4.6.22}/tinybird/datatypes.py +34 -12
  6. {tinybird-4.6.21.dev0 → tinybird-4.6.22}/tinybird/feedback_manager.py +5 -0
  7. {tinybird-4.6.21.dev0 → tinybird-4.6.22}/tinybird/sql_template.py +58 -24
  8. {tinybird-4.6.21.dev0 → tinybird-4.6.22}/tinybird/sql_template_fmt.py +2 -2
  9. {tinybird-4.6.21.dev0 → tinybird-4.6.22}/tinybird/syncasync.py +0 -43
  10. {tinybird-4.6.21.dev0 → tinybird-4.6.22}/tinybird/tb/__cli__.py +2 -2
  11. {tinybird-4.6.21.dev0 → tinybird-4.6.22}/tinybird/tb/client.py +4 -8
  12. {tinybird-4.6.21.dev0 → tinybird-4.6.22}/tinybird/tb/modules/common.py +3 -3
  13. {tinybird-4.6.21.dev0 → tinybird-4.6.22}/tinybird/tb/modules/copy.py +33 -1
  14. {tinybird-4.6.21.dev0 → tinybird-4.6.22}/tinybird/tb/modules/create.py +2 -4
  15. {tinybird-4.6.21.dev0 → tinybird-4.6.22}/tinybird/tb/modules/datafile/build.py +1 -0
  16. {tinybird-4.6.21.dev0 → tinybird-4.6.22}/tinybird/tb/modules/datafile/format_pipe.py +4 -0
  17. {tinybird-4.6.21.dev0 → tinybird-4.6.22}/tinybird/tb/modules/datafile/playground.py +1 -0
  18. {tinybird-4.6.21.dev0 → tinybird-4.6.22}/tinybird/tb/modules/datafile/pull.py +73 -3
  19. {tinybird-4.6.21.dev0 → tinybird-4.6.22}/tinybird/tb/modules/datasource.py +1 -1
  20. {tinybird-4.6.21.dev0 → tinybird-4.6.22}/tinybird/tb/modules/deployment_common.py +4 -6
  21. {tinybird-4.6.21.dev0 → tinybird-4.6.22}/tinybird/tb/modules/exceptions.py +0 -21
  22. {tinybird-4.6.21.dev0 → tinybird-4.6.22}/tinybird/tb/modules/feedback_manager.py +5 -0
  23. {tinybird-4.6.21.dev0 → tinybird-4.6.22}/tinybird/tb_cli_modules/common.py +3 -3
  24. {tinybird-4.6.21.dev0 → tinybird-4.6.22}/tinybird/tb_cli_modules/datasource.py +1 -1
  25. {tinybird-4.6.21.dev0 → tinybird-4.6.22}/tinybird/tb_cli_modules/pipe.py +40 -21
  26. {tinybird-4.6.21.dev0 → tinybird-4.6.22}/tinybird/tornado_template.py +215 -66
  27. tinybird-4.6.22/tinybird/utils/__init__.py +0 -0
  28. tinybird-4.6.22/tinybird/utils/bools.py +19 -0
  29. {tinybird-4.6.21.dev0 → tinybird-4.6.22}/tinybird.egg-info/PKG-INFO +16 -1
  30. {tinybird-4.6.21.dev0 → tinybird-4.6.22}/tinybird.egg-info/SOURCES.txt +3 -1
  31. {tinybird-4.6.21.dev0 → tinybird-4.6.22}/setup.cfg +0 -0
  32. {tinybird-4.6.21.dev0 → tinybird-4.6.22}/tinybird/__cli__.py +0 -0
  33. {tinybird-4.6.21.dev0 → tinybird-4.6.22}/tinybird/ch_utils/constants.py +0 -0
  34. {tinybird-4.6.21.dev0 → tinybird-4.6.22}/tinybird/ch_utils/engine.py +0 -0
  35. {tinybird-4.6.21.dev0 → tinybird-4.6.22}/tinybird/check_pypi.py +0 -0
  36. {tinybird-4.6.21.dev0 → tinybird-4.6.22}/tinybird/config.py +0 -0
  37. {tinybird-4.6.21.dev0 → tinybird-4.6.22}/tinybird/context.py +0 -0
  38. {tinybird-4.6.21.dev0 → tinybird-4.6.22}/tinybird/datafile/exceptions.py +0 -0
  39. {tinybird-4.6.21.dev0 → tinybird-4.6.22}/tinybird/datafile/parse_connection.py +0 -0
  40. {tinybird-4.6.21.dev0 → tinybird-4.6.22}/tinybird/datafile/parse_datasource.py +0 -0
  41. {tinybird-4.6.21.dev0 → tinybird-4.6.22}/tinybird/datafile/parse_pipe.py +0 -0
  42. {tinybird-4.6.21.dev0 → tinybird-4.6.22}/tinybird/git_settings.py +0 -0
  43. {tinybird-4.6.21.dev0 → tinybird-4.6.22}/tinybird/iterating/__init__.py +0 -0
  44. {tinybird-4.6.21.dev0 → tinybird-4.6.22}/tinybird/iterating/data_branch_modes.py +0 -0
  45. {tinybird-4.6.21.dev0 → tinybird-4.6.22}/tinybird/prompts.py +0 -0
  46. {tinybird-4.6.21.dev0 → tinybird-4.6.22}/tinybird/service_datasources.py +0 -0
  47. {tinybird-4.6.21.dev0 → tinybird-4.6.22}/tinybird/sql.py +0 -0
  48. {tinybird-4.6.21.dev0 → tinybird-4.6.22}/tinybird/sql_toolset.py +0 -0
  49. {tinybird-4.6.21.dev0 → tinybird-4.6.22}/tinybird/tb/check_pypi.py +0 -0
  50. {tinybird-4.6.21.dev0 → tinybird-4.6.22}/tinybird/tb/cli.py +0 -0
  51. {tinybird-4.6.21.dev0 → tinybird-4.6.22}/tinybird/tb/config.py +0 -0
  52. {tinybird-4.6.21.dev0 → tinybird-4.6.22}/tinybird/tb/modules/branch.py +0 -0
  53. {tinybird-4.6.21.dev0 → tinybird-4.6.22}/tinybird/tb/modules/build.py +0 -0
  54. {tinybird-4.6.21.dev0 → tinybird-4.6.22}/tinybird/tb/modules/build_common.py +0 -0
  55. {tinybird-4.6.21.dev0 → tinybird-4.6.22}/tinybird/tb/modules/cicd.py +0 -0
  56. {tinybird-4.6.21.dev0 → tinybird-4.6.22}/tinybird/tb/modules/cli.py +0 -0
  57. {tinybird-4.6.21.dev0 → tinybird-4.6.22}/tinybird/tb/modules/config.py +0 -0
  58. {tinybird-4.6.21.dev0 → tinybird-4.6.22}/tinybird/tb/modules/connection.py +0 -0
  59. {tinybird-4.6.21.dev0 → tinybird-4.6.22}/tinybird/tb/modules/connection_dynamodb.py +0 -0
  60. {tinybird-4.6.21.dev0 → tinybird-4.6.22}/tinybird/tb/modules/connection_kafka.py +0 -0
  61. {tinybird-4.6.21.dev0 → tinybird-4.6.22}/tinybird/tb/modules/connection_s3.py +0 -0
  62. {tinybird-4.6.21.dev0 → tinybird-4.6.22}/tinybird/tb/modules/datafile/build_common.py +0 -0
  63. {tinybird-4.6.21.dev0 → tinybird-4.6.22}/tinybird/tb/modules/datafile/build_datasource.py +0 -0
  64. {tinybird-4.6.21.dev0 → tinybird-4.6.22}/tinybird/tb/modules/datafile/build_pipe.py +0 -0
  65. {tinybird-4.6.21.dev0 → tinybird-4.6.22}/tinybird/tb/modules/datafile/diff.py +0 -0
  66. {tinybird-4.6.21.dev0 → tinybird-4.6.22}/tinybird/tb/modules/datafile/fixture.py +0 -0
  67. {tinybird-4.6.21.dev0 → tinybird-4.6.22}/tinybird/tb/modules/datafile/format_common.py +0 -0
  68. {tinybird-4.6.21.dev0 → tinybird-4.6.22}/tinybird/tb/modules/datafile/format_connection.py +0 -0
  69. {tinybird-4.6.21.dev0 → tinybird-4.6.22}/tinybird/tb/modules/datafile/format_datasource.py +0 -0
  70. {tinybird-4.6.21.dev0 → tinybird-4.6.22}/tinybird/tb/modules/datafile/pipe_checker.py +0 -0
  71. {tinybird-4.6.21.dev0 → tinybird-4.6.22}/tinybird/tb/modules/deployment.py +0 -0
  72. {tinybird-4.6.21.dev0 → tinybird-4.6.22}/tinybird/tb/modules/deprecations.py +0 -0
  73. {tinybird-4.6.21.dev0 → tinybird-4.6.22}/tinybird/tb/modules/endpoint.py +0 -0
  74. {tinybird-4.6.21.dev0 → tinybird-4.6.22}/tinybird/tb/modules/fmt.py +0 -0
  75. {tinybird-4.6.21.dev0 → tinybird-4.6.22}/tinybird/tb/modules/info.py +0 -0
  76. {tinybird-4.6.21.dev0 → tinybird-4.6.22}/tinybird/tb/modules/infra.py +0 -0
  77. {tinybird-4.6.21.dev0 → tinybird-4.6.22}/tinybird/tb/modules/job.py +0 -0
  78. {tinybird-4.6.21.dev0 → tinybird-4.6.22}/tinybird/tb/modules/job_common.py +0 -0
  79. {tinybird-4.6.21.dev0 → tinybird-4.6.22}/tinybird/tb/modules/llm.py +0 -0
  80. {tinybird-4.6.21.dev0 → tinybird-4.6.22}/tinybird/tb/modules/llm_utils.py +0 -0
  81. {tinybird-4.6.21.dev0 → tinybird-4.6.22}/tinybird/tb/modules/local.py +0 -0
  82. {tinybird-4.6.21.dev0 → tinybird-4.6.22}/tinybird/tb/modules/local_common.py +0 -0
  83. {tinybird-4.6.21.dev0 → tinybird-4.6.22}/tinybird/tb/modules/local_logs.py +0 -0
  84. {tinybird-4.6.21.dev0 → tinybird-4.6.22}/tinybird/tb/modules/login.py +0 -0
  85. {tinybird-4.6.21.dev0 → tinybird-4.6.22}/tinybird/tb/modules/login_common.py +0 -0
  86. {tinybird-4.6.21.dev0 → tinybird-4.6.22}/tinybird/tb/modules/logout.py +0 -0
  87. {tinybird-4.6.21.dev0 → tinybird-4.6.22}/tinybird/tb/modules/logs.py +0 -0
  88. {tinybird-4.6.21.dev0 → tinybird-4.6.22}/tinybird/tb/modules/materialization.py +0 -0
  89. {tinybird-4.6.21.dev0 → tinybird-4.6.22}/tinybird/tb/modules/open.py +0 -0
  90. {tinybird-4.6.21.dev0 → tinybird-4.6.22}/tinybird/tb/modules/pipe.py +0 -0
  91. {tinybird-4.6.21.dev0 → tinybird-4.6.22}/tinybird/tb/modules/preview.py +0 -0
  92. {tinybird-4.6.21.dev0 → tinybird-4.6.22}/tinybird/tb/modules/project.py +0 -0
  93. {tinybird-4.6.21.dev0 → tinybird-4.6.22}/tinybird/tb/modules/project_commands.py +0 -0
  94. {tinybird-4.6.21.dev0 → tinybird-4.6.22}/tinybird/tb/modules/py_project.py +0 -0
  95. {tinybird-4.6.21.dev0 → tinybird-4.6.22}/tinybird/tb/modules/query_output.py +0 -0
  96. {tinybird-4.6.21.dev0 → tinybird-4.6.22}/tinybird/tb/modules/regions.py +0 -0
  97. {tinybird-4.6.21.dev0 → tinybird-4.6.22}/tinybird/tb/modules/secret.py +0 -0
  98. {tinybird-4.6.21.dev0 → tinybird-4.6.22}/tinybird/tb/modules/secret_common.py +0 -0
  99. {tinybird-4.6.21.dev0 → tinybird-4.6.22}/tinybird/tb/modules/sink.py +0 -0
  100. {tinybird-4.6.21.dev0 → tinybird-4.6.22}/tinybird/tb/modules/table.py +0 -0
  101. {tinybird-4.6.21.dev0 → tinybird-4.6.22}/tinybird/tb/modules/telemetry.py +0 -0
  102. {tinybird-4.6.21.dev0 → tinybird-4.6.22}/tinybird/tb/modules/test.py +0 -0
  103. {tinybird-4.6.21.dev0 → tinybird-4.6.22}/tinybird/tb/modules/test_common.py +0 -0
  104. {tinybird-4.6.21.dev0 → tinybird-4.6.22}/tinybird/tb/modules/tinyunit/tinyunit.py +0 -0
  105. {tinybird-4.6.21.dev0 → tinybird-4.6.22}/tinybird/tb/modules/token.py +0 -0
  106. {tinybird-4.6.21.dev0 → tinybird-4.6.22}/tinybird/tb/modules/ts_project.py +0 -0
  107. {tinybird-4.6.21.dev0 → tinybird-4.6.22}/tinybird/tb/modules/watch.py +0 -0
  108. {tinybird-4.6.21.dev0 → tinybird-4.6.22}/tinybird/tb/modules/workspace.py +0 -0
  109. {tinybird-4.6.21.dev0 → tinybird-4.6.22}/tinybird/tb/modules/workspace_members.py +0 -0
  110. {tinybird-4.6.21.dev0 → tinybird-4.6.22}/tinybird/tb_cli.py +0 -0
  111. {tinybird-4.6.21.dev0 → tinybird-4.6.22}/tinybird/tb_cli_modules/auth.py +0 -0
  112. {tinybird-4.6.21.dev0 → tinybird-4.6.22}/tinybird/tb_cli_modules/branch.py +0 -0
  113. {tinybird-4.6.21.dev0 → tinybird-4.6.22}/tinybird/tb_cli_modules/cicd.py +0 -0
  114. {tinybird-4.6.21.dev0 → tinybird-4.6.22}/tinybird/tb_cli_modules/cli.py +0 -0
  115. {tinybird-4.6.21.dev0 → tinybird-4.6.22}/tinybird/tb_cli_modules/config.py +0 -0
  116. {tinybird-4.6.21.dev0 → tinybird-4.6.22}/tinybird/tb_cli_modules/connection.py +0 -0
  117. {tinybird-4.6.21.dev0 → tinybird-4.6.22}/tinybird/tb_cli_modules/exceptions.py +0 -0
  118. {tinybird-4.6.21.dev0 → tinybird-4.6.22}/tinybird/tb_cli_modules/fmt.py +0 -0
  119. {tinybird-4.6.21.dev0 → tinybird-4.6.22}/tinybird/tb_cli_modules/job.py +0 -0
  120. {tinybird-4.6.21.dev0 → tinybird-4.6.22}/tinybird/tb_cli_modules/regions.py +0 -0
  121. {tinybird-4.6.21.dev0 → tinybird-4.6.22}/tinybird/tb_cli_modules/tag.py +0 -0
  122. {tinybird-4.6.21.dev0 → tinybird-4.6.22}/tinybird/tb_cli_modules/telemetry.py +0 -0
  123. {tinybird-4.6.21.dev0 → tinybird-4.6.22}/tinybird/tb_cli_modules/test.py +0 -0
  124. {tinybird-4.6.21.dev0 → tinybird-4.6.22}/tinybird/tb_cli_modules/tinyunit/tinyunit.py +0 -0
  125. {tinybird-4.6.21.dev0 → tinybird-4.6.22}/tinybird/tb_cli_modules/workspace.py +0 -0
  126. {tinybird-4.6.21.dev0 → tinybird-4.6.22}/tinybird/tb_cli_modules/workspace_members.py +0 -0
  127. {tinybird-4.6.21.dev0 → tinybird-4.6.22}/tinybird.egg-info/dependency_links.txt +0 -0
  128. {tinybird-4.6.21.dev0 → tinybird-4.6.22}/tinybird.egg-info/entry_points.txt +0 -0
  129. {tinybird-4.6.21.dev0 → tinybird-4.6.22}/tinybird.egg-info/requires.txt +0 -0
  130. {tinybird-4.6.21.dev0 → tinybird-4.6.22}/tinybird.egg-info/top_level.txt +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.2
2
2
  Name: tinybird
3
- Version: 4.6.21.dev0
3
+ Version: 4.6.22
4
4
  Summary: Tinybird Command Line Tool
5
5
  Home-page: https://www.tinybird.co/docs/forward/commands
6
6
  Author: Tinybird
@@ -52,6 +52,21 @@ The Tinybird command-line tool allows you to use all the Tinybird functionality
52
52
  Changelog
53
53
  ----------
54
54
 
55
+ 4.6.22
56
+ ********
57
+
58
+ - `Added` ``tb pull`` demotes sink and copy pipes that exceed the workspace's sink/copy pipe plan limit to plain pipes instead of failing, stripping their ``TYPE``/``EXPORT_*``/``TARGET_DATASOURCE``/``COPY_SCHEDULE`` settings and warning that the plan must be upgraded to restore them.
59
+
60
+ 4.6.21
61
+ ********
62
+
63
+ - `Fixed` ``tb deployment discard`` now discards the deployment that is not live, instead of assuming the live deployment is listed first. Previously, if removing the old deployment failed after a promotion, the command refused to discard it.
64
+
65
+ 4.6.20
66
+ ********
67
+
68
+ - `Added` Copy pipes accept ``ON_DEMAND_COMPUTE true`` so scheduled runs use on-demand compute. Manual ``tb copy run`` still requires ``--on-demand-compute``, and asks for confirmation when the local datafile does not enable it (skip it with ``--yes``).
69
+
55
70
  4.6.19
56
71
  ********
57
72
 
@@ -18,34 +18,35 @@ authors = [
18
18
  dependencies = [
19
19
  "GitPython==3.1.44",
20
20
  "Markdown==3.5.1",
21
- "aioboto3==14.0.0",
21
+ "aioboto3~=15.0",
22
22
  "aiofiles==24.1.0",
23
- "aiohttp==3.11.2",
24
- "anthropic==0.55.0",
23
+ "aiohttp~=3.11",
24
+ "anthropic~=1.9",
25
+ "async-lru==2.3.0",
25
26
  "aws-msk-iam-sasl-signer-python==1.0.2",
26
27
  "cachetools==5.5.0",
27
28
  "cffi==1.17.1",
28
29
  "chardet==4.0.0",
29
30
  "ciso8601==2.2.0",
30
- "click==8.1.8",
31
+ "click~=8.5",
31
32
  "clickhouse-cityhash==1.0.2.5",
32
33
  "clickhouse-driver==0.2.9",
33
34
  "confluent-kafka[avro,json,schemaregistry]==2.12.2",
34
35
  "contextvars==2.4",
35
36
  "cramjam==2.11.0",
36
37
  "croniter==6.2.2",
37
- "cryptography==3.4.8",
38
+ "cryptography~=38.0",
38
39
  "datasketch==1.6.5",
39
40
  "dnspython==2.8.0",
40
41
  "docker==7.1.0",
41
42
  "fastavro==1.12.2",
42
43
  "fastmcp==2.12.2",
43
- "google-cloud-aiplatform==1.148.1",
44
+ "google-cloud-aiplatform~=1.148",
44
45
  "google-cloud-compute==1.48.0",
45
46
  "google-cloud-scheduler==2.20.0",
46
47
  "google-cloud-secret-manager==2.29.0",
47
- "google-cloud-storage==3.4.1",
48
- "google_api_core==2.25.2",
48
+ "google-cloud-storage~=3.4",
49
+ "google_api_core~=2.25",
49
50
  "googleapis_common_protos==1.66.0",
50
51
  "grpcio_status==1.67.1",
51
52
  "humanfriendly==10.0",
@@ -68,7 +69,7 @@ dependencies = [
68
69
  "opentelemetry-sdk==1.38.0",
69
70
  "orb-billing==2.17.0",
70
71
  "orjson==3.11.5",
71
- "packaging==23.2",
72
+ "packaging~=26.3",
72
73
  "passlib==1.7.1",
73
74
  "posthog==7.28.0",
74
75
  "pre-commit==4.6.2",
@@ -82,10 +83,10 @@ dependencies = [
82
83
  "py-spy==0.4.2",
83
84
  "pyarrow==23.0.1",
84
85
  "pycurl==7.45.7",
85
- "pydantic-ai-slim[anthropic]==0.5.0",
86
- "pydantic-ai-slim[google]==0.5.0",
87
- "pydantic-ai-slim[retries]==0.5.0",
88
- "pydantic==2.12.5",
86
+ "pydantic-ai-slim[anthropic]~=2.0",
87
+ "pydantic-ai-slim[google]~=2.0",
88
+ "pydantic-ai-slim[retries]~=2.0",
89
+ "pydantic~=2.12",
89
90
  "pyjwt[crypto]==2.9.0",
90
91
  "pympler==1.1",
91
92
  "pynacl==1.5.0",
@@ -93,10 +94,10 @@ dependencies = [
93
94
  "python-dateutil==2.9.0.post0",
94
95
  "python-dotenv==1.1.0",
95
96
  "python-ulid==2.7.0",
96
- "pyyaml==6.0.1",
97
+ "pyyaml~=6.0",
97
98
  "redis[hiredis]==5.3.1",
98
99
  "requests==2.32.4",
99
- "sentry-sdk==2.69.1",
100
+ "sentry-sdk==2.69.2",
100
101
  "setproctitle==1.3.3",
101
102
  "shandy-sqlfmt==0.11.1",
102
103
  "shandy-sqlfmt[jinjafmt]==0.11.1",
@@ -107,7 +108,7 @@ dependencies = [
107
108
  "tabulate==0.8.10",
108
109
  "thefuzz==0.22.1",
109
110
  "tinybird-python-sdk[ai]==0.3.7",
110
- "tinybird-toolset==2.6.2",
111
+ "tinybird-toolset==2.6.6",
111
112
  "toposort==1.10",
112
113
  "tornado==6.5.5",
113
114
  "urllib3==1.26.20",
@@ -124,9 +125,9 @@ deploy = [
124
125
  "GitPython==3.1.44",
125
126
  "ansible==8.5.0",
126
127
  "ara==1.6.1",
127
- "boto3==1.36.21",
128
+ "boto3~=1.40",
128
129
  "fabric==3.2.2",
129
- "google-auth==2.47.0",
130
+ "google-auth~=2.47",
130
131
  "jsonnet",
131
132
  "prettytable==3.8.0",
132
133
  "requests==2.32.4",
@@ -141,7 +142,7 @@ devtools = [
141
142
  ]
142
143
  doc = [
143
144
  "Pallets-Sphinx-Themes==2.3.0",
144
- "pyOpenSSL<24.0.0",
145
+ "pyOpenSSL~=23.2",
145
146
  "sphinx-autobuild==2024.2.4",
146
147
  "sphinx-sitemap",
147
148
  "sphinx==7.3.7",
@@ -149,13 +150,15 @@ doc = [
149
150
  "sphinxcontrib-jquery==4.1",
150
151
  ]
151
152
  lint = [
152
- "botocore-stubs==1.36.23",
153
+ "botocore-stubs~=1.40",
154
+ "flake8~=7.3",
155
+ "flake8-async~=26.8",
153
156
  "mypy==2.3.1",
154
- "ruff==0.16.8",
157
+ "ruff==0.16.9",
155
158
  "types-Markdown",
156
159
  "types-PyYAML",
157
160
  "types-aiofiles==24.1.0.20241221",
158
- "types-boto3-lite==1.36.21",
161
+ "types-boto3-lite~=1.40",
159
162
  "types-cachetools",
160
163
  "types-chardet",
161
164
  "types-paramiko",
@@ -169,21 +172,28 @@ lint = [
169
172
  test = [
170
173
  "coverage==7.10.6",
171
174
  "freezegun==1.5.5",
172
- "google-auth==2.47.0",
175
+ "google-auth~=2.47",
173
176
  "gsutil==4.58",
177
+ "httpx2~=2.13",
174
178
  "hypothesis==6.156.1",
175
179
  "oauth2client==3.0.0",
176
180
  "py==1.11.0",
177
- "pytest-asyncio==0.23.8",
181
+ "pytest-asyncio~=1.4",
178
182
  "pytest-cov==4.1.0",
179
183
  "pytest-env==1.1.3",
180
184
  "pytest-flakefinder==1.1.0",
181
185
  "pytest-timeout==2.3.1",
182
186
  "pytest-xdist==3.6.1",
183
- "pytest==8.1.2",
187
+ "pytest~=8.1",
184
188
  "rangehttpserver==1.4.0",
185
189
  "yappi==1.7.6",
186
190
  ]
191
+ # Synced automatically by plain `uv run`/`uv sync`, no `--group` needed.
192
+ dev = [
193
+ { include-group = "test" },
194
+ { include-group = "deploy" },
195
+ { include-group = "lint" },
196
+ ]
187
197
 
188
198
  [project.scripts]
189
199
  tinybird_server = "tinybird.app:run"
@@ -193,7 +203,12 @@ tb = "tinybird.tb.cli:cli"
193
203
  colibri = "tinybird.colibri.cli:cli"
194
204
 
195
205
  [tool.uv.workspace]
196
- members = ["data_connector"]
206
+ members = [
207
+ "chbackups",
208
+ "data_connector",
209
+ "gatherer",
210
+ "tinybird_orchestrator",
211
+ ]
197
212
 
198
213
  [tool.setuptools]
199
214
  include-package-data = true
@@ -47,14 +47,6 @@ class TimeoutException(Exception):
47
47
  pass
48
48
 
49
49
 
50
- class ReachRetryLimit(Exception):
51
- pass
52
-
53
-
54
- class ConnectorNothingToLoad(Exception):
55
- pass
56
-
57
-
58
50
  class JobException(Exception):
59
51
  pass
60
52
 
@@ -691,6 +683,7 @@ class TinyB:
691
683
  target_datasource: Optional[str] = None,
692
684
  schedule_cron: Optional[str] = None,
693
685
  mode: Optional[str] = None,
686
+ on_demand_compute: Optional[bool] = None,
694
687
  ):
695
688
  data = {"schedule_cron": schedule_cron}
696
689
 
@@ -700,6 +693,9 @@ class TinyB:
700
693
  if mode:
701
694
  data["mode"] = mode
702
695
 
696
+ if on_demand_compute is not None:
697
+ data["on_demand_compute"] = "true" if on_demand_compute else "false"
698
+
703
699
  return await self._req(f"/v0/pipes/{pipe_name_or_id}/nodes/{node_id}/copy", method="PUT", data=data)
704
700
 
705
701
  async def pipe_remove_copy(self, pipe_name_or_id: str, node_id: str):
@@ -28,6 +28,7 @@ from tinybird.ch_utils.engine import VALID_ENGINE_PARAMS, EngineParam
28
28
  from tinybird.datafile.exceptions import IncludeFileNotFoundException, ParseException, ValidationException
29
29
  from tinybird.tb.modules.exceptions import CLIPipeException
30
30
  from tinybird.tb.modules.feedback_manager import FeedbackManager
31
+ from tinybird.utils.bools import parse_optional_bool
31
32
 
32
33
  # Code from sql.py has been duplicated so I can change it without breaking absolutely everything in the app
33
34
  # I'll try not to make logic changes, just error reporting changes
@@ -179,9 +180,10 @@ class CopyParameters(Parameters):
179
180
  COPY_SCHEDULE = "copy_schedule"
180
181
  COPY_MODE = "copy_mode"
181
182
  COPY_MODE_ALIAS = "mode" # we need this because bot MODE and COPY_MODE go to `mode` variable inside the node
183
+ ON_DEMAND_COMPUTE = "on_demand_compute"
182
184
  MANDATORY_ATTRIBUTES = PipeParameters.MANDATORY_ATTRIBUTES.union({TARGET_DATASOURCE})
183
185
  ACCEPTED_ATTRIBUTES = PipeParameters.ACCEPTED_ATTRIBUTES.union(MANDATORY_ATTRIBUTES).union(
184
- {COPY_SCHEDULE, COPY_MODE_ALIAS}
186
+ {COPY_SCHEDULE, COPY_MODE_ALIAS, ON_DEMAND_COMPUTE}
185
187
  )
186
188
 
187
189
 
@@ -416,6 +418,11 @@ class Datafile:
416
418
  )
417
419
  if not croniter.is_valid(copy_schedule):
418
420
  raise DatafileValidationError("COPY node schedule must be @on-demand or a valid cron expression.")
421
+ if CopyParameters.ON_DEMAND_COMPUTE in node:
422
+ try:
423
+ node[CopyParameters.ON_DEMAND_COMPUTE] = parse_optional_bool(node[CopyParameters.ON_DEMAND_COMPUTE])
424
+ except ValueError:
425
+ raise DatafileValidationError("ON_DEMAND_COMPUTE must be true or false")
419
426
  for key in node.keys():
420
427
  if key not in CopyParameters.valid_params():
421
428
  raise DatafileValidationError(
@@ -2097,6 +2104,7 @@ def parse(
2097
2104
  "copy_schedule": assign_node_var(CopyParameters.COPY_SCHEDULE),
2098
2105
  "copy_mode": assign_node_var("mode"),
2099
2106
  "mode": assign_node_var("mode"),
2107
+ "on_demand_compute": assign_node_var(CopyParameters.ON_DEMAND_COMPUTE),
2100
2108
  "filter": assign_node_var("filter"),
2101
2109
  "token": add_token,
2102
2110
  "include": include,
@@ -120,52 +120,74 @@ def datetime_test(x: str) -> bool:
120
120
  return any(p.match(x) for p in datetime_patterns)
121
121
 
122
122
 
123
+ def bounded_integer(x: str, pattern: re.Pattern[str]) -> Optional[int]:
124
+ if pattern.match(x) is None:
125
+ return None
126
+ digits = x.lstrip("+-").lstrip("0")
127
+ # 256-bit unsigned integers have at most 78 decimal digits.
128
+ if len(digits) > 78:
129
+ return None
130
+ return int(x) if len(x) <= 78 else int(digits or "0") * (-1 if x.startswith("-") else 1)
131
+
132
+
123
133
  def int_8_test(x: str) -> bool:
124
- return intx_re.match(x) is not None and -int_8_max <= int(x) < int_8_max
134
+ value = bounded_integer(x, intx_re)
135
+ return value is not None and -int_8_max <= value < int_8_max
125
136
 
126
137
 
127
138
  def int16_test(x: str) -> bool:
128
- return intx_re.match(x) is not None and -int16_max <= int(x) < int16_max
139
+ value = bounded_integer(x, intx_re)
140
+ return value is not None and -int16_max <= value < int16_max
129
141
 
130
142
 
131
143
  def int32_test(x: str) -> bool:
132
- return intx_re.match(x) is not None and -int32_max <= int(x) < int32_max
144
+ value = bounded_integer(x, intx_re)
145
+ return value is not None and -int32_max <= value < int32_max
133
146
 
134
147
 
135
148
  def int64_test(x: str) -> bool:
136
- return intx_re.match(x) is not None and -int64_max <= int(x) < int64_max
149
+ value = bounded_integer(x, intx_re)
150
+ return value is not None and -int64_max <= value < int64_max
137
151
 
138
152
 
139
153
  def int128_test(x: str) -> bool:
140
- return intx_re.match(x) is not None and -int128_max <= int(x) < int128_max
154
+ value = bounded_integer(x, intx_re)
155
+ return value is not None and -int128_max <= value < int128_max
141
156
 
142
157
 
143
158
  def int256_test(x: str) -> bool:
144
- return intx_re.match(x) is not None and -int256_max <= int(x) < int256_max
159
+ value = bounded_integer(x, intx_re)
160
+ return value is not None and -int256_max <= value < int256_max
145
161
 
146
162
 
147
163
  def uint_8_test(x: str) -> bool:
148
- return uintx_re.match(x) is not None and 0 <= int(x) < uint_8_max
164
+ value = bounded_integer(x, uintx_re)
165
+ return value is not None and 0 <= value < uint_8_max
149
166
 
150
167
 
151
168
  def uint16_test(x: str) -> bool:
152
- return uintx_re.match(x) is not None and 0 <= int(x) < uint16_max
169
+ value = bounded_integer(x, uintx_re)
170
+ return value is not None and 0 <= value < uint16_max
153
171
 
154
172
 
155
173
  def uint32_test(x: str) -> bool:
156
- return uintx_re.match(x) is not None and 0 <= int(x) < uint32_max
174
+ value = bounded_integer(x, uintx_re)
175
+ return value is not None and 0 <= value < uint32_max
157
176
 
158
177
 
159
178
  def uint64_test(x: str) -> bool:
160
- return uintx_re.match(x) is not None and 0 <= int(x) < uint64_max
179
+ value = bounded_integer(x, uintx_re)
180
+ return value is not None and 0 <= value < uint64_max
161
181
 
162
182
 
163
183
  def uint128_test(x: str) -> bool:
164
- return intx_re.match(x) is not None and 0 <= int(x) < uint128_max
184
+ value = bounded_integer(x, intx_re)
185
+ return value is not None and 0 <= value < uint128_max
165
186
 
166
187
 
167
188
  def uint256_test(x: str) -> bool:
168
- return intx_re.match(x) is not None and 0 <= int(x) < uint256_max
189
+ value = bounded_integer(x, intx_re)
190
+ return value is not None and 0 <= value < uint256_max
169
191
 
170
192
 
171
193
  def float_test(x: str) -> bool:
@@ -572,6 +572,11 @@ class FeedbackManager:
572
572
  "You can read more about the pricing at https://www.tinybird.co/docs/classic/work-with-data/process-and-copy/materialized-views#compute-compute-separation-for-populates\n\n"
573
573
  "Do you want to proceed with on-demand compute?"
574
574
  )
575
+ warning_confirm_on_demand_compute_differs_from_datafile = warning_message(
576
+ "The local datafile of pipe '{pipe}' does not set ON_DEMAND_COMPUTE true, but --on-demand-compute was passed.\n"
577
+ "On-demand compute uses dedicated resources that are billed separately from your regular usage.\n\n"
578
+ "Do you want to proceed?"
579
+ )
575
580
 
576
581
  warning_confirm_delete_token = prompt_message("Do you want to delete Token {token}?")
577
582
  warning_confirm_refresh_token = prompt_message("Do you want to refresh Token {token}?")
@@ -1,5 +1,4 @@
1
1
  import ast
2
- import builtins
3
2
  import linecache
4
3
  import logging
5
4
  import re
@@ -18,7 +17,7 @@ from tinybird.context import (
18
17
  )
19
18
 
20
19
  from .datatypes import testers
21
- from .tornado_template import VALID_CUSTOM_FUNCTION_NAMES, SecurityException, Template
20
+ from .tornado_template import SAFE_BUILTINS, VALID_CUSTOM_FUNCTION_NAMES, SecurityException, Template
22
21
 
23
22
  VALID_ACTIVATE_FEATURES = frozenset(["analyzer", "parallel_replicas", "optimize_aggregation_in_order"])
24
23
 
@@ -1336,8 +1335,10 @@ sqlescape_for_string_expression = sqlescape_generator(
1336
1335
  )
1337
1336
 
1338
1337
 
1339
- def escape_single_quote_str(s):
1340
- return "'" + s.replace("'", "''") + "'"
1338
+ def escape_single_quote_str(s: str) -> str:
1339
+ # ClickHouse string literals treat backslash as an escape char, so it must be doubled too:
1340
+ # otherwise a trailing backslash escapes the closing quote and the following input becomes SQL
1341
+ return "'" + s.replace("\\", "\\\\").replace("'", "''") + "'"
1341
1342
 
1342
1343
 
1343
1344
  def expression_wrapper(x, name, escape_arrays: bool = False):
@@ -1375,7 +1376,7 @@ _namespace = {
1375
1376
  # they raise a pretty non understandable error but if someone
1376
1377
  # is using them they know what they are trying to do
1377
1378
  # read https://anee.me/escaping-python-jails-849c65cf306e on how to escape from python jails
1378
- "__buildins__": {},
1379
+ "__builtins__": SAFE_BUILTINS,
1379
1380
  "__import__": {},
1380
1381
  "__debug__": {},
1381
1382
  "__doc__": {},
@@ -1388,10 +1389,13 @@ _namespace = {
1388
1389
  }
1389
1390
 
1390
1391
 
1391
- reserved_vars = {"_tt_tmp", "_tt_append", "isinstance", "str", "error", "custom_error", *list(vars(builtins))}
1392
+ # Built from the builtins templates can actually resolve: names missing from SAFE_BUILTINS (`type`, `dir`,
1393
+ # `property`, `object`, ...) can only be template parameters, so they must be discovered as such to get a
1394
+ # value (None or a test-mode placeholder) instead of raising NameError.
1395
+ reserved_vars = {"_tt_tmp", "_tt_append", "isinstance", "str", "error", "custom_error", *SAFE_BUILTINS}
1392
1396
  for p in DEFAULT_PARAM_NAMES: # we handle these in an specific manner
1393
1397
  reserved_vars.discard(p) # `format` is part of builtins
1394
- # Allow 'id' to be used as a template parameter - https://gitlab.com/tinybird/analytics/-/issues/19119
1398
+ # Allow 'id' as a template parameter name despite being a builtin - https://gitlab.com/tinybird/analytics/-/issues/19119
1395
1399
  reserved_vars.discard("id")
1396
1400
  error_vars = ["error", "custom_error"]
1397
1401
 
@@ -1468,8 +1472,7 @@ def generate(self, **kwargs) -> Tuple[str, TemplateExecutionResults]:
1468
1472
  return Expression(f"-- disable {feature}\n")
1469
1473
 
1470
1474
  namespace.update(_namespace)
1471
- # This is to fix the issue https://gitlab.com/tinybird/analytics/-/issues/19119
1472
- # We need to do the override here because if we modify the _namespace, we would filter out parameters from the users
1475
+ # Default it here, not in _namespace, so it stays visible to variable extraction.
1473
1476
  namespace.update({"id": None})
1474
1477
  namespace.update(kwargs)
1475
1478
  namespace.update(
@@ -1478,6 +1481,8 @@ def generate(self, **kwargs) -> Tuple[str, TemplateExecutionResults]:
1478
1481
  # the generated source code.
1479
1482
  "__name__": self.name.replace(".", "_"),
1480
1483
  "__loader__": ObjectDict(get_source=lambda name: self.code),
1484
+ # Set again after kwargs so request parameters can't override it
1485
+ "__builtins__": SAFE_BUILTINS,
1481
1486
  "max_threads": set_max_threads,
1482
1487
  "tb_secret": set_tb_secret,
1483
1488
  "tb_var": set_tb_secret,
@@ -1817,7 +1822,7 @@ def get_var_names(t: Template):
1817
1822
  >>> [v['name'] for v in get_var_names(Template("{% for name in secret_names %}{{tb_secret(name)}}{% end %}"))]
1818
1823
  ['secret_names', 'name', 'tb_secret', 'name']
1819
1824
 
1820
- Type casting with date function in elif chain (type is in reserved_vars):
1825
+ Type casting with date function in elif chain:
1821
1826
  >>> t = Template("{% if kind == 'int' %}{{Int32(val, 0)}}{% elif kind == 'date' %}{{DateTime(val)}}{% elif kind == 'diff' %}{{date_diff_in_days(val, now)}}{% end %}")
1822
1827
  >>> [v['name'] for v in get_var_names(t)]
1823
1828
  ['kind', 'Int32', 'val', 'kind', 'DateTime', 'val', 'kind', 'date_diff_in_days', 'val', 'now']
@@ -2369,7 +2374,24 @@ def wrap_vars(t, escape_arrays: bool = False):
2369
2374
  return var
2370
2375
 
2371
2376
 
2372
- def get_used_tables_in_template(sql):
2377
+ def _constant_table_name(node: ast.expr) -> Optional[str]:
2378
+ """Value of a constant table() argument: a literal or a concatenation of string literals."""
2379
+ if isinstance(node, ast.Constant) and isinstance(node.value, (int, float)):
2380
+ return str(node.value)
2381
+ return _string_literal(node)
2382
+
2383
+
2384
+ def _string_literal(node: ast.expr) -> Optional[str]:
2385
+ if isinstance(node, ast.Constant) and isinstance(node.value, str):
2386
+ return node.value
2387
+ if isinstance(node, ast.BinOp) and isinstance(node.op, ast.Add):
2388
+ left, right = _string_literal(node.left), _string_literal(node.right)
2389
+ if left is not None and right is not None:
2390
+ return left + right
2391
+ return None
2392
+
2393
+
2394
+ def get_used_tables_in_template(sql: str) -> List[str]:
2373
2395
  """
2374
2396
  >>> get_used_tables_in_template("select * from {{table('test')}}")
2375
2397
  ['test']
@@ -2383,30 +2405,42 @@ def get_used_tables_in_template(sql):
2383
2405
  []
2384
2406
  >>> get_used_tables_in_template("select * from {{table('my.test')}}, {{table('another.one')}}")
2385
2407
  ['my.test', 'another.one']
2408
+
2409
+ Only constant table names can be known without rendering, so anything else is skipped
2410
+ >>> get_used_tables_in_template("select * from {{table(tbl)}}, {{TABLE('b')}}, {{table('my.' + 'c')}}")
2411
+ ['b', 'my.c']
2386
2412
  """
2413
+ return list(_get_used_tables_in_template_cached(sql))
2414
+
2415
+
2416
+ # Building the Template dominates the cost and the result only depends on the SQL, which deployments and
2417
+ # /v0/sql pipelines scan repeatedly
2418
+ @lru_cache(maxsize=2**10)
2419
+ def _get_used_tables_in_template_cached(sql: str) -> Tuple[str, ...]:
2387
2420
  try:
2388
2421
  t = Template(sql)
2389
2422
 
2390
2423
  def _n(chunks, tables):
2391
2424
  for x in chunks:
2392
2425
  if type(x).__name__ == "_Expression":
2393
- c = compile(x.expression, "<string>", "exec", dont_inherit=True)
2394
- v = [x.lower() for x in c.co_names if x not in _namespace and x not in reserved_vars]
2395
- if "table" in v:
2396
-
2397
- def _t(*args, **kwargs):
2398
- return str(args[0])
2399
-
2400
- n = {"table": _t, "TABLE": _t}
2401
- e = "_tt_tmp = %s" % x.expression
2402
- exec_in(e, n)
2403
- tables += [n["_tt_tmp"]]
2426
+ # Read the names from the AST instead of executing the expression: this runs outside
2427
+ # generate(), without the restricted builtins, on every pipe, query and deployment.
2428
+ for node in ast.walk(ast.parse(x.expression)):
2429
+ if (
2430
+ isinstance(node, ast.Call)
2431
+ and isinstance(node.func, ast.Name)
2432
+ and node.func.id in ("table", "TABLE")
2433
+ and node.args
2434
+ ):
2435
+ name = _constant_table_name(node.args[0])
2436
+ if name is not None:
2437
+ tables.append(name)
2404
2438
  elif type(x).__name__ == "_ControlBlock":
2405
2439
  _n(x.body.chunks, tables)
2406
2440
 
2407
- tables = []
2441
+ tables: List[str] = []
2408
2442
  _n(t.file.body.chunks, tables)
2409
- return tables
2443
+ return tuple(tables)
2410
2444
  except SecurityException as e:
2411
2445
  raise SQLTemplateException(e)
2412
2446
 
@@ -214,8 +214,8 @@ class TinybirdDialect(ClickHouse):
214
214
  ],
215
215
  }
216
216
 
217
- for section in override_rules:
218
- for rule in override_rules[section]:
217
+ for section, rules in override_rules.items():
218
+ for rule in rules:
219
219
  for rr in self.RULES[section]:
220
220
  if rr.name == rule.name:
221
221
  self.RULES[section].remove(rr)
@@ -260,49 +260,6 @@ def _iscoroutinefunction_or_partial(func: Any) -> bool:
260
260
  return asyncio.iscoroutinefunction(func)
261
261
 
262
262
 
263
- class ThreadSensitiveContext:
264
- """Async context manager to manage context for thread sensitive mode
265
- This context manager controls which thread pool executor is used when in
266
- thread sensitive mode. By default, a single thread pool executor is shared
267
- within a process.
268
- In Python 3.7+, the ThreadSensitiveContext() context manager may be used to
269
- specify a thread pool per context.
270
- In Python 3.6, usage of this context manager has no effect.
271
- This context manager is re-entrant, so only the outer-most call to
272
- ThreadSensitiveContext will set the context.
273
- """
274
-
275
- def __init__(self):
276
- self.token = None
277
-
278
- if contextvars:
279
-
280
- async def __aenter__(self):
281
- try:
282
- SyncToAsync.thread_sensitive_context.get()
283
- except LookupError:
284
- self.token = SyncToAsync.thread_sensitive_context.set(self)
285
-
286
- return self
287
-
288
- async def __aexit__(self, exc, value, tb):
289
- if not self.token:
290
- return
291
-
292
- executor = SyncToAsync.context_to_thread_executor.pop(self, None)
293
- if executor:
294
- executor.shutdown()
295
- SyncToAsync.thread_sensitive_context.reset(self.token)
296
-
297
- else:
298
-
299
- async def __aenter__(self):
300
- return self
301
-
302
- async def __aexit__(self, exc, value, tb):
303
- pass
304
-
305
-
306
263
  class AsyncToSync:
307
264
  """
308
265
  Utility class which turns an awaitable that only works on the thread with
@@ -4,5 +4,5 @@ __description__ = 'Tinybird Command Line Tool'
4
4
  __url__ = 'https://www.tinybird.co/docs/forward/commands'
5
5
  __author__ = 'Tinybird'
6
6
  __author_email__ = 'support@tinybird.co'
7
- __version__ = '4.6.21.dev0'
8
- __revision__ = '965c47ce'
7
+ __version__ = '4.6.22'
8
+ __revision__ = '5269fe22'
@@ -58,14 +58,6 @@ class TimeoutException(Exception):
58
58
  pass
59
59
 
60
60
 
61
- class ReachRetryLimit(Exception):
62
- pass
63
-
64
-
65
- class ConnectorNothingToLoad(Exception):
66
- pass
67
-
68
-
69
61
  class JobException(Exception):
70
62
  pass
71
63
 
@@ -743,6 +735,7 @@ class TinyB:
743
735
  target_datasource: Optional[str] = None,
744
736
  schedule_cron: Optional[str] = None,
745
737
  mode: Optional[str] = None,
738
+ on_demand_compute: Optional[bool] = None,
746
739
  ):
747
740
  data = {"schedule_cron": schedule_cron}
748
741
 
@@ -752,6 +745,9 @@ class TinyB:
752
745
  if mode:
753
746
  data["mode"] = mode
754
747
 
748
+ if on_demand_compute is not None:
749
+ data["on_demand_compute"] = "true" if on_demand_compute else "false"
750
+
755
751
  return self._req(f"/v0/pipes/{pipe_name_or_id}/nodes/{node_id}/copy", method="PUT", data=data)
756
752
 
757
753
  def pipe_remove_copy(self, pipe_name_or_id: str, node_id: str):
@@ -1277,11 +1277,11 @@ class ConnectionReplacements:
1277
1277
  """Maps prompt parameters to API parameters."""
1278
1278
 
1279
1279
  api_params = {}
1280
- for key in params.keys():
1280
+ for key, value in params.items():
1281
1281
  try:
1282
- api_params[ConnectionReplacements._PARAMS_REPLACEMENTS[service][key]] = params[key]
1282
+ api_params[ConnectionReplacements._PARAMS_REPLACEMENTS[service][key]] = value
1283
1283
  except KeyError:
1284
- api_params[key] = params[key]
1284
+ api_params[key] = value
1285
1285
 
1286
1286
  api_params["service"] = service
1287
1287
  return api_params