tinybird 4.6.20.dev0__tar.gz → 4.6.21__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (130) hide show
  1. {tinybird-4.6.20.dev0 → tinybird-4.6.21}/PKG-INFO +21 -1
  2. tinybird-4.6.21/pyproject.toml +240 -0
  3. {tinybird-4.6.20.dev0 → tinybird-4.6.21}/tinybird/ch_utils/constants.py +1 -0
  4. {tinybird-4.6.20.dev0 → tinybird-4.6.21}/tinybird/ch_utils/engine.py +49 -32
  5. {tinybird-4.6.20.dev0 → tinybird-4.6.21}/tinybird/client.py +41 -0
  6. {tinybird-4.6.20.dev0 → tinybird-4.6.21}/tinybird/datafile/common.py +9 -1
  7. {tinybird-4.6.20.dev0 → tinybird-4.6.21}/tinybird/datatypes.py +34 -12
  8. {tinybird-4.6.20.dev0 → tinybird-4.6.21}/tinybird/feedback_manager.py +5 -0
  9. {tinybird-4.6.20.dev0 → tinybird-4.6.21}/tinybird/service_datasources.py +7 -8
  10. {tinybird-4.6.20.dev0 → tinybird-4.6.21}/tinybird/sql.py +0 -6
  11. {tinybird-4.6.20.dev0 → tinybird-4.6.21}/tinybird/sql_template.py +10 -6
  12. {tinybird-4.6.20.dev0 → tinybird-4.6.21}/tinybird/sql_toolset.py +59 -0
  13. {tinybird-4.6.20.dev0 → tinybird-4.6.21}/tinybird/tb/__cli__.py +2 -2
  14. {tinybird-4.6.20.dev0 → tinybird-4.6.21}/tinybird/tb/client.py +41 -0
  15. {tinybird-4.6.20.dev0 → tinybird-4.6.21}/tinybird/tb/modules/common.py +15 -77
  16. {tinybird-4.6.20.dev0 → tinybird-4.6.21}/tinybird/tb/modules/copy.py +33 -1
  17. {tinybird-4.6.20.dev0 → tinybird-4.6.21}/tinybird/tb/modules/datafile/build.py +1 -0
  18. {tinybird-4.6.20.dev0 → tinybird-4.6.21}/tinybird/tb/modules/datafile/format_pipe.py +4 -0
  19. {tinybird-4.6.20.dev0 → tinybird-4.6.21}/tinybird/tb/modules/datafile/playground.py +1 -0
  20. {tinybird-4.6.20.dev0 → tinybird-4.6.21}/tinybird/tb/modules/deployment_common.py +4 -6
  21. {tinybird-4.6.20.dev0 → tinybird-4.6.21}/tinybird/tb/modules/feedback_manager.py +5 -0
  22. {tinybird-4.6.20.dev0 → tinybird-4.6.21}/tinybird/tb/modules/infra.py +1 -2
  23. {tinybird-4.6.20.dev0 → tinybird-4.6.21}/tinybird/tb/modules/logs.py +1 -2
  24. {tinybird-4.6.20.dev0 → tinybird-4.6.21}/tinybird/tb/modules/materialization.py +1 -1
  25. {tinybird-4.6.20.dev0 → tinybird-4.6.21}/tinybird/tb/modules/project.py +4 -4
  26. {tinybird-4.6.20.dev0 → tinybird-4.6.21}/tinybird/tb/modules/tinyunit/tinyunit.py +4 -4
  27. {tinybird-4.6.20.dev0 → tinybird-4.6.21}/tinybird/tb/modules/token.py +2 -5
  28. {tinybird-4.6.20.dev0 → tinybird-4.6.21}/tinybird/tb/modules/watch.py +1 -1
  29. {tinybird-4.6.20.dev0 → tinybird-4.6.21}/tinybird/tb_cli_modules/branch.py +1 -2
  30. {tinybird-4.6.20.dev0 → tinybird-4.6.21}/tinybird/tb_cli_modules/cicd.py +2 -0
  31. {tinybird-4.6.20.dev0 → tinybird-4.6.21}/tinybird/tb_cli_modules/cli.py +5 -9
  32. {tinybird-4.6.20.dev0 → tinybird-4.6.21}/tinybird/tb_cli_modules/common.py +27 -94
  33. {tinybird-4.6.20.dev0 → tinybird-4.6.21}/tinybird/tb_cli_modules/config.py +1 -2
  34. {tinybird-4.6.20.dev0 → tinybird-4.6.21}/tinybird/tb_cli_modules/datasource.py +1 -1
  35. {tinybird-4.6.20.dev0 → tinybird-4.6.21}/tinybird/tb_cli_modules/pipe.py +49 -34
  36. {tinybird-4.6.20.dev0 → tinybird-4.6.21}/tinybird/tb_cli_modules/tinyunit/tinyunit.py +4 -4
  37. {tinybird-4.6.20.dev0 → tinybird-4.6.21}/tinybird/tb_cli_modules/workspace.py +11 -13
  38. {tinybird-4.6.20.dev0 → tinybird-4.6.21}/tinybird/tornado_template.py +216 -17
  39. tinybird-4.6.21/tinybird/utils/__init__.py +0 -0
  40. tinybird-4.6.21/tinybird/utils/bools.py +19 -0
  41. {tinybird-4.6.20.dev0 → tinybird-4.6.21}/tinybird.egg-info/PKG-INFO +21 -1
  42. {tinybird-4.6.20.dev0 → tinybird-4.6.21}/tinybird.egg-info/SOURCES.txt +4 -1
  43. {tinybird-4.6.20.dev0 → tinybird-4.6.21}/setup.cfg +0 -0
  44. {tinybird-4.6.20.dev0 → tinybird-4.6.21}/tinybird/__cli__.py +0 -0
  45. {tinybird-4.6.20.dev0 → tinybird-4.6.21}/tinybird/check_pypi.py +0 -0
  46. {tinybird-4.6.20.dev0 → tinybird-4.6.21}/tinybird/config.py +0 -0
  47. {tinybird-4.6.20.dev0 → tinybird-4.6.21}/tinybird/context.py +0 -0
  48. {tinybird-4.6.20.dev0 → tinybird-4.6.21}/tinybird/datafile/exceptions.py +0 -0
  49. {tinybird-4.6.20.dev0 → tinybird-4.6.21}/tinybird/datafile/parse_connection.py +0 -0
  50. {tinybird-4.6.20.dev0 → tinybird-4.6.21}/tinybird/datafile/parse_datasource.py +0 -0
  51. {tinybird-4.6.20.dev0 → tinybird-4.6.21}/tinybird/datafile/parse_pipe.py +0 -0
  52. {tinybird-4.6.20.dev0 → tinybird-4.6.21}/tinybird/git_settings.py +0 -0
  53. {tinybird-4.6.20.dev0 → tinybird-4.6.21}/tinybird/iterating/__init__.py +0 -0
  54. {tinybird-4.6.20.dev0 → tinybird-4.6.21}/tinybird/iterating/data_branch_modes.py +0 -0
  55. {tinybird-4.6.20.dev0 → tinybird-4.6.21}/tinybird/prompts.py +0 -0
  56. {tinybird-4.6.20.dev0 → tinybird-4.6.21}/tinybird/sql_template_fmt.py +0 -0
  57. {tinybird-4.6.20.dev0 → tinybird-4.6.21}/tinybird/syncasync.py +0 -0
  58. {tinybird-4.6.20.dev0 → tinybird-4.6.21}/tinybird/tb/check_pypi.py +0 -0
  59. {tinybird-4.6.20.dev0 → tinybird-4.6.21}/tinybird/tb/cli.py +0 -0
  60. {tinybird-4.6.20.dev0 → tinybird-4.6.21}/tinybird/tb/config.py +0 -0
  61. {tinybird-4.6.20.dev0 → tinybird-4.6.21}/tinybird/tb/modules/branch.py +0 -0
  62. {tinybird-4.6.20.dev0 → tinybird-4.6.21}/tinybird/tb/modules/build.py +0 -0
  63. {tinybird-4.6.20.dev0 → tinybird-4.6.21}/tinybird/tb/modules/build_common.py +0 -0
  64. {tinybird-4.6.20.dev0 → tinybird-4.6.21}/tinybird/tb/modules/cicd.py +0 -0
  65. {tinybird-4.6.20.dev0 → tinybird-4.6.21}/tinybird/tb/modules/cli.py +0 -0
  66. {tinybird-4.6.20.dev0 → tinybird-4.6.21}/tinybird/tb/modules/config.py +0 -0
  67. {tinybird-4.6.20.dev0 → tinybird-4.6.21}/tinybird/tb/modules/connection.py +0 -0
  68. {tinybird-4.6.20.dev0 → tinybird-4.6.21}/tinybird/tb/modules/connection_dynamodb.py +0 -0
  69. {tinybird-4.6.20.dev0 → tinybird-4.6.21}/tinybird/tb/modules/connection_kafka.py +0 -0
  70. {tinybird-4.6.20.dev0 → tinybird-4.6.21}/tinybird/tb/modules/connection_s3.py +0 -0
  71. {tinybird-4.6.20.dev0 → tinybird-4.6.21}/tinybird/tb/modules/create.py +0 -0
  72. {tinybird-4.6.20.dev0 → tinybird-4.6.21}/tinybird/tb/modules/datafile/build_common.py +0 -0
  73. {tinybird-4.6.20.dev0 → tinybird-4.6.21}/tinybird/tb/modules/datafile/build_datasource.py +0 -0
  74. {tinybird-4.6.20.dev0 → tinybird-4.6.21}/tinybird/tb/modules/datafile/build_pipe.py +0 -0
  75. {tinybird-4.6.20.dev0 → tinybird-4.6.21}/tinybird/tb/modules/datafile/diff.py +0 -0
  76. {tinybird-4.6.20.dev0 → tinybird-4.6.21}/tinybird/tb/modules/datafile/fixture.py +0 -0
  77. {tinybird-4.6.20.dev0 → tinybird-4.6.21}/tinybird/tb/modules/datafile/format_common.py +0 -0
  78. {tinybird-4.6.20.dev0 → tinybird-4.6.21}/tinybird/tb/modules/datafile/format_connection.py +0 -0
  79. {tinybird-4.6.20.dev0 → tinybird-4.6.21}/tinybird/tb/modules/datafile/format_datasource.py +0 -0
  80. {tinybird-4.6.20.dev0 → tinybird-4.6.21}/tinybird/tb/modules/datafile/pipe_checker.py +0 -0
  81. {tinybird-4.6.20.dev0 → tinybird-4.6.21}/tinybird/tb/modules/datafile/pull.py +0 -0
  82. {tinybird-4.6.20.dev0 → tinybird-4.6.21}/tinybird/tb/modules/datasource.py +0 -0
  83. {tinybird-4.6.20.dev0 → tinybird-4.6.21}/tinybird/tb/modules/deployment.py +0 -0
  84. {tinybird-4.6.20.dev0 → tinybird-4.6.21}/tinybird/tb/modules/deprecations.py +0 -0
  85. {tinybird-4.6.20.dev0 → tinybird-4.6.21}/tinybird/tb/modules/endpoint.py +0 -0
  86. {tinybird-4.6.20.dev0 → tinybird-4.6.21}/tinybird/tb/modules/exceptions.py +0 -0
  87. {tinybird-4.6.20.dev0 → tinybird-4.6.21}/tinybird/tb/modules/fmt.py +0 -0
  88. {tinybird-4.6.20.dev0 → tinybird-4.6.21}/tinybird/tb/modules/info.py +0 -0
  89. {tinybird-4.6.20.dev0 → tinybird-4.6.21}/tinybird/tb/modules/job.py +0 -0
  90. {tinybird-4.6.20.dev0 → tinybird-4.6.21}/tinybird/tb/modules/job_common.py +0 -0
  91. {tinybird-4.6.20.dev0 → tinybird-4.6.21}/tinybird/tb/modules/llm.py +0 -0
  92. {tinybird-4.6.20.dev0 → tinybird-4.6.21}/tinybird/tb/modules/llm_utils.py +0 -0
  93. {tinybird-4.6.20.dev0 → tinybird-4.6.21}/tinybird/tb/modules/local.py +0 -0
  94. {tinybird-4.6.20.dev0 → tinybird-4.6.21}/tinybird/tb/modules/local_common.py +0 -0
  95. {tinybird-4.6.20.dev0 → tinybird-4.6.21}/tinybird/tb/modules/local_logs.py +0 -0
  96. {tinybird-4.6.20.dev0 → tinybird-4.6.21}/tinybird/tb/modules/login.py +0 -0
  97. {tinybird-4.6.20.dev0 → tinybird-4.6.21}/tinybird/tb/modules/login_common.py +0 -0
  98. {tinybird-4.6.20.dev0 → tinybird-4.6.21}/tinybird/tb/modules/logout.py +0 -0
  99. {tinybird-4.6.20.dev0 → tinybird-4.6.21}/tinybird/tb/modules/open.py +0 -0
  100. {tinybird-4.6.20.dev0 → tinybird-4.6.21}/tinybird/tb/modules/pipe.py +0 -0
  101. {tinybird-4.6.20.dev0 → tinybird-4.6.21}/tinybird/tb/modules/preview.py +0 -0
  102. {tinybird-4.6.20.dev0 → tinybird-4.6.21}/tinybird/tb/modules/project_commands.py +0 -0
  103. {tinybird-4.6.20.dev0 → tinybird-4.6.21}/tinybird/tb/modules/py_project.py +0 -0
  104. {tinybird-4.6.20.dev0 → tinybird-4.6.21}/tinybird/tb/modules/query_output.py +0 -0
  105. {tinybird-4.6.20.dev0 → tinybird-4.6.21}/tinybird/tb/modules/regions.py +0 -0
  106. {tinybird-4.6.20.dev0 → tinybird-4.6.21}/tinybird/tb/modules/secret.py +0 -0
  107. {tinybird-4.6.20.dev0 → tinybird-4.6.21}/tinybird/tb/modules/secret_common.py +0 -0
  108. {tinybird-4.6.20.dev0 → tinybird-4.6.21}/tinybird/tb/modules/sink.py +0 -0
  109. {tinybird-4.6.20.dev0 → tinybird-4.6.21}/tinybird/tb/modules/table.py +0 -0
  110. {tinybird-4.6.20.dev0 → tinybird-4.6.21}/tinybird/tb/modules/telemetry.py +0 -0
  111. {tinybird-4.6.20.dev0 → tinybird-4.6.21}/tinybird/tb/modules/test.py +0 -0
  112. {tinybird-4.6.20.dev0 → tinybird-4.6.21}/tinybird/tb/modules/test_common.py +0 -0
  113. {tinybird-4.6.20.dev0 → tinybird-4.6.21}/tinybird/tb/modules/ts_project.py +0 -0
  114. {tinybird-4.6.20.dev0 → tinybird-4.6.21}/tinybird/tb/modules/workspace.py +0 -0
  115. {tinybird-4.6.20.dev0 → tinybird-4.6.21}/tinybird/tb/modules/workspace_members.py +0 -0
  116. {tinybird-4.6.20.dev0 → tinybird-4.6.21}/tinybird/tb_cli.py +0 -0
  117. {tinybird-4.6.20.dev0 → tinybird-4.6.21}/tinybird/tb_cli_modules/auth.py +0 -0
  118. {tinybird-4.6.20.dev0 → tinybird-4.6.21}/tinybird/tb_cli_modules/connection.py +0 -0
  119. {tinybird-4.6.20.dev0 → tinybird-4.6.21}/tinybird/tb_cli_modules/exceptions.py +0 -0
  120. {tinybird-4.6.20.dev0 → tinybird-4.6.21}/tinybird/tb_cli_modules/fmt.py +0 -0
  121. {tinybird-4.6.20.dev0 → tinybird-4.6.21}/tinybird/tb_cli_modules/job.py +0 -0
  122. {tinybird-4.6.20.dev0 → tinybird-4.6.21}/tinybird/tb_cli_modules/regions.py +0 -0
  123. {tinybird-4.6.20.dev0 → tinybird-4.6.21}/tinybird/tb_cli_modules/tag.py +0 -0
  124. {tinybird-4.6.20.dev0 → tinybird-4.6.21}/tinybird/tb_cli_modules/telemetry.py +0 -0
  125. {tinybird-4.6.20.dev0 → tinybird-4.6.21}/tinybird/tb_cli_modules/test.py +0 -0
  126. {tinybird-4.6.20.dev0 → tinybird-4.6.21}/tinybird/tb_cli_modules/workspace_members.py +0 -0
  127. {tinybird-4.6.20.dev0 → tinybird-4.6.21}/tinybird.egg-info/dependency_links.txt +0 -0
  128. {tinybird-4.6.20.dev0 → tinybird-4.6.21}/tinybird.egg-info/entry_points.txt +0 -0
  129. {tinybird-4.6.20.dev0 → tinybird-4.6.21}/tinybird.egg-info/requires.txt +0 -0
  130. {tinybird-4.6.20.dev0 → tinybird-4.6.21}/tinybird.egg-info/top_level.txt +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.2
2
2
  Name: tinybird
3
- Version: 4.6.20.dev0
3
+ Version: 4.6.21
4
4
  Summary: Tinybird Command Line Tool
5
5
  Home-page: https://www.tinybird.co/docs/forward/commands
6
6
  Author: Tinybird
@@ -52,6 +52,21 @@ The Tinybird command-line tool allows you to use all the Tinybird functionality
52
52
  Changelog
53
53
  ----------
54
54
 
55
+ 4.6.21
56
+ ********
57
+
58
+ - `Fixed` ``tb deployment discard`` now discards the deployment that is not live, instead of assuming the live deployment is listed first. Previously, if removing the old deployment failed after a promotion, the command refused to discard it.
59
+
60
+ 4.6.20
61
+ ********
62
+
63
+ - `Added` Copy pipes accept ``ON_DEMAND_COMPUTE true`` so scheduled runs use on-demand compute. Manual ``tb copy run`` still requires ``--on-demand-compute``, and asks for confirmation when the local datafile does not enable it (skip it with ``--yes``).
64
+
65
+ 4.6.19
66
+ ********
67
+
68
+ - `Changed` ``tb datasource append`` and ``tb datasource replace`` submit remote CSV, NDJSON, and Parquet URLs through the v1 import endpoint by default.
69
+
55
70
  4.6.18
56
71
  ********
57
72
 
@@ -62,6 +77,11 @@ Changelog
62
77
 
63
78
  - `Added` ``tb job retry <job_id>`` to retry eligible import and S3/GCS sync jobs in ``error`` or ``cancelled`` status.
64
79
 
80
+ 4.6.18
81
+ ********
82
+
83
+ - `Added` ``Markdown`` is now a valid ``FORMAT`` for SQL queries (for example ``tb sql "SELECT 1 FORMAT Markdown"``), returned as a Markdown table.
84
+
65
85
  4.6.16
66
86
  ********
67
87
 
@@ -0,0 +1,240 @@
1
+ [build-system]
2
+ requires = [
3
+ "setuptools>=61.0",
4
+ "wheel==0.37.1",
5
+ "cffi==1.17.1",
6
+ ]
7
+ build-backend = "setuptools.build_meta"
8
+
9
+ [project]
10
+ name = "tinybird"
11
+ version = "1.0"
12
+ description = "tinybird"
13
+ readme = { text = "tinybird analytics", content-type = "text/markdown" }
14
+ requires-python = ">=3.13,<3.15"
15
+ authors = [
16
+ { name = "tinybird.co" },
17
+ ]
18
+ dependencies = [
19
+ "GitPython==3.1.44",
20
+ "Markdown==3.5.1",
21
+ "aioboto3==14.0.0",
22
+ "aiofiles==24.1.0",
23
+ "aiohttp==3.11.2",
24
+ "anthropic==0.55.0",
25
+ "async-lru==2.3.0",
26
+ "aws-msk-iam-sasl-signer-python==1.0.2",
27
+ "cachetools==5.5.0",
28
+ "cffi==1.17.1",
29
+ "chardet==4.0.0",
30
+ "ciso8601==2.2.0",
31
+ "click==8.1.8",
32
+ "clickhouse-cityhash==1.0.2.5",
33
+ "clickhouse-driver==0.2.9",
34
+ "confluent-kafka[avro,json,schemaregistry]==2.12.2",
35
+ "contextvars==2.4",
36
+ "cramjam==2.11.0",
37
+ "croniter==6.2.2",
38
+ "cryptography==3.4.8",
39
+ "datasketch==1.6.5",
40
+ "dnspython==2.8.0",
41
+ "docker==7.1.0",
42
+ "fastavro==1.12.2",
43
+ "fastmcp==2.12.2",
44
+ "google-cloud-aiplatform==1.148.1",
45
+ "google-cloud-compute==1.48.0",
46
+ "google-cloud-scheduler==2.20.0",
47
+ "google-cloud-secret-manager==2.29.0",
48
+ "google-cloud-storage==3.4.1",
49
+ "google_api_core==2.25.2",
50
+ "googleapis_common_protos==1.66.0",
51
+ "grpcio_status==1.67.1",
52
+ "humanfriendly==10.0",
53
+ "isal==1.8.0",
54
+ "jinja2",
55
+ "jsondiff==2.2.1",
56
+ "jsonschema==4.23.0",
57
+ "litellm==1.80.11",
58
+ "logfire-api==4.2.0",
59
+ "lru-dict==1.3.0",
60
+ "lz4==4.4.5",
61
+ "memray==1.20.0",
62
+ "msgspec==0.21.1",
63
+ "multidict==6.6.4",
64
+ "networkx==3.4.2",
65
+ "numpy==1.26.4",
66
+ "objgraph==3.6.2",
67
+ "opentelemetry-api==1.38.0",
68
+ "opentelemetry-exporter-otlp-proto-http==1.38.0",
69
+ "opentelemetry-sdk==1.38.0",
70
+ "orb-billing==2.17.0",
71
+ "orjson==3.11.5",
72
+ "packaging==23.2",
73
+ "passlib==1.7.1",
74
+ "posthog==7.28.0",
75
+ "pre-commit==4.6.2",
76
+ "prometheus_client==0.26.0",
77
+ "prompt_toolkit==3.0.48",
78
+ "propcache==0.3.2",
79
+ "proto_plus==1.25.0",
80
+ "protobuf==5.29.5",
81
+ "psutil==7.2.2",
82
+ "psycopg2-binary==2.9.12",
83
+ "py-spy==0.4.2",
84
+ "pyarrow==23.0.1",
85
+ "pycurl==7.45.7",
86
+ "pydantic-ai-slim[anthropic]==0.5.0",
87
+ "pydantic-ai-slim[google]==0.5.0",
88
+ "pydantic-ai-slim[retries]==0.5.0",
89
+ "pydantic==2.12.5",
90
+ "pyjwt[crypto]==2.9.0",
91
+ "pympler==1.1",
92
+ "pynacl==1.5.0",
93
+ "pyperclip==1.9.0",
94
+ "python-dateutil==2.9.0.post0",
95
+ "python-dotenv==1.1.0",
96
+ "python-ulid==2.7.0",
97
+ "pyyaml==6.0.1",
98
+ "redis[hiredis]==5.3.1",
99
+ "requests==2.32.4",
100
+ "sentry-sdk==2.69.1",
101
+ "setproctitle==1.3.3",
102
+ "shandy-sqlfmt==0.11.1",
103
+ "shandy-sqlfmt[jinjafmt]==0.11.1",
104
+ "sqlparse==0.4.3",
105
+ "starlette==1.0.1",
106
+ "streaming-form-data==1.20.0",
107
+ "stripe==3.1.0",
108
+ "tabulate==0.8.10",
109
+ "thefuzz==0.22.1",
110
+ "tinybird-python-sdk[ai]==0.3.7",
111
+ "tinybird-toolset==2.6.2",
112
+ "toposort==1.10",
113
+ "tornado==6.5.5",
114
+ "urllib3==1.26.20",
115
+ "uvicorn[standard]==0.49.0",
116
+ "virtualenv==20.21.1",
117
+ "watchdog==6.0.0",
118
+ "wheel==0.37.1",
119
+ "zlib-ng==1.0.0",
120
+ "zstandard==0.25.0",
121
+ ]
122
+
123
+ [dependency-groups]
124
+ deploy = [
125
+ "GitPython==3.1.44",
126
+ "ansible==8.5.0",
127
+ "ara==1.6.1",
128
+ "boto3==1.36.21",
129
+ "fabric==3.2.2",
130
+ "google-auth==2.47.0",
131
+ "jsonnet",
132
+ "prettytable==3.8.0",
133
+ "requests==2.32.4",
134
+ "requests[security]",
135
+ "xmltodict",
136
+ ]
137
+ devtools = [
138
+ "flameprof",
139
+ "matplotlib",
140
+ "memory_profiler",
141
+ "rope",
142
+ ]
143
+ doc = [
144
+ "Pallets-Sphinx-Themes==2.3.0",
145
+ "pyOpenSSL<24.0.0",
146
+ "sphinx-autobuild==2024.2.4",
147
+ "sphinx-sitemap",
148
+ "sphinx==7.3.7",
149
+ "sphinxcontrib-httpdomain",
150
+ "sphinxcontrib-jquery==4.1",
151
+ ]
152
+ lint = [
153
+ "botocore-stubs==1.36.23",
154
+ "flake8~=7.3",
155
+ "flake8-async~=26.8",
156
+ "mypy==2.3.1",
157
+ "ruff==0.16.8",
158
+ "types-Markdown",
159
+ "types-PyYAML",
160
+ "types-aiofiles==24.1.0.20241221",
161
+ "types-boto3-lite==1.36.21",
162
+ "types-cachetools",
163
+ "types-chardet",
164
+ "types-paramiko",
165
+ "types-pycurl",
166
+ "types-python-dateutil",
167
+ "types-pytz",
168
+ "types-requests",
169
+ "types-tabulate",
170
+ "types_croniter",
171
+ ]
172
+ test = [
173
+ "coverage==7.10.6",
174
+ "freezegun==1.5.5",
175
+ "google-auth==2.47.0",
176
+ "gsutil==4.58",
177
+ "hypothesis==6.156.1",
178
+ "oauth2client==3.0.0",
179
+ "py==1.11.0",
180
+ "pytest-asyncio==0.23.8",
181
+ "pytest-cov==4.1.0",
182
+ "pytest-env==1.1.3",
183
+ "pytest-flakefinder==1.1.0",
184
+ "pytest-timeout==2.3.1",
185
+ "pytest-xdist==3.6.1",
186
+ "pytest==8.1.2",
187
+ "rangehttpserver==1.4.0",
188
+ "yappi==1.7.6",
189
+ ]
190
+ # Synced automatically by plain `uv run`/`uv sync`, no `--group` needed.
191
+ dev = [
192
+ { include-group = "test" },
193
+ { include-group = "deploy" },
194
+ { include-group = "lint" },
195
+ ]
196
+
197
+ [project.scripts]
198
+ tinybird_server = "tinybird.app:run"
199
+ tinybird_tool = "tinybird.tinybird_tool.main:cli"
200
+ tb_old = "tinybird.tb_cli:cli"
201
+ tb = "tinybird.tb.cli:cli"
202
+ colibri = "tinybird.colibri.cli:cli"
203
+
204
+ [tool.uv.workspace]
205
+ members = [
206
+ "chbackups",
207
+ "data_connector",
208
+ ]
209
+
210
+ [tool.setuptools]
211
+ include-package-data = true
212
+
213
+ [tool.setuptools.packages.find]
214
+ namespaces = false
215
+ include = ["tinybird", "tinybird.*", "tinybird_shared", "tinybird_shared.*", "scripts"]
216
+ exclude = ["tests*"]
217
+
218
+ [tool.setuptools.package-data]
219
+ tinybird = [
220
+ "templates/**/*",
221
+ "static/**/*",
222
+ "default_pipes/**/*",
223
+ "sql/**/*",
224
+ "monitors/**/*.sql",
225
+ "scripts/**/*",
226
+ "data_projects/**/*",
227
+ "data_projects/**/.*",
228
+ "raw_events/**/*",
229
+ "query_booster/**/*",
230
+ "deployments/**/*",
231
+ ]
232
+ tinybird_shared = [
233
+ "redis_client/**/*",
234
+ "retry/**/*",
235
+ "metrics/**/*",
236
+ "gatherer_settings/**/*",
237
+ "clickhouse/**/*",
238
+ "gatherer_backpressure/**/*",
239
+ "data_connector_circuit_breaker/**/*",
240
+ ]
@@ -263,4 +263,5 @@ VALID_QUERY_FORMATS = (
263
263
  "JSONCompact",
264
264
  "JSONStringsEachRowWithProgress",
265
265
  "ODBCDriver2",
266
+ "Markdown",
266
267
  )
@@ -440,35 +440,64 @@ def get_engine_config(engine: str):
440
440
  )
441
441
 
442
442
 
443
- def engine_params(columns, params: List[EngineParam], args: Dict):
444
- params_values = []
443
+ def validate_engine_config(
444
+ engine: str, args: dict, schema: Optional[str] = None, columns: Optional[List[Dict[str, Any]]] = None
445
+ ) -> None:
446
+ if schema is not None and columns is not None:
447
+ raise ValueError("You can not use 'schema' and 'columns' at the same time")
448
+ _, (params, options) = get_engine_config(engine)
449
+ schema_columns = parse_table_structure(schema) if schema is not None else (columns or [])
450
+ engine_settings = {key.replace("engine_", ""): value for key, value in args.items()}
451
+
452
+ for arg in engine_settings:
453
+ if not hasattr(TableDetails, arg):
454
+ raise ValueError(f"engine_{arg} is not a valid option")
455
+
445
456
  for p in params:
446
- if p.required and p.name not in args:
457
+ if p.required and p.name not in engine_settings:
447
458
  raise ValueError(f"Missing required parameter '{p.name}'")
448
- param_value = args.get(p.name, None) or p.default_value
459
+ param_value = engine_settings.get(p.name) or p.default_value
449
460
  if not param_value:
450
461
  continue
451
462
  if p.is_valid:
452
463
  check_is_valid(
453
- valid_check=p.is_valid, check_type="parameter", columns=columns, tb_param=p.tb_param, value=param_value
464
+ valid_check=p.is_valid,
465
+ check_type="parameter",
466
+ columns=schema_columns,
467
+ tb_param=p.tb_param,
468
+ value=param_value,
454
469
  )
470
+
471
+ for o in options:
472
+ if o.required and o.name not in engine_settings:
473
+ raise ValueError(f"Missing required option '{o.name}'")
474
+ option_value = engine_settings.get(o.name) or o.default_value
475
+ if o.is_valid:
476
+ check_is_valid(
477
+ valid_check=o.is_valid,
478
+ check_type="option",
479
+ columns=schema_columns,
480
+ tb_param=o.tb_param,
481
+ value=option_value,
482
+ )
483
+
484
+
485
+ def engine_params(params: List[EngineParam], args: Dict) -> List[str]:
486
+ params_values = []
487
+ for p in params:
488
+ param_value = args.get(p.name, None) or p.default_value
489
+ if not param_value:
490
+ continue
455
491
  params_values.append(param_value)
456
492
  return params_values
457
493
 
458
494
 
459
- def engine_options(columns, options: List[EngineOption], args: Dict):
495
+ def engine_options(options: List[EngineOption], args: Dict) -> List[str]:
460
496
  options_values = []
461
497
  engine_settings = ""
462
498
 
463
499
  for o in options:
464
- if o.required and o.name not in args:
465
- raise ValueError(f"Missing required option '{o.name}'")
466
500
  option_value = args.get(o.name) or o.default_value
467
- if o.is_valid:
468
- check_is_valid(
469
- valid_check=o.is_valid, check_type="option", columns=columns, tb_param=o.tb_param, value=option_value
470
- )
471
-
472
501
  if option_value:
473
502
  if o.sql.lower() == "settings":
474
503
  engine_settings = f"{o.sql} {option_value}"
@@ -490,7 +519,7 @@ def check_is_valid(
490
519
  columns: List[Dict[str, Any]],
491
520
  tb_param: str,
492
521
  value: str,
493
- ):
522
+ ) -> None:
494
523
  """
495
524
  >>> check_is_valid(sorting_key_is_valid, 'option', ['column-name'], 'sorting_key', 'date')
496
525
 
@@ -507,15 +536,13 @@ def check_is_valid(
507
536
  raise ValueError(f"Invalid value '{value}' for {check_type} '{tb_param}', reason: {e}")
508
537
 
509
538
 
510
- def build_engine(
511
- engine: str, columns: Optional[List], params: List[EngineParam], options: List[EngineOption], args: Dict
512
- ):
513
- return f"{engine}({', '.join(engine_params(columns, params, args))}) {' '.join(engine_options(columns, options, args))}".strip()
539
+ def build_engine(engine: str, params: List[EngineParam], options: List[EngineOption], args: Dict) -> str:
540
+ return f"{engine}({', '.join(engine_params(params, args))}) {' '.join(engine_options(options, args))}".strip()
514
541
 
515
542
 
516
543
  def engine_full_from_dict(
517
544
  engine: str, args: dict, schema: Optional[str] = None, columns: Optional[List[Dict[str, Any]]] = None
518
- ):
545
+ ) -> str:
519
546
  """
520
547
  >>> schema = ''
521
548
  >>> engine_full_from_dict('wadus', {}, schema=schema)
@@ -606,20 +633,10 @@ def engine_full_from_dict(
606
633
  'ReplacingMergeTree(updated_at) PARTITION BY (tuple()) ORDER BY (project_id) SETTINGS index_granularity = 32'
607
634
  """
608
635
 
609
- if schema is not None and columns is not None:
610
- raise ValueError("You can not use 'schema' and 'columns' at the same time")
611
- engine_config = get_engine_config(engine)
612
- name, (params, options) = engine_config
613
- if columns is None and schema is not None:
614
- columns = parse_table_structure(schema)
615
-
636
+ validate_engine_config(engine, args, schema=schema, columns=columns)
637
+ name, (params, options) = get_engine_config(engine)
616
638
  engine_settings = {key.replace("engine_", ""): value for key, value in args.items()}
617
-
618
- for arg in engine_settings:
619
- if not hasattr(TableDetails, arg):
620
- raise ValueError(f"engine_{arg} is not a valid option")
621
-
622
- return build_engine(name, columns, params, options, engine_settings)
639
+ return build_engine(name, params, options, engine_settings)
623
640
 
624
641
 
625
642
  def engine_params_from_engine_full(engine_full: str) -> Dict[str, Any]:
@@ -408,6 +408,16 @@ class TinyB:
408
408
  format: str = "csv",
409
409
  replace_options: Optional[Set[str]] = None,
410
410
  ) -> Dict[str, Any]:
411
+ if mode in ("append", "replace"):
412
+ return await self.datasource_append_url(
413
+ table_name,
414
+ url,
415
+ mode=mode,
416
+ sql_condition=sql_condition,
417
+ format=format,
418
+ replace_options=replace_options,
419
+ )
420
+
411
421
  params = {"name": table_name, "url": url, "mode": mode, "debug": "blocks_block_log", "format": format}
412
422
 
413
423
  if sql_condition:
@@ -424,6 +434,33 @@ class TinyB:
424
434
 
425
435
  return await self.wait_for_job(res["id"], status_callback, backoff_multiplier=1.5, maximum_backoff_seconds=20)
426
436
 
437
+ async def datasource_append_url(
438
+ self,
439
+ datasource_name: str,
440
+ url: str,
441
+ mode: str = "append",
442
+ sql_condition: Optional[str] = None,
443
+ format: str = "csv",
444
+ replace_options: Optional[Set[str]] = None,
445
+ ) -> Dict[str, Any]:
446
+ params = {"format": format, "url": url}
447
+ if sql_condition:
448
+ params["replace_condition"] = sql_condition
449
+ if replace_options:
450
+ for option in replace_options:
451
+ params[option] = "true"
452
+
453
+ res = await self._req(
454
+ f"/v1/datasources/{quote(datasource_name, safe='')}/{mode}?{urlencode(params, safe='')}",
455
+ method="POST",
456
+ data=b"",
457
+ )
458
+ if not isinstance(res, dict):
459
+ raise RuntimeError("We couldn't confirm that your import started. Please try again.")
460
+ if "error" in res:
461
+ raise Exception(res["error"])
462
+ return res
463
+
427
464
  async def datasource_delete(self, datasource_name: str, force: bool = False, dry_run: bool = False):
428
465
  params = {"force": "true" if force else "false", "dry_run": "true" if dry_run else "false"}
429
466
  return await self._req(f"/v0/datasources/{datasource_name}?{urlencode(params)}", method="DELETE")
@@ -654,6 +691,7 @@ class TinyB:
654
691
  target_datasource: Optional[str] = None,
655
692
  schedule_cron: Optional[str] = None,
656
693
  mode: Optional[str] = None,
694
+ on_demand_compute: Optional[bool] = None,
657
695
  ):
658
696
  data = {"schedule_cron": schedule_cron}
659
697
 
@@ -663,6 +701,9 @@ class TinyB:
663
701
  if mode:
664
702
  data["mode"] = mode
665
703
 
704
+ if on_demand_compute is not None:
705
+ data["on_demand_compute"] = "true" if on_demand_compute else "false"
706
+
666
707
  return await self._req(f"/v0/pipes/{pipe_name_or_id}/nodes/{node_id}/copy", method="PUT", data=data)
667
708
 
668
709
  async def pipe_remove_copy(self, pipe_name_or_id: str, node_id: str):
@@ -28,6 +28,7 @@ from tinybird.ch_utils.engine import VALID_ENGINE_PARAMS, EngineParam
28
28
  from tinybird.datafile.exceptions import IncludeFileNotFoundException, ParseException, ValidationException
29
29
  from tinybird.tb.modules.exceptions import CLIPipeException
30
30
  from tinybird.tb.modules.feedback_manager import FeedbackManager
31
+ from tinybird.utils.bools import parse_optional_bool
31
32
 
32
33
  # Code from sql.py has been duplicated so I can change it without breaking absolutely everything in the app
33
34
  # I'll try not to make logic changes, just error reporting changes
@@ -179,9 +180,10 @@ class CopyParameters(Parameters):
179
180
  COPY_SCHEDULE = "copy_schedule"
180
181
  COPY_MODE = "copy_mode"
181
182
  COPY_MODE_ALIAS = "mode" # we need this because bot MODE and COPY_MODE go to `mode` variable inside the node
183
+ ON_DEMAND_COMPUTE = "on_demand_compute"
182
184
  MANDATORY_ATTRIBUTES = PipeParameters.MANDATORY_ATTRIBUTES.union({TARGET_DATASOURCE})
183
185
  ACCEPTED_ATTRIBUTES = PipeParameters.ACCEPTED_ATTRIBUTES.union(MANDATORY_ATTRIBUTES).union(
184
- {COPY_SCHEDULE, COPY_MODE_ALIAS}
186
+ {COPY_SCHEDULE, COPY_MODE_ALIAS, ON_DEMAND_COMPUTE}
185
187
  )
186
188
 
187
189
 
@@ -416,6 +418,11 @@ class Datafile:
416
418
  )
417
419
  if not croniter.is_valid(copy_schedule):
418
420
  raise DatafileValidationError("COPY node schedule must be @on-demand or a valid cron expression.")
421
+ if CopyParameters.ON_DEMAND_COMPUTE in node:
422
+ try:
423
+ node[CopyParameters.ON_DEMAND_COMPUTE] = parse_optional_bool(node[CopyParameters.ON_DEMAND_COMPUTE])
424
+ except ValueError:
425
+ raise DatafileValidationError("ON_DEMAND_COMPUTE must be true or false")
419
426
  for key in node.keys():
420
427
  if key not in CopyParameters.valid_params():
421
428
  raise DatafileValidationError(
@@ -2097,6 +2104,7 @@ def parse(
2097
2104
  "copy_schedule": assign_node_var(CopyParameters.COPY_SCHEDULE),
2098
2105
  "copy_mode": assign_node_var("mode"),
2099
2106
  "mode": assign_node_var("mode"),
2107
+ "on_demand_compute": assign_node_var(CopyParameters.ON_DEMAND_COMPUTE),
2100
2108
  "filter": assign_node_var("filter"),
2101
2109
  "token": add_token,
2102
2110
  "include": include,
@@ -120,52 +120,74 @@ def datetime_test(x: str) -> bool:
120
120
  return any(p.match(x) for p in datetime_patterns)
121
121
 
122
122
 
123
+ def bounded_integer(x: str, pattern: re.Pattern[str]) -> Optional[int]:
124
+ if pattern.match(x) is None:
125
+ return None
126
+ digits = x.lstrip("+-").lstrip("0")
127
+ # 256-bit unsigned integers have at most 78 decimal digits.
128
+ if len(digits) > 78:
129
+ return None
130
+ return int(x) if len(x) <= 78 else int(digits or "0") * (-1 if x.startswith("-") else 1)
131
+
132
+
123
133
  def int_8_test(x: str) -> bool:
124
- return intx_re.match(x) is not None and -int_8_max <= int(x) < int_8_max
134
+ value = bounded_integer(x, intx_re)
135
+ return value is not None and -int_8_max <= value < int_8_max
125
136
 
126
137
 
127
138
  def int16_test(x: str) -> bool:
128
- return intx_re.match(x) is not None and -int16_max <= int(x) < int16_max
139
+ value = bounded_integer(x, intx_re)
140
+ return value is not None and -int16_max <= value < int16_max
129
141
 
130
142
 
131
143
  def int32_test(x: str) -> bool:
132
- return intx_re.match(x) is not None and -int32_max <= int(x) < int32_max
144
+ value = bounded_integer(x, intx_re)
145
+ return value is not None and -int32_max <= value < int32_max
133
146
 
134
147
 
135
148
  def int64_test(x: str) -> bool:
136
- return intx_re.match(x) is not None and -int64_max <= int(x) < int64_max
149
+ value = bounded_integer(x, intx_re)
150
+ return value is not None and -int64_max <= value < int64_max
137
151
 
138
152
 
139
153
  def int128_test(x: str) -> bool:
140
- return intx_re.match(x) is not None and -int128_max <= int(x) < int128_max
154
+ value = bounded_integer(x, intx_re)
155
+ return value is not None and -int128_max <= value < int128_max
141
156
 
142
157
 
143
158
  def int256_test(x: str) -> bool:
144
- return intx_re.match(x) is not None and -int256_max <= int(x) < int256_max
159
+ value = bounded_integer(x, intx_re)
160
+ return value is not None and -int256_max <= value < int256_max
145
161
 
146
162
 
147
163
  def uint_8_test(x: str) -> bool:
148
- return uintx_re.match(x) is not None and 0 <= int(x) < uint_8_max
164
+ value = bounded_integer(x, uintx_re)
165
+ return value is not None and 0 <= value < uint_8_max
149
166
 
150
167
 
151
168
  def uint16_test(x: str) -> bool:
152
- return uintx_re.match(x) is not None and 0 <= int(x) < uint16_max
169
+ value = bounded_integer(x, uintx_re)
170
+ return value is not None and 0 <= value < uint16_max
153
171
 
154
172
 
155
173
  def uint32_test(x: str) -> bool:
156
- return uintx_re.match(x) is not None and 0 <= int(x) < uint32_max
174
+ value = bounded_integer(x, uintx_re)
175
+ return value is not None and 0 <= value < uint32_max
157
176
 
158
177
 
159
178
  def uint64_test(x: str) -> bool:
160
- return uintx_re.match(x) is not None and 0 <= int(x) < uint64_max
179
+ value = bounded_integer(x, uintx_re)
180
+ return value is not None and 0 <= value < uint64_max
161
181
 
162
182
 
163
183
  def uint128_test(x: str) -> bool:
164
- return intx_re.match(x) is not None and 0 <= int(x) < uint128_max
184
+ value = bounded_integer(x, intx_re)
185
+ return value is not None and 0 <= value < uint128_max
165
186
 
166
187
 
167
188
  def uint256_test(x: str) -> bool:
168
- return intx_re.match(x) is not None and 0 <= int(x) < uint256_max
189
+ value = bounded_integer(x, intx_re)
190
+ return value is not None and 0 <= value < uint256_max
169
191
 
170
192
 
171
193
  def float_test(x: str) -> bool:
@@ -572,6 +572,11 @@ class FeedbackManager:
572
572
  "You can read more about the pricing at https://www.tinybird.co/docs/classic/work-with-data/process-and-copy/materialized-views#compute-compute-separation-for-populates\n\n"
573
573
  "Do you want to proceed with on-demand compute?"
574
574
  )
575
+ warning_confirm_on_demand_compute_differs_from_datafile = warning_message(
576
+ "The local datafile of pipe '{pipe}' does not set ON_DEMAND_COMPUTE true, but --on-demand-compute was passed.\n"
577
+ "On-demand compute uses dedicated resources that are billed separately from your regular usage.\n\n"
578
+ "Do you want to proceed?"
579
+ )
575
580
 
576
581
  warning_confirm_delete_token = prompt_message("Do you want to delete Token {token}?")
577
582
  warning_confirm_refresh_token = prompt_message("Do you want to refresh Token {token}?")