tinybird 4.6.20.dev0__tar.gz → 4.6.21.dev0__tar.gz

{tinybird-4.6.20.dev0 → tinybird-4.6.21.dev0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: tinybird
-Version: 4.6.20.dev0
+Version: 4.6.21.dev0
 Summary: Tinybird Command Line Tool
 Home-page: https://www.tinybird.co/docs/forward/commands
 Author: Tinybird
@@ -52,6 +52,11 @@ The Tinybird command-line tool allows you to use all the Tinybird functionality
 Changelog
 ----------
 
+4.6.19
+********
+
+- `Changed` ``tb datasource append`` and ``tb datasource replace`` submit remote CSV, NDJSON, and Parquet URLs through the v1 import endpoint by default.
+
 4.6.18
 ********
 
@@ -62,6 +67,11 @@ Changelog
 
 - `Added` ``tb job retry <job_id>`` to retry eligible import and S3/GCS sync jobs in ``error`` or ``cancelled`` status.
 
+4.6.18
+********
+
+- `Added` ``Markdown`` is now a valid ``FORMAT`` for SQL queries (for example ``tb sql "SELECT 1 FORMAT Markdown"``), returned as a Markdown table.
+
 4.6.16
 ********
 

tinybird-4.6.21.dev0/pyproject.toml

@@ -0,0 +1,228 @@
+[build-system]
+requires = [
+    "setuptools>=61.0",
+    "wheel==0.37.1",
+    "cffi==1.17.1",
+]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "tinybird"
+version = "1.0"
+description = "tinybird"
+readme = { text = "tinybird analytics", content-type = "text/markdown" }
+requires-python = ">=3.13,<3.15"
+authors = [
+    { name = "tinybird.co" },
+]
+dependencies = [
+    "GitPython==3.1.44",
+    "Markdown==3.5.1",
+    "aioboto3==14.0.0",
+    "aiofiles==24.1.0",
+    "aiohttp==3.11.2",
+    "anthropic==0.55.0",
+    "aws-msk-iam-sasl-signer-python==1.0.2",
+    "cachetools==5.5.0",
+    "cffi==1.17.1",
+    "chardet==4.0.0",
+    "ciso8601==2.2.0",
+    "click==8.1.8",
+    "clickhouse-cityhash==1.0.2.5",
+    "clickhouse-driver==0.2.9",
+    "confluent-kafka[avro,json,schemaregistry]==2.12.2",
+    "contextvars==2.4",
+    "cramjam==2.11.0",
+    "croniter==6.2.2",
+    "cryptography==3.4.8",
+    "datasketch==1.6.5",
+    "dnspython==2.8.0",
+    "docker==7.1.0",
+    "fastavro==1.12.2",
+    "fastmcp==2.12.2",
+    "google-cloud-aiplatform==1.148.1",
+    "google-cloud-compute==1.48.0",
+    "google-cloud-scheduler==2.20.0",
+    "google-cloud-secret-manager==2.29.0",
+    "google-cloud-storage==3.4.1",
+    "google_api_core==2.25.2",
+    "googleapis_common_protos==1.66.0",
+    "grpcio_status==1.67.1",
+    "humanfriendly==10.0",
+    "isal==1.8.0",
+    "jinja2",
+    "jsondiff==2.2.1",
+    "jsonschema==4.23.0",
+    "litellm==1.80.11",
+    "logfire-api==4.2.0",
+    "lru-dict==1.3.0",
+    "lz4==4.4.5",
+    "memray==1.20.0",
+    "msgspec==0.21.1",
+    "multidict==6.6.4",
+    "networkx==3.4.2",
+    "numpy==1.26.4",
+    "objgraph==3.6.2",
+    "opentelemetry-api==1.38.0",
+    "opentelemetry-exporter-otlp-proto-http==1.38.0",
+    "opentelemetry-sdk==1.38.0",
+    "orb-billing==2.17.0",
+    "orjson==3.11.5",
+    "packaging==23.2",
+    "passlib==1.7.1",
+    "posthog==7.28.0",
+    "pre-commit==4.6.2",
+    "prometheus_client==0.26.0",
+    "prompt_toolkit==3.0.48",
+    "propcache==0.3.2",
+    "proto_plus==1.25.0",
+    "protobuf==5.29.5",
+    "psutil==7.2.2",
+    "psycopg2-binary==2.9.12",
+    "py-spy==0.4.2",
+    "pyarrow==23.0.1",
+    "pycurl==7.45.7",
+    "pydantic-ai-slim[anthropic]==0.5.0",
+    "pydantic-ai-slim[google]==0.5.0",
+    "pydantic-ai-slim[retries]==0.5.0",
+    "pydantic==2.12.5",
+    "pyjwt[crypto]==2.9.0",
+    "pympler==1.1",
+    "pynacl==1.5.0",
+    "pyperclip==1.9.0",
+    "python-dateutil==2.9.0.post0",
+    "python-dotenv==1.1.0",
+    "python-ulid==2.7.0",
+    "pyyaml==6.0.1",
+    "redis[hiredis]==5.3.1",
+    "requests==2.32.4",
+    "sentry-sdk==2.69.1",
+    "setproctitle==1.3.3",
+    "shandy-sqlfmt==0.11.1",
+    "shandy-sqlfmt[jinjafmt]==0.11.1",
+    "sqlparse==0.4.3",
+    "starlette==1.0.1",
+    "streaming-form-data==1.20.0",
+    "stripe==3.1.0",
+    "tabulate==0.8.10",
+    "thefuzz==0.22.1",
+    "tinybird-python-sdk[ai]==0.3.7",
+    "tinybird-toolset==2.6.2",
+    "toposort==1.10",
+    "tornado==6.5.5",
+    "urllib3==1.26.20",
+    "uvicorn[standard]==0.49.0",
+    "virtualenv==20.21.1",
+    "watchdog==6.0.0",
+    "wheel==0.37.1",
+    "zlib-ng==1.0.0",
+    "zstandard==0.25.0",
+]
+
+[dependency-groups]
+deploy = [
+    "GitPython==3.1.44",
+    "ansible==8.5.0",
+    "ara==1.6.1",
+    "boto3==1.36.21",
+    "fabric==3.2.2",
+    "google-auth==2.47.0",
+    "jsonnet",
+    "prettytable==3.8.0",
+    "requests==2.32.4",
+    "requests[security]",
+    "xmltodict",
+]
+devtools = [
+    "flameprof",
+    "matplotlib",
+    "memory_profiler",
+    "rope",
+]
+doc = [
+    "Pallets-Sphinx-Themes==2.3.0",
+    "pyOpenSSL<24.0.0",
+    "sphinx-autobuild==2024.2.4",
+    "sphinx-sitemap",
+    "sphinx==7.3.7",
+    "sphinxcontrib-httpdomain",
+    "sphinxcontrib-jquery==4.1",
+]
+lint = [
+    "botocore-stubs==1.36.23",
+    "mypy==2.3.1",
+    "ruff==0.16.8",
+    "types-Markdown",
+    "types-PyYAML",
+    "types-aiofiles==24.1.0.20241221",
+    "types-boto3-lite==1.36.21",
+    "types-cachetools",
+    "types-chardet",
+    "types-paramiko",
+    "types-pycurl",
+    "types-python-dateutil",
+    "types-pytz",
+    "types-requests",
+    "types-tabulate",
+    "types_croniter",
+]
+test = [
+    "coverage==7.10.6",
+    "freezegun==1.5.5",
+    "google-auth==2.47.0",
+    "gsutil==4.58",
+    "hypothesis==6.156.1",
+    "oauth2client==3.0.0",
+    "py==1.11.0",
+    "pytest-asyncio==0.23.8",
+    "pytest-cov==4.1.0",
+    "pytest-env==1.1.3",
+    "pytest-flakefinder==1.1.0",
+    "pytest-timeout==2.3.1",
+    "pytest-xdist==3.6.1",
+    "pytest==8.1.2",
+    "rangehttpserver==1.4.0",
+    "yappi==1.7.6",
+]
+
+[project.scripts]
+tinybird_server = "tinybird.app:run"
+tinybird_tool = "tinybird.tinybird_tool.main:cli"
+tb_old = "tinybird.tb_cli:cli"
+tb = "tinybird.tb.cli:cli"
+colibri = "tinybird.colibri.cli:cli"
+
+[tool.uv.workspace]
+members = ["data_connector"]
+
+[tool.setuptools]
+include-package-data = true
+
+[tool.setuptools.packages.find]
+namespaces = false
+include = ["tinybird", "tinybird.*", "tinybird_shared", "tinybird_shared.*", "scripts"]
+exclude = ["tests*"]
+
+[tool.setuptools.package-data]
+tinybird = [
+    "templates/**/*",
+    "static/**/*",
+    "default_pipes/**/*",
+    "sql/**/*",
+    "monitors/**/*.sql",
+    "scripts/**/*",
+    "data_projects/**/*",
+    "data_projects/**/.*",
+    "raw_events/**/*",
+    "query_booster/**/*",
+    "deployments/**/*",
+]
+tinybird_shared = [
+    "redis_client/**/*",
+    "retry/**/*",
+    "metrics/**/*",
+    "gatherer_settings/**/*",
+    "clickhouse/**/*",
+    "gatherer_backpressure/**/*",
+    "data_connector_circuit_breaker/**/*",
+]

{tinybird-4.6.20.dev0 → tinybird-4.6.21.dev0}/tinybird/ch_utils/constants.py

@@ -263,4 +263,5 @@ VALID_QUERY_FORMATS = (
     "JSONCompact",
     "JSONStringsEachRowWithProgress",
     "ODBCDriver2",
+    "Markdown",
 )

{tinybird-4.6.20.dev0 → tinybird-4.6.21.dev0}/tinybird/ch_utils/engine.py

@@ -440,35 +440,64 @@ def get_engine_config(engine: str):
     )
 
 
-def engine_params(columns, params: List[EngineParam], args: Dict):
-    params_values = []
+def validate_engine_config(
+    engine: str, args: dict, schema: Optional[str] = None, columns: Optional[List[Dict[str, Any]]] = None
+) -> None:
+    if schema is not None and columns is not None:
+        raise ValueError("You can not use 'schema' and 'columns' at the same time")
+    _, (params, options) = get_engine_config(engine)
+    schema_columns = parse_table_structure(schema) if schema is not None else (columns or [])
+    engine_settings = {key.replace("engine_", ""): value for key, value in args.items()}
+
+    for arg in engine_settings:
+        if not hasattr(TableDetails, arg):
+            raise ValueError(f"engine_{arg} is not a valid option")
+
     for p in params:
-        if p.required and p.name not in args:
+        if p.required and p.name not in engine_settings:
             raise ValueError(f"Missing required parameter '{p.name}'")
-        param_value = args.get(p.name, None) or p.default_value
+        param_value = engine_settings.get(p.name) or p.default_value
         if not param_value:
             continue
         if p.is_valid:
             check_is_valid(
-                valid_check=p.is_valid, check_type="parameter", columns=columns, tb_param=p.tb_param, value=param_value
+                valid_check=p.is_valid,
+                check_type="parameter",
+                columns=schema_columns,
+                tb_param=p.tb_param,
+                value=param_value,
             )
+
+    for o in options:
+        if o.required and o.name not in engine_settings:
+            raise ValueError(f"Missing required option '{o.name}'")
+        option_value = engine_settings.get(o.name) or o.default_value
+        if o.is_valid:
+            check_is_valid(
+                valid_check=o.is_valid,
+                check_type="option",
+                columns=schema_columns,
+                tb_param=o.tb_param,
+                value=option_value,
+            )
+
+
+def engine_params(params: List[EngineParam], args: Dict) -> List[str]:
+    params_values = []
+    for p in params:
+        param_value = args.get(p.name, None) or p.default_value
+        if not param_value:
+            continue
         params_values.append(param_value)
     return params_values
 
 
-def engine_options(columns, options: List[EngineOption], args: Dict):
+def engine_options(options: List[EngineOption], args: Dict) -> List[str]:
     options_values = []
     engine_settings = ""
 
     for o in options:
-        if o.required and o.name not in args:
-            raise ValueError(f"Missing required option '{o.name}'")
         option_value = args.get(o.name) or o.default_value
-        if o.is_valid:
-            check_is_valid(
-                valid_check=o.is_valid, check_type="option", columns=columns, tb_param=o.tb_param, value=option_value
-            )
-
         if option_value:
             if o.sql.lower() == "settings":
                 engine_settings = f"{o.sql} {option_value}"
@@ -490,7 +519,7 @@ def check_is_valid(
     columns: List[Dict[str, Any]],
     tb_param: str,
     value: str,
-):
+) -> None:
     """
     >>> check_is_valid(sorting_key_is_valid, 'option', ['column-name'], 'sorting_key', 'date')
 
@@ -507,15 +536,13 @@ def check_is_valid(
         raise ValueError(f"Invalid value '{value}' for {check_type} '{tb_param}', reason: {e}")
 
 
-def build_engine(
-    engine: str, columns: Optional[List], params: List[EngineParam], options: List[EngineOption], args: Dict
-):
-    return f"{engine}({', '.join(engine_params(columns, params, args))}) {' '.join(engine_options(columns, options, args))}".strip()
+def build_engine(engine: str, params: List[EngineParam], options: List[EngineOption], args: Dict) -> str:
+    return f"{engine}({', '.join(engine_params(params, args))}) {' '.join(engine_options(options, args))}".strip()
 
 
 def engine_full_from_dict(
     engine: str, args: dict, schema: Optional[str] = None, columns: Optional[List[Dict[str, Any]]] = None
-):
+) -> str:
     """
     >>> schema = ''
     >>> engine_full_from_dict('wadus', {}, schema=schema)
@@ -606,20 +633,10 @@ def engine_full_from_dict(
     'ReplacingMergeTree(updated_at) PARTITION BY (tuple()) ORDER BY (project_id) SETTINGS index_granularity = 32'
     """
 
-    if schema is not None and columns is not None:
-        raise ValueError("You can not use 'schema' and 'columns' at the same time")
-    engine_config = get_engine_config(engine)
-    name, (params, options) = engine_config
-    if columns is None and schema is not None:
-        columns = parse_table_structure(schema)
-
+    validate_engine_config(engine, args, schema=schema, columns=columns)
+    name, (params, options) = get_engine_config(engine)
     engine_settings = {key.replace("engine_", ""): value for key, value in args.items()}
-
-    for arg in engine_settings:
-        if not hasattr(TableDetails, arg):
-            raise ValueError(f"engine_{arg} is not a valid option")
-
-    return build_engine(name, columns, params, options, engine_settings)
+    return build_engine(name, params, options, engine_settings)
 
 
 def engine_params_from_engine_full(engine_full: str) -> Dict[str, Any]:

{tinybird-4.6.20.dev0 → tinybird-4.6.21.dev0}/tinybird/client.py

@@ -408,6 +408,16 @@ class TinyB:
         format: str = "csv",
         replace_options: Optional[Set[str]] = None,
     ) -> Dict[str, Any]:
+        if mode in ("append", "replace"):
+            return await self.datasource_append_url(
+                table_name,
+                url,
+                mode=mode,
+                sql_condition=sql_condition,
+                format=format,
+                replace_options=replace_options,
+            )
+
         params = {"name": table_name, "url": url, "mode": mode, "debug": "blocks_block_log", "format": format}
 
         if sql_condition:
@@ -424,6 +434,33 @@ class TinyB:
 
         return await self.wait_for_job(res["id"], status_callback, backoff_multiplier=1.5, maximum_backoff_seconds=20)
 
+    async def datasource_append_url(
+        self,
+        datasource_name: str,
+        url: str,
+        mode: str = "append",
+        sql_condition: Optional[str] = None,
+        format: str = "csv",
+        replace_options: Optional[Set[str]] = None,
+    ) -> Dict[str, Any]:
+        params = {"format": format, "url": url}
+        if sql_condition:
+            params["replace_condition"] = sql_condition
+        if replace_options:
+            for option in replace_options:
+                params[option] = "true"
+
+        res = await self._req(
+            f"/v1/datasources/{quote(datasource_name, safe='')}/{mode}?{urlencode(params, safe='')}",
+            method="POST",
+            data=b"",
+        )
+        if not isinstance(res, dict):
+            raise RuntimeError("We couldn't confirm that your import started. Please try again.")
+        if "error" in res:
+            raise Exception(res["error"])
+        return res
+
     async def datasource_delete(self, datasource_name: str, force: bool = False, dry_run: bool = False):
         params = {"force": "true" if force else "false", "dry_run": "true" if dry_run else "false"}
         return await self._req(f"/v0/datasources/{datasource_name}?{urlencode(params)}", method="DELETE")

{tinybird-4.6.20.dev0 → tinybird-4.6.21.dev0}/tinybird/service_datasources.py

@@ -39,6 +39,7 @@ def get_tinybird_service_datasources() -> List[Dict[str, Any]]:
                 {"name": "result_rows", "type": "UInt64"},
                 {"name": "url", "type": "Nullable(String)"},
                 {"name": "error", "type": "UInt8"},
+                {"name": "error_message", "type": "Nullable(String)"},
                 {"name": "status_code", "type": "Int32"},
                 {"name": "request_id", "type": "String"},
                 {"name": "parameters", "type": "Map(String, String)"},
@@ -443,25 +444,21 @@ def get_tinybird_service_datasources() -> List[Dict[str, Any]]:
             "dateColumn": "run_validation",
             "engine": {
                 "engine": "MergeTree",
-                "sorting_key": "database, host, run_validation, query_hash",
+                "sorting_key": "workspace, host, query_hash, run_validation",
                 "partition_key": "toYYYYMM(run_validation)",
             },
             "columns": [
-                {"name": "host", "type": "LowCardinality(String)"},
                 {"name": "version", "type": "LowCardinality(String)"},
                 {"name": "stable_version", "type": "LowCardinality(String)"},
                 {"name": "query_hash", "type": "UInt64"},
                 {"name": "query_last_execution", "type": "DateTime"},
-                {"name": "region", "type": "String"},
-                {"name": "workspace", "type": "String"},
-                {"name": "database", "type": "String"},
-                {"name": "pipe_name", "type": "String"},
+                {"name": "resource_type", "type": "LowCardinality(String)"},
+                {"name": "resource_name", "type": "String"},
                 {"name": "query_id", "type": "String"},
                 {"name": "query", "type": "String"},
                 {"name": "error_code", "type": "Int16"},
                 {"name": "error", "type": "String"},
                 {"name": "fix_suggestion", "type": "String"},
-                {"name": "fix_suggestion_ai", "type": "String"},
                 {"name": "run_validation", "type": "DateTime"},
             ],
         },
@@ -680,6 +677,7 @@ def get_organization_service_datasources() -> List[Dict[str, Any]]:
                 {"name": "result_rows", "type": "UInt64"},
                 {"name": "url", "type": "Nullable(String)"},
                 {"name": "error", "type": "UInt8"},
+                {"name": "error_message", "type": "Nullable(String)"},
                 {"name": "status_code", "type": "Int32"},
                 {"name": "request_id", "type": "String"},
                 {"name": "parameters", "type": "Map(String, String)"},
@@ -784,7 +782,7 @@ def get_organization_service_datasources() -> List[Dict[str, Any]]:
         },
         {
             "name": "organization.metrics_logs",
-            "description": "Metrics of your organization's dedicated clusters",
+            "description": "Metrics of your organization's dedicated ClickHouse clusters, sampled roughly every 10 seconds. One row is one metric for one host at one point in time, so filter by metric and group by host or cluster. value is a String: cast it before doing arithmetic, for example toFloat64(value). description carries ClickHouse's own explanation of the metric. The metrics available cover queries and query concurrency, CPU and load, memory, replication lag and queues, MergeTree storage size and part counts, background merges and mutations, connections, caches, and disk space. The exact set depends on the ClickHouse version and the host platform, so list it with SELECT DISTINCT metric FROM organization.metrics_logs. host is an obfuscated identifier of the form cluster_shard, not the real hostname. This data source has a TTL of 30 days.",
             "dateColumn": "timestamp",
             "engine": {
                 "engine": "MergeTree",
@@ -798,6 +796,7 @@ def get_organization_service_datasources() -> List[Dict[str, Any]]:
                 {"name": "metric", "type": "LowCardinality(String)"},
                 {"name": "value", "type": "String"},
                 {"name": "description", "type": "LowCardinality(String)"},
+                {"name": "organization_id", "type": "String"},
             ],
         },
         {

{tinybird-4.6.20.dev0 → tinybird-4.6.21.dev0}/tinybird/sql.py

@@ -129,15 +129,9 @@ class TableProjection:
     def add_projection_sql(self):
         return f"ADD {self.to_sql()}"
 
-    def drop_projection_sql(self):
-        return f"DROP PROJECTION IF EXISTS {self.name}"
-
     def materialize_projection_sql(self):
         return f"MATERIALIZE PROJECTION IF EXISTS {self.name}"
 
-    def clear_projection_sql(self):
-        return f"CLEAR PROJECTION IF EXISTS {self.name}"
-
 
 def as_subquery(sql: str) -> str:
     return f"""(\n{sql}\n)"""

{tinybird-4.6.20.dev0 → tinybird-4.6.21.dev0}/tinybird/sql_template.py

@@ -84,6 +84,7 @@ class SQLTemplateException(ValueError):
 JOB_TIMESTAMP_PARAM = "job_timestamp"
 DEFAULT_PARAM_NAMES = ["format", "q"]
 RESERVED_PARAM_NAMES = [
+    "__tb__csv_header",
     "__tb__semver",
     "debug_source_tables",
     "debug",

{tinybird-4.6.20.dev0 → tinybird-4.6.21.dev0}/tinybird/sql_toolset.py

@@ -699,3 +699,62 @@ def is_invalid_resource_from_current_workspace(
     valid_tables: Optional[Set[Tuple[str, str]]],
 ) -> bool:
     return bool(database == default_database and valid_tables and r not in valid_tables and r not in _replaced_with)
+
+
+def validate_ttl_expression(
+    columns: list[dict[str, str]],
+    ttl: Optional[str],
+    sorting_key: str,
+    primary_key: Optional[str] = None,
+    partition_key: Optional[str] = None,
+) -> Optional[str]:
+    if not ttl:
+        return None
+
+    kwargs = {"partition_key": partition_key} if partition_key else {}
+    error = chquery.validate_ttl(
+        ttl=ttl,
+        columns=columns,
+        sorting_key=sorting_key,
+        primary_key=primary_key or sorting_key,
+        **kwargs,
+    )
+    if error is None:
+        return None
+    return error.removeprefix("DB::Exception: ")
+
+
+TTL_ONLY_DROP_PARTS_SETTING = "ttl_only_drop_parts"
+
+
+def should_default_ttl_only_drop_parts(
+    columns: list[dict[str, str]],
+    ttl: Optional[str],
+    sorting_key: str,
+    partition_key: Optional[str],
+    primary_key: Optional[str] = None,
+) -> bool:
+    """Whether ``ttl_only_drop_parts = 1`` should be defaulted for this table."""
+    if not ttl or not partition_key:
+        return False
+    result = chquery.check_ttl_partition_compatibility(
+        columns=columns,
+        ttl=ttl,
+        sorting_key=sorting_key,
+        primary_key=primary_key or sorting_key,
+        partition_key=partition_key,
+    )
+    return result["checked"] and result["error"] is None
+
+
+def validate_partition_key_expression(
+    columns: list[dict[str, str]],
+    partition_key: Optional[str],
+) -> Optional[str]:
+    if not partition_key:
+        return None
+
+    error = chquery.validate_partition_key(columns=columns, partition_key=partition_key)
+    if error is None:
+        return None
+    return error.removeprefix("DB::Exception: ")

{tinybird-4.6.20.dev0 → tinybird-4.6.21.dev0}/tinybird/tb/__cli__.py

@@ -4,5 +4,5 @@ __description__ = 'Tinybird Command Line Tool'
 __url__ = 'https://www.tinybird.co/docs/forward/commands'
 __author__ = 'Tinybird'
 __author_email__ = 'support@tinybird.co'
-__version__ = '4.6.20.dev0'
-__revision__ = 'fee8cae'
+__version__ = '4.6.21.dev0'
+__revision__ = '965c47ce'

{tinybird-4.6.20.dev0 → tinybird-4.6.21.dev0}/tinybird/tb/client.py

@@ -432,6 +432,16 @@ class TinyB:
         format: str = "csv",
         replace_options: Optional[Set[str]] = None,
     ) -> Dict[str, Any]:
+        if mode in ("append", "replace"):
+            return self.datasource_append_url(
+                table_name,
+                url,
+                mode=mode,
+                sql_condition=sql_condition,
+                format=format,
+                replace_options=replace_options,
+            )
+
         params = {"name": table_name, "url": url, "mode": mode, "debug": "blocks_block_log", "format": format}
 
         if sql_condition:
@@ -448,6 +458,33 @@ class TinyB:
 
         return self.wait_for_job(res["id"], status_callback, backoff_multiplier=1.5, maximum_backoff_seconds=20)
 
+    def datasource_append_url(
+        self,
+        datasource_name: str,
+        url: str,
+        mode: str = "append",
+        sql_condition: Optional[str] = None,
+        format: str = "csv",
+        replace_options: Optional[Set[str]] = None,
+    ) -> Dict[str, Any]:
+        params = {"format": format, "url": url}
+        if sql_condition:
+            params["replace_condition"] = sql_condition
+        if replace_options:
+            for option in replace_options:
+                params[option] = "true"
+
+        res = self._req(
+            f"/v1/datasources/{quote(datasource_name, safe='')}/{mode}?{urlencode(params, safe='')}",
+            method="POST",
+            data=b"",
+        )
+        if not isinstance(res, dict):
+            raise RuntimeError("We couldn't confirm that your import started. Please try again.")
+        if "error" in res:
+            raise Exception(res["error"])
+        return res
+
     def datasource_delete(self, datasource_name: str, force: bool = False, dry_run: bool = False):
         params = {"force": "true" if force else "false", "dry_run": "true" if dry_run else "false"}
         return self._req(f"/v0/datasources/{datasource_name}?{urlencode(params)}", method="DELETE")