datacontract-cli 1.0.1__tar.gz → 1.0.2__tar.gz

{datacontract_cli-1.0.1/datacontract_cli.egg-info → datacontract_cli-1.0.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: datacontract-cli
-Version: 1.0.1
+Version: 1.0.2
 Summary: The datacontract CLI is an open source command-line tool for working with Data Contracts. It uses data contract YAML files to lint the data contract, connect to data sources and execute schema and quality tests, detect breaking changes, and export to different formats. The tool is written in Python. It can be used as a standalone CLI tool, in a CI/CD pipeline, or directly as a Python library.
 Author-email: Jochen Christ <jochen.christ@innoq.com>, Stefan Negele <stefan.negele@innoq.com>, Simon Harrer <simon.harrer@innoq.com>
 License-Expression: MIT
@@ -51,8 +51,10 @@ Provides-Extra: mysql
 Requires-Dist: datacontract-cli[duckdb]; extra == "mysql"
 Provides-Extra: postgres
 Requires-Dist: ibis-framework[postgres]<13.0.0,>=10.0.0; extra == "postgres"
+Requires-Dist: psycopg[binary]<4.0,>=3.1; extra == "postgres"
 Provides-Extra: redshift
 Requires-Dist: ibis-framework[postgres]<13.0.0,>=10.0.0; extra == "redshift"
+Requires-Dist: psycopg[binary]<4.0,>=3.1; extra == "redshift"
 Provides-Extra: s3
 Requires-Dist: s3fs<2027.0.0,>=2025.2.0; extra == "s3"
 Requires-Dist: duckdb-extension-httpfs<1.6.0,>=1.5.0; extra == "s3"
@@ -430,6 +432,7 @@ Usage: datacontract edit [OPTIONS] [LOCATION]
  Starts a local web server that opens the Data Contract Editor for the given file.                  
  The editor is bundled with the CLI, so no internet access is required.                             
  Saving in the editor writes directly back to the local file.                                       
+ If a URL is given, you are asked whether to download a local copy, which is then edited.           
  The server also acts as the editor's test runner: "Run test" in the editor executes                
  the data contract tests locally against the servers defined in the data contract.                  
  Credentials for the data sources must be provided as environment variables, see                    
@@ -437,7 +440,9 @@ Usage: datacontract edit [OPTIONS] [LOCATION]
                                                                                                     
 ╭─ Arguments ──────────────────────────────────────────────────────────────────────────────────────╮
 │   location      [LOCATION]  The path of the data contract yaml to edit. If the file does not     │
-│                             exist, you are asked whether to initialize a new data contract.      │
+│                             exist, you are asked whether to initialize a new data contract. If a │
+│                             URL is given, you are asked whether to download a local copy to      │
+│                             edit.                                                                │
 │                             [default: datacontract.yaml]                                         │
 ╰──────────────────────────────────────────────────────────────────────────────────────────────────╯
 ╭─ Options ────────────────────────────────────────────────────────────────────────────────────────╮
@@ -791,6 +796,7 @@ models:
 |----------------------------------------------|---------------------------|---------------------------------------------------------|
 | `DATACONTRACT_BIGQUERY_ACCOUNT_INFO_JSON_PATH` | `~/service-access-key.json` | Service Account key JSON file. If not set, ADC/WIF is used automatically. |
 | `DATACONTRACT_BIGQUERY_IMPERSONATION_ACCOUNT` | `sa@project.iam.gserviceaccount.com` | Optional. Service account to impersonate. Works with both key file and ADC auth. |
+| `DATACONTRACT_BIGQUERY_BILLING_PROJECT` | `my-compute-project` | Optional. Google Cloud project ID to bill query jobs to. Use when the data lives in an external project and you want charges routed to your own project. Requires `bigquery.jobUser` on the billing project and `bigquery.dataViewer` on the data project. |
 
 </details>
 

{datacontract_cli-1.0.1 → datacontract_cli-1.0.2}/README.md

@@ -318,6 +318,7 @@ Usage: datacontract edit [OPTIONS] [LOCATION]
  Starts a local web server that opens the Data Contract Editor for the given file.                  
  The editor is bundled with the CLI, so no internet access is required.                             
  Saving in the editor writes directly back to the local file.                                       
+ If a URL is given, you are asked whether to download a local copy, which is then edited.           
  The server also acts as the editor's test runner: "Run test" in the editor executes                
  the data contract tests locally against the servers defined in the data contract.                  
  Credentials for the data sources must be provided as environment variables, see                    
@@ -325,7 +326,9 @@ Usage: datacontract edit [OPTIONS] [LOCATION]
                                                                                                     
 ╭─ Arguments ──────────────────────────────────────────────────────────────────────────────────────╮
 │   location      [LOCATION]  The path of the data contract yaml to edit. If the file does not     │
-│                             exist, you are asked whether to initialize a new data contract.      │
+│                             exist, you are asked whether to initialize a new data contract. If a │
+│                             URL is given, you are asked whether to download a local copy to      │
+│                             edit.                                                                │
 │                             [default: datacontract.yaml]                                         │
 ╰──────────────────────────────────────────────────────────────────────────────────────────────────╯
 ╭─ Options ────────────────────────────────────────────────────────────────────────────────────────╮
@@ -679,6 +682,7 @@ models:
 |----------------------------------------------|---------------------------|---------------------------------------------------------|
 | `DATACONTRACT_BIGQUERY_ACCOUNT_INFO_JSON_PATH` | `~/service-access-key.json` | Service Account key JSON file. If not set, ADC/WIF is used automatically. |
 | `DATACONTRACT_BIGQUERY_IMPERSONATION_ACCOUNT` | `sa@project.iam.gserviceaccount.com` | Optional. Service account to impersonate. Works with both key file and ADC auth. |
+| `DATACONTRACT_BIGQUERY_BILLING_PROJECT` | `my-compute-project` | Optional. Google Cloud project ID to bill query jobs to. Use when the data lives in an external project and you want charges routed to your own project. Requires `bigquery.jobUser` on the billing project and `bigquery.dataViewer` on the data project. |
 
 </details>
 

{datacontract_cli-1.0.1 → datacontract_cli-1.0.2}/datacontract/command_edit.py

@@ -3,14 +3,16 @@ import mimetypes
 import os
 import webbrowser
 from importlib import metadata
-from pathlib import Path
-from urllib.parse import quote
+from pathlib import Path, PurePosixPath
+from urllib.parse import quote, unquote, urlparse
 
 import typer
 from typing_extensions import Annotated
 
 from datacontract.cli import app, console, debug_option, enable_debug_logging
 from datacontract.init.init_template import get_init_template
+from datacontract.lint.urls import fetch_resource
+from datacontract.model.exceptions import DataContractException
 
 EDITOR_ASSETS_URL_TEMPLATE = "https://cdn.jsdelivr.net/npm/datacontract-editor@{version}/dist"
 
@@ -34,6 +36,36 @@ def resolve_editor_assets_url(editor_version: str | None, editor_assets_url: str
     return None
 
 
+def local_copy_filename(url: str) -> str:
+    """Return the filename for the local copy of a remote data contract URL."""
+    name = PurePosixPath(unquote(urlparse(url).path)).name
+    if name.endswith((".yaml", ".yml")):
+        return name
+    return "datacontract.yaml"
+
+
+def _download_local_copy(url: str) -> str:
+    """Ask the user to download a local copy of the remote data contract and return its path."""
+    local_filename = local_copy_filename(url)
+    console.print("The editor edits local files only, but the data contract is available via a URL.")
+    if not typer.confirm(f"Download a local copy to '{local_filename}' and edit that?"):
+        console.print("Aborted.")
+        raise typer.Exit(code=1)
+    local_path = Path(local_filename)
+    if local_path.exists() and not typer.confirm(f"File '{local_filename}' already exists. Overwrite it?"):
+        console.print("Aborted.")
+        raise typer.Exit(code=1)
+    try:
+        content = fetch_resource(url)
+    except DataContractException as e:
+        console.print(f"[red]Error: {e.reason}[/red]")
+        raise typer.Exit(code=1)
+    local_path.write_text(content, encoding="utf-8")
+    console.print(f"📄 Downloaded {url} to {local_filename}")
+    console.print("Note: Saving in the editor writes to the local copy only, not back to the URL.")
+    return local_filename
+
+
 def _cli_version() -> str:
     try:
         return metadata.version("datacontract-cli")
@@ -238,7 +270,8 @@ def edit(
         str,
         typer.Argument(
             help="The path of the data contract yaml to edit. "
-            "If the file does not exist, you are asked whether to initialize a new data contract."
+            "If the file does not exist, you are asked whether to initialize a new data contract. "
+            "If a URL is given, you are asked whether to download a local copy to edit."
         ),
     ] = "datacontract.yaml",
     port: Annotated[int, typer.Option(help="Bind socket to this port.")] = 4243,
@@ -270,6 +303,7 @@ def edit(
     Starts a local web server that opens the Data Contract Editor for the given file.
     The editor is bundled with the CLI, so no internet access is required.
     Saving in the editor writes directly back to the local file.
+    If a URL is given, you are asked whether to download a local copy, which is then edited.
     The server also acts as the editor's test runner: "Run test" in the editor executes
     the data contract tests locally against the servers defined in the data contract.
     Credentials for the data sources must be provided as environment variables, see
@@ -283,6 +317,9 @@ def edit(
         console.print("[red]Install the extra datacontract-cli\\[api] to use edit.[/red]")
         raise typer.Exit(code=1)
 
+    if location.startswith("http://") or location.startswith("https://"):
+        location = _download_local_copy(location)
+
     file_path = Path(location).resolve()
     if file_path.suffix not in (".yaml", ".yml"):
         console.print("[red]Error: File must be a YAML file (.yaml or .yml).[/red]")

{datacontract_cli-1.0.1 → datacontract_cli-1.0.2}/datacontract/engines/ibis/connections/connect.py

@@ -142,12 +142,28 @@ def connect_ibis(
         )
 
     if server_type == "bigquery":
-        kwargs = dict(project_id=server.project, dataset_id=server.dataset)
         credentials_path = os.getenv("DATACONTRACT_BIGQUERY_ACCOUNT_INFO_JSON_PATH")
+        credentials = None
         if credentials_path:
             from google.oauth2 import service_account
 
-            kwargs["credentials"] = service_account.Credentials.from_service_account_file(credentials_path)
+            credentials = service_account.Credentials.from_service_account_file(credentials_path)
+
+        billing_project = os.getenv("DATACONTRACT_BIGQUERY_BILLING_PROJECT")
+
+        if billing_project and billing_project != server.project:
+            from google.cloud import bigquery as bq_client_lib
+
+            client = bq_client_lib.Client(project=billing_project, credentials=credentials)
+            return ibis.bigquery.connect(
+                project_id=server.project,
+                dataset_id=server.dataset,
+                client=client,
+            )
+
+        kwargs = dict(project_id=server.project, dataset_id=server.dataset)
+        if credentials:
+            kwargs["credentials"] = credentials
         return ibis.bigquery.connect(**kwargs)
 
     if server_type == "sqlserver":

{datacontract_cli-1.0.1 → datacontract_cli-1.0.2}/datacontract/engines/ibis/ibis_check_execute.py

@@ -848,6 +848,8 @@ def _resolve_col(columns: dict, field: str) -> str:
 
 def _resolve_table(con, model: str):
     """Resolve a table by name, tolerating case differences across dialects."""
+    if getattr(con, "name", None) == "pyspark":
+        return _pyspark_table_unconvertible_as_unknown(con, model)
     try:
         return con.table(model)
     except Exception:
@@ -861,6 +863,32 @@ def _resolve_table(con, model: str):
         return con.table(match)
 
 
+def _pyspark_table_unconvertible_as_unknown(con, name: str):
+    """Reflect a pyspark table, typing any column whose Spark type Ibis cannot
+    convert (e.g. Databricks `VariantType`) as `Unknown`.
+    """
+    import ibis
+    import ibis.expr.datatypes as dt
+    import ibis.expr.operations as ops
+    from ibis.backends.pyspark.datatypes import PySparkType
+
+    fields, unknown_columns = {}, []
+    for field in con._session.table(name).schema.fields:
+        try:
+            fields[field.name] = PySparkType.to_ibis(field.dataType)
+        except Exception:
+            fields[field.name] = dt.unknown
+            unknown_columns.append(f"{field.name} ({field.dataType.simpleString()})")
+    if not unknown_columns:
+        # Nothing unconvertible, resolve the table normally
+        return con.table(name)
+    logger.warning(
+        f"Model '{name}': column(s) {', '.join(unknown_columns)} have a type ibis cannot represent. "
+        f"Type checks for these columns will fail."
+    )
+    return ops.DatabaseTable(name, schema=ibis.schema(fields), source=con).to_expr()
+
+
 def _py(value):
     if value is None:
         return None

{datacontract_cli-1.0.1 → datacontract_cli-1.0.2/datacontract_cli.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: datacontract-cli
-Version: 1.0.1
+Version: 1.0.2
 Summary: The datacontract CLI is an open source command-line tool for working with Data Contracts. It uses data contract YAML files to lint the data contract, connect to data sources and execute schema and quality tests, detect breaking changes, and export to different formats. The tool is written in Python. It can be used as a standalone CLI tool, in a CI/CD pipeline, or directly as a Python library.
 Author-email: Jochen Christ <jochen.christ@innoq.com>, Stefan Negele <stefan.negele@innoq.com>, Simon Harrer <simon.harrer@innoq.com>
 License-Expression: MIT
@@ -51,8 +51,10 @@ Provides-Extra: mysql
 Requires-Dist: datacontract-cli[duckdb]; extra == "mysql"
 Provides-Extra: postgres
 Requires-Dist: ibis-framework[postgres]<13.0.0,>=10.0.0; extra == "postgres"
+Requires-Dist: psycopg[binary]<4.0,>=3.1; extra == "postgres"
 Provides-Extra: redshift
 Requires-Dist: ibis-framework[postgres]<13.0.0,>=10.0.0; extra == "redshift"
+Requires-Dist: psycopg[binary]<4.0,>=3.1; extra == "redshift"
 Provides-Extra: s3
 Requires-Dist: s3fs<2027.0.0,>=2025.2.0; extra == "s3"
 Requires-Dist: duckdb-extension-httpfs<1.6.0,>=1.5.0; extra == "s3"
@@ -430,6 +432,7 @@ Usage: datacontract edit [OPTIONS] [LOCATION]
  Starts a local web server that opens the Data Contract Editor for the given file.                  
  The editor is bundled with the CLI, so no internet access is required.                             
  Saving in the editor writes directly back to the local file.                                       
+ If a URL is given, you are asked whether to download a local copy, which is then edited.           
  The server also acts as the editor's test runner: "Run test" in the editor executes                
  the data contract tests locally against the servers defined in the data contract.                  
  Credentials for the data sources must be provided as environment variables, see                    
@@ -437,7 +440,9 @@ Usage: datacontract edit [OPTIONS] [LOCATION]
                                                                                                     
 ╭─ Arguments ──────────────────────────────────────────────────────────────────────────────────────╮
 │   location      [LOCATION]  The path of the data contract yaml to edit. If the file does not     │
-│                             exist, you are asked whether to initialize a new data contract.      │
+│                             exist, you are asked whether to initialize a new data contract. If a │
+│                             URL is given, you are asked whether to download a local copy to      │
+│                             edit.                                                                │
 │                             [default: datacontract.yaml]                                         │
 ╰──────────────────────────────────────────────────────────────────────────────────────────────────╯
 ╭─ Options ────────────────────────────────────────────────────────────────────────────────────────╮
@@ -791,6 +796,7 @@ models:
 |----------------------------------------------|---------------------------|---------------------------------------------------------|
 | `DATACONTRACT_BIGQUERY_ACCOUNT_INFO_JSON_PATH` | `~/service-access-key.json` | Service Account key JSON file. If not set, ADC/WIF is used automatically. |
 | `DATACONTRACT_BIGQUERY_IMPERSONATION_ACCOUNT` | `sa@project.iam.gserviceaccount.com` | Optional. Service account to impersonate. Works with both key file and ADC auth. |
+| `DATACONTRACT_BIGQUERY_BILLING_PROJECT` | `my-compute-project` | Optional. Google Cloud project ID to bill query jobs to. Use when the data lives in an external project and you want charges routed to your own project. Requires `bigquery.jobUser` on the billing project and `bigquery.dataViewer` on the data project. |
 
 </details>
 

{datacontract_cli-1.0.1 → datacontract_cli-1.0.2}/datacontract_cli.egg-info/requires.txt

@@ -98,6 +98,7 @@ pyarrow>=18.1.0
 
 [postgres]
 ibis-framework[postgres]<13.0.0,>=10.0.0
+psycopg[binary]<4.0,>=3.1
 
 [protobuf]
 proto-schema-parser<3.0,>=1.0
@@ -107,6 +108,7 @@ rdflib==7.6.0
 
 [redshift]
 ibis-framework[postgres]<13.0.0,>=10.0.0
+psycopg[binary]<4.0,>=3.1
 
 [s3]
 s3fs<2027.0.0,>=2025.2.0

{datacontract_cli-1.0.1 → datacontract_cli-1.0.2}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "datacontract-cli"
-version = "1.0.1"
+version = "1.0.2"
 description = "The datacontract CLI is an open source command-line tool for working with Data Contracts. It uses data contract YAML files to lint the data contract, connect to data sources and execute schema and quality tests, detect breaking changes, and export to different formats. The tool is written in Python. It can be used as a standalone CLI tool, in a CI/CD pipeline, or directly as a Python library."
 license = "MIT"
 readme = "README.md"
@@ -86,12 +86,18 @@ mysql = [
 ]
 
 postgres = [
-  "ibis-framework[postgres]>=10.0.0,<13.0.0"
+  "ibis-framework[postgres]>=10.0.0,<13.0.0",
+  # psycopg's binary wheels bundle libpq, so the extra works without
+  # PostgreSQL client libraries installed on the system.
+  "psycopg[binary]>=3.1,<4.0",
 ]
 
 redshift = [
   # Redshift speaks the postgres wire protocol; ibis has no dedicated backend.
-  "ibis-framework[postgres]>=10.0.0,<13.0.0"
+  "ibis-framework[postgres]>=10.0.0,<13.0.0",
+  # psycopg's binary wheels bundle libpq, so the extra works without
+  # PostgreSQL client libraries installed on the system.
+  "psycopg[binary]>=3.1,<4.0",
 ]
 
 # DuckDB extension wheels are bundled for air-gapped installs. The 1.5.x line

{datacontract_cli-1.0.1 → datacontract_cli-1.0.2}/tests/test_edit.py

@@ -6,7 +6,12 @@ from fastapi.testclient import TestClient
 from typer.testing import CliRunner
 
 from datacontract.cli import app
-from datacontract.command_edit import BUNDLED_EDITOR_ASSETS_DIR, create_app, resolve_editor_assets_url
+from datacontract.command_edit import (
+    BUNDLED_EDITOR_ASSETS_DIR,
+    create_app,
+    local_copy_filename,
+    resolve_editor_assets_url,
+)
 
 YAML_CONTENT = """apiVersion: v3.1.0
 kind: DataContract
@@ -49,6 +54,70 @@ def test_edit_aborts_when_initialization_is_declined(tmp_path):
     assert not file_path.exists()
 
 
+def test_edit_downloads_a_local_copy_of_a_url(tmp_path, monkeypatch):
+    monkeypatch.chdir(tmp_path)
+    runner = CliRunner()
+    url = "https://example.com/contracts/orders.yaml"
+    with (
+        patch("datacontract.command_edit.fetch_resource", return_value=YAML_CONTENT) as mock_fetch,
+        patch("uvicorn.run") as mock_run,
+    ):
+        result = runner.invoke(app, ["edit", url, "--no-open"], input="y\n")
+    assert result.exit_code == 0
+    mock_fetch.assert_called_once_with(url)
+    assert (tmp_path / "orders.yaml").read_text(encoding="utf-8") == YAML_CONTENT
+    # saving in the editor goes to the local copy, not back to the URL
+    assert "local copy" in result.output
+    mock_run.assert_called_once()
+
+
+def test_edit_downloads_a_url_with_the_entropy_data_api_key(tmp_path, monkeypatch):
+    monkeypatch.chdir(tmp_path)
+    monkeypatch.setenv("ENTROPY_DATA_API_KEY", "secret-key")
+    monkeypatch.delenv("DATAMESH_MANAGER_API_KEY", raising=False)
+    monkeypatch.delenv("DATACONTRACT_MANAGER_API_KEY", raising=False)
+    runner = CliRunner()
+    url = "https://app.entropy-data.com/contracts/orders.yaml"
+    response = MagicMock(status_code=200, text=YAML_CONTENT)
+    with patch("datacontract.lint.urls.requests.get", return_value=response) as mock_get, patch("uvicorn.run"):
+        result = runner.invoke(app, ["edit", url, "--no-open"], input="y\n")
+    assert result.exit_code == 0
+    mock_get.assert_called_once()
+    assert mock_get.call_args.kwargs["headers"]["x-api-key"] == "secret-key"
+    assert (tmp_path / "orders.yaml").read_text(encoding="utf-8") == YAML_CONTENT
+
+
+def test_edit_aborts_when_local_copy_of_a_url_is_declined(tmp_path, monkeypatch):
+    monkeypatch.chdir(tmp_path)
+    runner = CliRunner()
+    with patch("datacontract.command_edit.fetch_resource") as mock_fetch:
+        result = runner.invoke(app, ["edit", "https://example.com/orders.yaml", "--no-open"], input="n\n")
+    assert result.exit_code == 1
+    mock_fetch.assert_not_called()
+    assert not (tmp_path / "orders.yaml").exists()
+
+
+def test_edit_asks_before_overwriting_an_existing_local_copy(tmp_path, monkeypatch):
+    monkeypatch.chdir(tmp_path)
+    existing = tmp_path / "orders.yaml"
+    existing.write_text("local changes", encoding="utf-8")
+    runner = CliRunner()
+    with patch("datacontract.command_edit.fetch_resource") as mock_fetch:
+        result = runner.invoke(app, ["edit", "https://example.com/orders.yaml", "--no-open"], input="y\nn\n")
+    assert result.exit_code == 1
+    mock_fetch.assert_not_called()
+    assert existing.read_text(encoding="utf-8") == "local changes"
+
+
+def test_local_copy_filename():
+    assert local_copy_filename("https://example.com/contracts/orders.yaml") == "orders.yaml"
+    assert local_copy_filename("https://example.com/contracts/orders.yml?version=2") == "orders.yml"
+    assert local_copy_filename("https://example.com/orders%20v2.yaml") == "orders v2.yaml"
+    # URLs without a YAML filename fall back to the default
+    assert local_copy_filename("https://example.com/api/contracts/12345") == "datacontract.yaml"
+    assert local_copy_filename("https://example.com/") == "datacontract.yaml"
+
+
 def test_index_serves_editor_page(client, contract_file):
     response = client.get("/")
     assert response.status_code == 200

{datacontract_cli-1.0.1 → datacontract_cli-1.0.2}/tests/test_test_dataframe.py

@@ -172,3 +172,29 @@ def _prepare_fail_dataframe(spark):
     # Create temporary view
     # Name must match the model name in the data contract
     df.createOrReplaceTempView("my_table")
+
+
+def test_pyspark_unconvertible_column_as_unknown(spark: SparkSession):
+    import ibis
+    import ibis.expr.datatypes as dt
+    from ibis.backends.pyspark.datatypes import PySparkType
+
+    from datacontract.engines.ibis.ibis_check_execute import _resolve_table
+
+    spark.sql("SELECT 1 AS id, INTERVAL '1-2' YEAR TO MONTH AS payload, 10 AS amount").createOrReplaceTempView(
+        "unconvertible_model"
+    )
+
+    # Sanity: the fixture column really is unconvertible for ibis
+    payload_type = spark.table("unconvertible_model").schema["payload"].dataType
+    with pytest.raises(Exception):
+        PySparkType.to_ibis(payload_type)
+
+    con = ibis.pyspark.connect(session=spark)
+    t = _resolve_table(con, "unconvertible_model")
+
+    assert set(t.columns) == {"id", "payload", "amount"}  # kept
+    assert isinstance(t.schema()["payload"], dt.Unknown)  # typed opaque
+    assert t.count().execute() == 1  # table is queryable
+    assert t.amount.isnull().sum().execute() == 0  # checks on other columns run
+    assert t.payload.isnull().sum().execute() == 0  # not-null still works on the opaque column