datacontract-cli 1.0.1__tar.gz → 1.0.3__tar.gz

{datacontract_cli-1.0.1/datacontract_cli.egg-info → datacontract_cli-1.0.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: datacontract-cli
-Version: 1.0.1
+Version: 1.0.3
 Summary: The datacontract CLI is an open source command-line tool for working with Data Contracts. It uses data contract YAML files to lint the data contract, connect to data sources and execute schema and quality tests, detect breaking changes, and export to different formats. The tool is written in Python. It can be used as a standalone CLI tool, in a CI/CD pipeline, or directly as a Python library.
 Author-email: Jochen Christ <jochen.christ@innoq.com>, Stefan Negele <stefan.negele@innoq.com>, Simon Harrer <simon.harrer@innoq.com>
 License-Expression: MIT
@@ -11,7 +11,8 @@ Classifier: Operating System :: OS Independent
 Requires-Python: <3.15,>=3.10
 Description-Content-Type: text/markdown
 License-File: LICENSE
-Requires-Dist: typer<0.26,>=0.18.0
+Requires-Dist: typer<0.27,>=0.18.0
+Requires-Dist: click<9.0.0,>=8.1.0
 Requires-Dist: pydantic<2.14.0,>=2.8.2
 Requires-Dist: pyyaml~=6.0.1
 Requires-Dist: requests<2.35,>=2.31
@@ -38,8 +39,8 @@ Provides-Extra: excel
 Requires-Dist: openpyxl<4.0.0,>=3.1.5; extra == "excel"
 Provides-Extra: databricks
 Requires-Dist: ibis-framework[databricks,pyspark]<13.0.0,>=10.0.0; extra == "databricks"
-Requires-Dist: databricks-sql-connector<4.3.0,>=3.7.0; extra == "databricks"
-Requires-Dist: databricks-sdk<0.116.0; extra == "databricks"
+Requires-Dist: databricks-sql-connector<4.4.0,>=3.7.0; extra == "databricks"
+Requires-Dist: databricks-sdk<0.118.0; extra == "databricks"
 Requires-Dist: pyspark<5.0.0,>=3.5.0; extra == "databricks"
 Provides-Extra: iceberg
 Requires-Dist: pyiceberg==0.11.1; extra == "iceberg"
@@ -51,8 +52,10 @@ Provides-Extra: mysql
 Requires-Dist: datacontract-cli[duckdb]; extra == "mysql"
 Provides-Extra: postgres
 Requires-Dist: ibis-framework[postgres]<13.0.0,>=10.0.0; extra == "postgres"
+Requires-Dist: psycopg[binary]<4.0,>=3.1; extra == "postgres"
 Provides-Extra: redshift
 Requires-Dist: ibis-framework[postgres]<13.0.0,>=10.0.0; extra == "redshift"
+Requires-Dist: psycopg[binary]<4.0,>=3.1; extra == "redshift"
 Provides-Extra: s3
 Requires-Dist: s3fs<2027.0.0,>=2025.2.0; extra == "s3"
 Requires-Dist: duckdb-extension-httpfs<1.6.0,>=1.5.0; extra == "s3"
@@ -86,7 +89,7 @@ Requires-Dist: pyarrow>=18.1.0; extra == "parquet"
 Provides-Extra: rdf
 Requires-Dist: rdflib==7.6.0; extra == "rdf"
 Provides-Extra: api
-Requires-Dist: fastapi<0.137.0,>=0.115.0; extra == "api"
+Requires-Dist: fastapi<0.138.0,>=0.115.0; extra == "api"
 Requires-Dist: uvicorn<0.50.0,>=0.44.0; extra == "api"
 Provides-Extra: protobuf
 Requires-Dist: proto-schema-parser<3.0,>=1.0; extra == "protobuf"
@@ -105,7 +108,7 @@ Requires-Dist: pytest-xdist; extra == "dev"
 Requires-Dist: pymssql==2.3.13; extra == "dev"
 Requires-Dist: psycopg2-binary<3.0,>=2.9; extra == "dev"
 Requires-Dist: mysql-connector-python<9.7.0,>=8.0.30; extra == "dev"
-Requires-Dist: ruff==0.15.16; extra == "dev"
+Requires-Dist: ruff==0.15.17; extra == "dev"
 Requires-Dist: testcontainers[kafka,minio,mssql,mysql,postgres]==4.14.2; extra == "dev"
 Requires-Dist: trino==0.337.0; extra == "dev"
 Dynamic: license-file
@@ -430,6 +433,7 @@ Usage: datacontract edit [OPTIONS] [LOCATION]
  Starts a local web server that opens the Data Contract Editor for the given file.                  
  The editor is bundled with the CLI, so no internet access is required.                             
  Saving in the editor writes directly back to the local file.                                       
+ If a URL is given, you are asked whether to download a local copy, which is then edited.           
  The server also acts as the editor's test runner: "Run test" in the editor executes                
  the data contract tests locally against the servers defined in the data contract.                  
  Credentials for the data sources must be provided as environment variables, see                    
@@ -437,7 +441,9 @@ Usage: datacontract edit [OPTIONS] [LOCATION]
                                                                                                     
 ╭─ Arguments ──────────────────────────────────────────────────────────────────────────────────────╮
 │   location      [LOCATION]  The path of the data contract yaml to edit. If the file does not     │
-│                             exist, you are asked whether to initialize a new data contract.      │
+│                             exist, you are asked whether to initialize a new data contract. If a │
+│                             URL is given, you are asked whether to download a local copy to      │
+│                             edit.                                                                │
 │                             [default: datacontract.yaml]                                         │
 ╰──────────────────────────────────────────────────────────────────────────────────────────────────╯
 ╭─ Options ────────────────────────────────────────────────────────────────────────────────────────╮
@@ -791,6 +797,7 @@ models:
 |----------------------------------------------|---------------------------|---------------------------------------------------------|
 | `DATACONTRACT_BIGQUERY_ACCOUNT_INFO_JSON_PATH` | `~/service-access-key.json` | Service Account key JSON file. If not set, ADC/WIF is used automatically. |
 | `DATACONTRACT_BIGQUERY_IMPERSONATION_ACCOUNT` | `sa@project.iam.gserviceaccount.com` | Optional. Service account to impersonate. Works with both key file and ADC auth. |
+| `DATACONTRACT_BIGQUERY_BILLING_PROJECT` | `my-compute-project` | Optional. Google Cloud project ID to bill query jobs to. Use when the data lives in an external project and you want charges routed to your own project. Requires `bigquery.jobUser` on the billing project and `bigquery.dataViewer` on the data project. |
 
 </details>
 
@@ -2011,6 +2018,7 @@ Usage: datacontract import [OPTIONS] COMMAND [ARGS]...
 │ spark       Import a data contract from a Spark schema.                                          │
 │ iceberg     Import a data contract from an Iceberg schema.                                       │
 │ excel       Import a data contract from an Excel file.                                           │
+│ powerbi     Import a data contract from a Power BI semantic model (.pbit, .bim, or .json) file.  │
 │ snowflake   Import a data contract from a Snowflake workspace.                                   │
 ╰──────────────────────────────────────────────────────────────────────────────────────────────────╯
                                                                                                     
@@ -2256,6 +2264,24 @@ datacontract import snowflake --source account.canada-central.azure --database d
 ```
 </details>
 
+<details markdown="1">
+<summary><strong>Power BI</strong></summary>
+
+Imports a data contract from a Power BI semantic model. Pass the file via the `source` parameter — a `.pbit` template, a `.bim`, or a `.json` (BIM/TMSL) file. Only the semantic model (tables, columns, measures, and hierarchies) is imported; report pages and visuals are not.
+
+See the [Power BI documentation](https://learn.microsoft.com/en-us/power-bi/create-reports/desktop-templates) for how to export a `.pbit` file from a `.pbix` file.
+
+Example:
+
+```bash
+datacontract import powerbi --source "test.pbit"
+
+datacontract import powerbi --source "test.bim"
+```
+
+</details>
+
+
 ### catalog
 ```
 Usage: datacontract catalog [OPTIONS]                                                              

{datacontract_cli-1.0.1 → datacontract_cli-1.0.3}/README.md

@@ -318,6 +318,7 @@ Usage: datacontract edit [OPTIONS] [LOCATION]
  Starts a local web server that opens the Data Contract Editor for the given file.                  
  The editor is bundled with the CLI, so no internet access is required.                             
  Saving in the editor writes directly back to the local file.                                       
+ If a URL is given, you are asked whether to download a local copy, which is then edited.           
  The server also acts as the editor's test runner: "Run test" in the editor executes                
  the data contract tests locally against the servers defined in the data contract.                  
  Credentials for the data sources must be provided as environment variables, see                    
@@ -325,7 +326,9 @@ Usage: datacontract edit [OPTIONS] [LOCATION]
                                                                                                     
 ╭─ Arguments ──────────────────────────────────────────────────────────────────────────────────────╮
 │   location      [LOCATION]  The path of the data contract yaml to edit. If the file does not     │
-│                             exist, you are asked whether to initialize a new data contract.      │
+│                             exist, you are asked whether to initialize a new data contract. If a │
+│                             URL is given, you are asked whether to download a local copy to      │
+│                             edit.                                                                │
 │                             [default: datacontract.yaml]                                         │
 ╰──────────────────────────────────────────────────────────────────────────────────────────────────╯
 ╭─ Options ────────────────────────────────────────────────────────────────────────────────────────╮
@@ -679,6 +682,7 @@ models:
 |----------------------------------------------|---------------------------|---------------------------------------------------------|
 | `DATACONTRACT_BIGQUERY_ACCOUNT_INFO_JSON_PATH` | `~/service-access-key.json` | Service Account key JSON file. If not set, ADC/WIF is used automatically. |
 | `DATACONTRACT_BIGQUERY_IMPERSONATION_ACCOUNT` | `sa@project.iam.gserviceaccount.com` | Optional. Service account to impersonate. Works with both key file and ADC auth. |
+| `DATACONTRACT_BIGQUERY_BILLING_PROJECT` | `my-compute-project` | Optional. Google Cloud project ID to bill query jobs to. Use when the data lives in an external project and you want charges routed to your own project. Requires `bigquery.jobUser` on the billing project and `bigquery.dataViewer` on the data project. |
 
 </details>
 
@@ -1899,6 +1903,7 @@ Usage: datacontract import [OPTIONS] COMMAND [ARGS]...
 │ spark       Import a data contract from a Spark schema.                                          │
 │ iceberg     Import a data contract from an Iceberg schema.                                       │
 │ excel       Import a data contract from an Excel file.                                           │
+│ powerbi     Import a data contract from a Power BI semantic model (.pbit, .bim, or .json) file.  │
 │ snowflake   Import a data contract from a Snowflake workspace.                                   │
 ╰──────────────────────────────────────────────────────────────────────────────────────────────────╯
                                                                                                     
@@ -2144,6 +2149,24 @@ datacontract import snowflake --source account.canada-central.azure --database d
 ```
 </details>
 
+<details markdown="1">
+<summary><strong>Power BI</strong></summary>
+
+Imports a data contract from a Power BI semantic model. Pass the file via the `source` parameter — a `.pbit` template, a `.bim`, or a `.json` (BIM/TMSL) file. Only the semantic model (tables, columns, measures, and hierarchies) is imported; report pages and visuals are not.
+
+See the [Power BI documentation](https://learn.microsoft.com/en-us/power-bi/create-reports/desktop-templates) for how to export a `.pbit` file from a `.pbix` file.
+
+Example:
+
+```bash
+datacontract import powerbi --source "test.pbit"
+
+datacontract import powerbi --source "test.bim"
+```
+
+</details>
+
+
 ### catalog
 ```
 Usage: datacontract catalog [OPTIONS]                                                              

{datacontract_cli-1.0.1 → datacontract_cli-1.0.3}/datacontract/command_edit.py

@@ -3,14 +3,16 @@ import mimetypes
 import os
 import webbrowser
 from importlib import metadata
-from pathlib import Path
-from urllib.parse import quote
+from pathlib import Path, PurePosixPath
+from urllib.parse import quote, unquote, urlparse
 
 import typer
 from typing_extensions import Annotated
 
 from datacontract.cli import app, console, debug_option, enable_debug_logging
 from datacontract.init.init_template import get_init_template
+from datacontract.lint.urls import fetch_resource
+from datacontract.model.exceptions import DataContractException
 
 EDITOR_ASSETS_URL_TEMPLATE = "https://cdn.jsdelivr.net/npm/datacontract-editor@{version}/dist"
 
@@ -34,6 +36,36 @@ def resolve_editor_assets_url(editor_version: str | None, editor_assets_url: str
     return None
 
 
+def local_copy_filename(url: str) -> str:
+    """Return the filename for the local copy of a remote data contract URL."""
+    name = PurePosixPath(unquote(urlparse(url).path)).name
+    if name.endswith((".yaml", ".yml")):
+        return name
+    return "datacontract.yaml"
+
+
+def _download_local_copy(url: str) -> str:
+    """Ask the user to download a local copy of the remote data contract and return its path."""
+    local_filename = local_copy_filename(url)
+    console.print("The editor edits local files only, but the data contract is available via a URL.")
+    if not typer.confirm(f"Download a local copy to '{local_filename}' and edit that?"):
+        console.print("Aborted.")
+        raise typer.Exit(code=1)
+    local_path = Path(local_filename)
+    if local_path.exists() and not typer.confirm(f"File '{local_filename}' already exists. Overwrite it?"):
+        console.print("Aborted.")
+        raise typer.Exit(code=1)
+    try:
+        content = fetch_resource(url)
+    except DataContractException as e:
+        console.print(f"[red]Error: {e.reason}[/red]")
+        raise typer.Exit(code=1)
+    local_path.write_text(content, encoding="utf-8")
+    console.print(f"📄 Downloaded {url} to {local_filename}")
+    console.print("Note: Saving in the editor writes to the local copy only, not back to the URL.")
+    return local_filename
+
+
 def _cli_version() -> str:
     try:
         return metadata.version("datacontract-cli")
@@ -43,9 +75,12 @@ def _cli_version() -> str:
 
 def _generate_index_html(filename: str) -> str:
     """
-    Generate the editor page, mirroring the CLI mode of the datacontract-editor npm launcher:
-    load the YAML from the local file API, write it back on save, and point the
-    editor's test runner at this server's own /test endpoint.
+    Generate the editor page: load the YAML from the local file API, write it back on
+    save, and point the editor's test runner at this server's own /test endpoint.
+
+    The editor runs in EMBEDDED mode, since the file menu (New, Load Example, Open)
+    makes no sense when the editor is bound to a single local file. The filename is
+    shown in the header via titlePrefix, and Cancel reverts to the file on disk.
     """
     filename_js = json.dumps(filename)
     file_api_path = f"/api/files/{quote(filename)}"
@@ -65,7 +100,7 @@ def _generate_index_html(filename: str) -> str:
   <body>
     <div id="root"></div>
     <script type="module">
-      import {{ init }} from '{EDITOR_ASSETS_PATH}/datacontract-editor.es.js';
+      import {{ init, parseYaml }} from '{EDITOR_ASSETS_PATH}/datacontract-editor.es.js';
 
       async function initCli() {{
         try {{
@@ -77,9 +112,22 @@ def _generate_index_html(filename: str) -> str:
 
           const notify = (notification) => editor?.getStore().getState().addNotification(notification);
 
+          // The header renders the titlePrefix followed by the contract's name,
+          // so only show the filename when the contract has a name.
+          let titlePrefix = null;
+          try {{
+            if (parseYaml(yaml)?.name) {{
+              titlePrefix = {filename_js};
+            }}
+          }} catch {{
+            // not parseable: no contract name to show the filename next to
+          }}
+
           const editor = init({{
             container: '#root',
-            mode: 'CLI',
+            mode: 'EMBEDDED',
+            showDelete: false,
+            titlePrefix: titlePrefix,
             yaml: yaml,
             persistence: 'none',
             initialView: 'form',
@@ -87,6 +135,18 @@ def _generate_index_html(filename: str) -> str:
               enabled: true,
               dataContractCliApiServerUrl: window.location.origin,
             }},
+            onCancel: async () => {{
+              try {{
+                const response = await fetch('{file_api_path}');
+                if (!response.ok) {{
+                  throw new Error(response.statusText);
+                }}
+                editor.setYaml(await response.text());
+                notify({{ type: 'info', message: 'Discarded changes, reloaded ' + {filename_js} }});
+              }} catch (error) {{
+                notify({{ type: 'error', message: 'Failed to reload ' + {filename_js} + ': ' + error.message, duration: 5000 }});
+              }}
+            }},
             onSave: async (yamlContent) => {{
               try {{
                 const saveResponse = await fetch('{file_api_path}', {{
@@ -238,7 +298,8 @@ def edit(
         str,
         typer.Argument(
             help="The path of the data contract yaml to edit. "
-            "If the file does not exist, you are asked whether to initialize a new data contract."
+            "If the file does not exist, you are asked whether to initialize a new data contract. "
+            "If a URL is given, you are asked whether to download a local copy to edit."
         ),
     ] = "datacontract.yaml",
     port: Annotated[int, typer.Option(help="Bind socket to this port.")] = 4243,
@@ -270,6 +331,7 @@ def edit(
     Starts a local web server that opens the Data Contract Editor for the given file.
     The editor is bundled with the CLI, so no internet access is required.
     Saving in the editor writes directly back to the local file.
+    If a URL is given, you are asked whether to download a local copy, which is then edited.
     The server also acts as the editor's test runner: "Run test" in the editor executes
     the data contract tests locally against the servers defined in the data contract.
     Credentials for the data sources must be provided as environment variables, see
@@ -283,6 +345,9 @@ def edit(
         console.print("[red]Install the extra datacontract-cli\\[api] to use edit.[/red]")
         raise typer.Exit(code=1)
 
+    if location.startswith("http://") or location.startswith("https://"):
+        location = _download_local_copy(location)
+
     file_path = Path(location).resolve()
     if file_path.suffix not in (".yaml", ".yml"):
         console.print("[red]Error: File must be a YAML file (.yaml or .yml).[/red]")

{datacontract_cli-1.0.1 → datacontract_cli-1.0.3}/datacontract/command_import.py

@@ -431,6 +431,26 @@ def import_excel(
     _write_result(result, output)
 
 
+@import_app.command(
+    name="powerbi",
+    epilog="Example: datacontract import powerbi --source SemanticModel.pbit --output datacontract.yaml",
+)
+def import_powerbi(
+    source: Annotated[
+        Optional[str], typer.Option(help="Path to a Power BI .pbit, .bim, or .json semantic model file.")
+    ] = None,
+    output: output_option = None,
+    schema: schema_option = None,
+    owner: owner_option = None,
+    id: id_option = None,
+    debug: debug_option = None,
+):
+    """Import a data contract from a Power BI semantic model (.pbit, .bim, or .json) file."""
+    enable_debug_logging(debug)
+    result = DataContract.import_from_source(format="powerbi", source=source, schema=schema, owner=owner, id=id)
+    _write_result(result, output)
+
+
 @import_app.command(
     name="snowflake",
     epilog="Example: datacontract import snowflake --source account --database DEMO_DB --schema PUBLIC --output datacontract.yaml",

{datacontract_cli-1.0.1 → datacontract_cli-1.0.3}/datacontract/engines/ibis/connections/connect.py

@@ -142,12 +142,28 @@ def connect_ibis(
         )
 
     if server_type == "bigquery":
-        kwargs = dict(project_id=server.project, dataset_id=server.dataset)
         credentials_path = os.getenv("DATACONTRACT_BIGQUERY_ACCOUNT_INFO_JSON_PATH")
+        credentials = None
         if credentials_path:
             from google.oauth2 import service_account
 
-            kwargs["credentials"] = service_account.Credentials.from_service_account_file(credentials_path)
+            credentials = service_account.Credentials.from_service_account_file(credentials_path)
+
+        billing_project = os.getenv("DATACONTRACT_BIGQUERY_BILLING_PROJECT")
+
+        if billing_project and billing_project != server.project:
+            from google.cloud import bigquery as bq_client_lib
+
+            client = bq_client_lib.Client(project=billing_project, credentials=credentials)
+            return ibis.bigquery.connect(
+                project_id=server.project,
+                dataset_id=server.dataset,
+                client=client,
+            )
+
+        kwargs = dict(project_id=server.project, dataset_id=server.dataset)
+        if credentials:
+            kwargs["credentials"] = credentials
         return ibis.bigquery.connect(**kwargs)
 
     if server_type == "sqlserver":
@@ -412,6 +428,7 @@ def _sqlserver_connection_kwargs(server: Server) -> dict:
 def _connect_athena(ibis, server: Server):
     s3_access_key_id = os.getenv("DATACONTRACT_S3_ACCESS_KEY_ID")
     s3_secret_access_key = os.getenv("DATACONTRACT_S3_SECRET_ACCESS_KEY")
+    s3_session_token = os.getenv("DATACONTRACT_S3_SESSION_TOKEN")
     if not server.schema_:
         raise DataContractException(
             type="athena-connection",
@@ -430,6 +447,7 @@ def _connect_athena(ibis, server: Server):
         s3_staging_dir=server.stagingDir,
         aws_access_key_id=s3_access_key_id,
         aws_secret_access_key=s3_secret_access_key,
+        aws_session_token=s3_session_token,
         region_name=os.getenv("DATACONTRACT_S3_REGION") or getattr(server, "region_name", None),
         schema_name=server.schema_,
     )

{datacontract_cli-1.0.1 → datacontract_cli-1.0.3}/datacontract/engines/ibis/ibis_check_execute.py

@@ -695,10 +695,13 @@ def _run_scalar(con, query: str, dialect: Optional[str]):
         try:
             row = cursor.fetchone()
         finally:
-            try:
-                cursor.close()
-            except Exception:
-                pass
+            # On DuckDB, raw_sql returns the shared connection itself; closing it
+            # would tear down the connection and break every subsequent check.
+            if cursor is not getattr(con, "con", None):
+                try:
+                    cursor.close()
+                except Exception:
+                    pass
         return _py(row[0]) if row else None
 
 
@@ -848,6 +851,8 @@ def _resolve_col(columns: dict, field: str) -> str:
 
 def _resolve_table(con, model: str):
     """Resolve a table by name, tolerating case differences across dialects."""
+    if getattr(con, "name", None) == "pyspark":
+        return _pyspark_table_unconvertible_as_unknown(con, model)
     try:
         return con.table(model)
     except Exception:
@@ -861,6 +866,32 @@ def _resolve_table(con, model: str):
         return con.table(match)
 
 
+def _pyspark_table_unconvertible_as_unknown(con, name: str):
+    """Reflect a pyspark table, typing any column whose Spark type Ibis cannot
+    convert (e.g. Databricks `VariantType`) as `Unknown`.
+    """
+    import ibis
+    import ibis.expr.datatypes as dt
+    import ibis.expr.operations as ops
+    from ibis.backends.pyspark.datatypes import PySparkType
+
+    fields, unknown_columns = {}, []
+    for field in con._session.table(name).schema.fields:
+        try:
+            fields[field.name] = PySparkType.to_ibis(field.dataType)
+        except Exception:
+            fields[field.name] = dt.unknown
+            unknown_columns.append(f"{field.name} ({field.dataType.simpleString()})")
+    if not unknown_columns:
+        # Nothing unconvertible, resolve the table normally
+        return con.table(name)
+    logger.warning(
+        f"Model '{name}': column(s) {', '.join(unknown_columns)} have a type ibis cannot represent. "
+        f"Type checks for these columns will fail."
+    )
+    return ops.DatabaseTable(name, schema=ibis.schema(fields), source=con).to_expr()
+
+
 def _py(value):
     if value is None:
         return None

{datacontract_cli-1.0.1 → datacontract_cli-1.0.3}/datacontract/imports/importer.py

@@ -38,6 +38,7 @@ class ImportFormat(str, Enum):
     csv = "csv"
     protobuf = "protobuf"
     excel = "excel"
+    powerbi = "powerbi"
     snowflake = "snowflake"
 
     @classmethod

{datacontract_cli-1.0.1 → datacontract_cli-1.0.3}/datacontract/imports/importer_factory.py

@@ -119,12 +119,16 @@ importer_factory.register_lazy_importer(
     module_path="datacontract.imports.excel_importer",
     class_name="ExcelImporter",
 )
+importer_factory.register_lazy_importer(
+    name=ImportFormat.powerbi,
+    module_path="datacontract.imports.powerbi_importer",
+    class_name="PowerBiImporter",
+)
 importer_factory.register_lazy_importer(
     name=ImportFormat.snowflake,
     module_path="datacontract.imports.snowflake_importer",
     class_name="SnowflakeImporter",
 )
-
 importer_factory.register_lazy_importer(
     name=ImportFormat.json,
     module_path="datacontract.imports.json_importer",