laketower 0.5.1__py3-none-any.whl → 0.6.0__py3-none-any.whl

laketower/tables.py

@@ -1,5 +1,6 @@
+import enum
 from datetime import datetime, timezone
-from typing import Any, Protocol
+from typing import Any, BinaryIO, Protocol, TextIO
 
 import deltalake
 import duckdb
@@ -17,6 +18,15 @@ from laketower.config import ConfigTable, TableFormats
 DEFAULT_LIMIT = 10
 
 
+class ImportModeEnum(str, enum.Enum):
+    append = "append"
+    overwrite = "overwrite"
+
+
+class ImportFileFormatEnum(str, enum.Enum):
+    csv = "csv"
+
+
 class TableMetadata(pydantic.BaseModel):
     table_format: TableFormats
     name: str | None = None
@@ -43,17 +53,112 @@ class TableHistory(pydantic.BaseModel):
 
 
 class TableProtocol(Protocol):  # pragma: no cover
+    @classmethod
+    def is_valid(cls, table_config: ConfigTable) -> bool: ...
+    def __init__(self, table_config: ConfigTable) -> None: ...
     def metadata(self) -> TableMetadata: ...
     def schema(self) -> pa.Schema: ...
     def history(self) -> TableHistory: ...
     def dataset(self, version: int | str | None = None) -> padataset.Dataset: ...
+    def import_data(
+        self, data: pd.DataFrame, mode: ImportModeEnum = ImportModeEnum.append
+    ) -> None: ...
 
 
 class DeltaTable:
     def __init__(self, table_config: ConfigTable):
         super().__init__()
         self.table_config = table_config
-        self._impl = deltalake.DeltaTable(table_config.uri)
+        storage_options = self._generate_storage_options(table_config)
+        self._impl = deltalake.DeltaTable(
+            table_config.uri, storage_options=storage_options
+        )
+
+    @classmethod
+    def _generate_storage_options(
+        cls, table_config: ConfigTable
+    ) -> dict[str, str] | None:
+        # documentation from `object-store` Rust crate:
+        # - s3: https://docs.rs/object_store/latest/object_store/aws/enum.AmazonS3ConfigKey.html
+        # - adls: https://docs.rs/object_store/latest/object_store/azure/enum.AzureConfigKey.html
+        storage_options = None
+        conn_s3 = (
+            table_config.connection.s3
+            if table_config.connection and table_config.connection.s3
+            else None
+        )
+        conn_adls = (
+            table_config.connection.adls
+            if table_config.connection and table_config.connection.adls
+            else None
+        )
+        if conn_s3:
+            storage_options = (
+                {
+                    "aws_access_key_id": conn_s3.s3_access_key_id,
+                    "aws_secret_access_key": conn_s3.s3_secret_access_key.get_secret_value(),
+                    "aws_allow_http": str(conn_s3.s3_allow_http).lower(),
+                }
+                | ({"aws_region": conn_s3.s3_region} if conn_s3.s3_region else {})
+                | (
+                    {"aws_endpoint_url": str(conn_s3.s3_endpoint_url).rstrip("/")}
+                    if conn_s3.s3_endpoint_url
+                    else {}
+                )
+            )
+        elif conn_adls:
+            storage_options = (
+                {
+                    "azure_storage_account_name": conn_adls.adls_account_name,
+                    "azure_use_azure_cli": str(conn_adls.use_azure_cli).lower(),
+                }
+                | (
+                    {
+                        "azure_storage_access_key": conn_adls.adls_access_key.get_secret_value()
+                    }
+                    if conn_adls.adls_access_key
+                    else {}
+                )
+                | (
+                    {"azure_storage_sas_key": conn_adls.adls_sas_key.get_secret_value()}
+                    if conn_adls.adls_sas_key
+                    else {}
+                )
+                | (
+                    {"azure_storage_tenant_id": conn_adls.adls_tenant_id}
+                    if conn_adls.adls_tenant_id
+                    else {}
+                )
+                | (
+                    {"azure_storage_client_id": conn_adls.adls_client_id}
+                    if conn_adls.adls_client_id
+                    else {}
+                )
+                | (
+                    {
+                        "azure_storage_client_secret": conn_adls.adls_client_secret.get_secret_value()
+                    }
+                    if conn_adls.adls_client_secret
+                    else {}
+                )
+                | (
+                    {
+                        "azure_msi_endpoint": str(conn_adls.azure_msi_endpoint).rstrip(
+                            "/"
+                        )
+                    }
+                    if conn_adls.azure_msi_endpoint
+                    else {}
+                )
+            )
+        return storage_options
+
+    @classmethod
+    def is_valid(cls, table_config: ConfigTable) -> bool:
+        storage_options = cls._generate_storage_options(table_config)
+        return deltalake.DeltaTable.is_deltatable(
+            table_config.uri, storage_options=storage_options
+        )
 
     def metadata(self) -> TableMetadata:
         metadata = self._impl.metadata()
@@ -96,10 +201,32 @@ class DeltaTable:
             self._impl.load_as_version(version)
         return self._impl.to_pyarrow_dataset()
 
+    def import_data(
+        self, data: pd.DataFrame, mode: ImportModeEnum = ImportModeEnum.append
+    ) -> None:
+        deltalake.write_deltalake(
+            self.table_config.uri, data, mode=mode.value, schema_mode="merge"
+        )
+
 
 def load_table(table_config: ConfigTable) -> TableProtocol:
-    format_handler = {TableFormats.delta: DeltaTable}
-    return format_handler[table_config.table_format](table_config)
+    format_handler: dict[TableFormats, type[TableProtocol]] = {
+        TableFormats.delta: DeltaTable
+    }
+    table_handler = format_handler[table_config.table_format]
+    if not table_handler.is_valid(table_config):
+        raise ValueError(f"Invalid table: {table_config.uri}")
+    return table_handler(table_config)
+
+
+def load_datasets(table_configs: list[ConfigTable]) -> dict[str, padataset.Dataset]:
+    tables_dataset = {}
+    for table_config in table_configs:
+        try:
+            tables_dataset[table_config.name] = load_table(table_config).dataset()
+        except ValueError:
+            pass
+    return tables_dataset
 
 
 def generate_table_query(
@@ -110,21 +237,26 @@ def generate_table_query(
     sort_desc: str | None = None,
 ) -> str:
     query_expr = (
-        sqlglot.select(*(cols or ["*"])).from_(table_name).limit(limit or DEFAULT_LIMIT)
+        sqlglot.select(*([f'"{col}"' for col in cols] if cols else ["*"]))
+        .from_(f'"{table_name}"')
+        .limit(limit or DEFAULT_LIMIT)
     )
     if sort_asc:
         query_expr = query_expr.order_by(f"{sort_asc} asc")
     elif sort_desc:
         query_expr = query_expr.order_by(f"{sort_desc} desc")
-    return sqlglot.Generator(dialect=sqlglot.dialects.duckdb.DuckDB).generate(
-        query_expr
-    )
+    return query_expr.sql(dialect=sqlglot.dialects.duckdb.DuckDB, identify="always")
 
 
 def generate_table_statistics_query(table_name: str) -> str:
-    return (
-        f"SELECT column_name, count, avg, std, min, max FROM (SUMMARIZE {table_name})"  # nosec B608
+    summarize_expr = sqlglot.expressions.Summarize(
+        this=sqlglot.expressions.Table(this=f'"{table_name}"')
     )
+    subquery_expr = sqlglot.expressions.Subquery(this=summarize_expr)
+    query_expr = sqlglot.select(
+        "column_name", "count", "avg", "std", "min", "max"
+    ).from_(subquery_expr)
+    return query_expr.sql(dialect=sqlglot.dialects.duckdb.DuckDB, identify="always")
 
 
 def execute_query(
@@ -133,9 +265,31 @@ def execute_query(
     try:
         conn = duckdb.connect()
         for table_name, table_dataset in tables_datasets.items():
+            # ATTACH IF NOT EXISTS ':memory:' AS {catalog.name};
+            # CREATE SCHEMA IF NOT EXISTS {catalog.name}.{database.name};
+            # USE {catalog.name}.{database.name};
+            # CREATE VIEW IF NOT EXISTS {table.name} AS FROM {table.name}_dataset;
+
             view_name = f"{table_name}_view"
             conn.register(view_name, table_dataset)
-            conn.execute(f"create table {table_name} as select * from {view_name}")  # nosec B608
+            conn.execute(f'create table "{table_name}" as select * from "{view_name}"')  # nosec B608
         return conn.execute(sql_query).df()
     except duckdb.Error as e:
         raise ValueError(str(e)) from e
+
+
+def import_file_to_table(
+    table_config: ConfigTable,
+    file_path: BinaryIO | TextIO,
+    mode: ImportModeEnum = ImportModeEnum.append,
+    file_format: ImportFileFormatEnum = ImportFileFormatEnum.csv,
+    delimiter: str = ",",
+    encoding: str = "utf-8",
+) -> int:
+    file_format_handler = {
+        ImportFileFormatEnum.csv: lambda f, d, e: pd.read_csv(f, sep=d, encoding=e)
+    }
+    table = load_table(table_config)
+    df = file_format_handler[file_format](file_path, delimiter, encoding)
+    table.import_data(df, mode=mode)
+    return len(df)

laketower/templates/queries/view.html

@@ -34,6 +34,12 @@
       {{ error.message }}
     </div>
     {% else %}
+    <div class="d-flex justify-content-between align-items-center mb-2">
+      <h3>Results</h3>
+      <a href="/tables/query/csv?sql={{ query.sql | urlencode }}" class="btn btn-outline-secondary btn-sm">
+        <i class="bi-download" aria-hidden="true"></i> Export CSV
+      </a>
+    </div>
     <div class="table-responsive">
       <table class="table table-sm table-bordered table-striped table-hover">
         <thead>

laketower/templates/tables/_macros.html

@@ -12,5 +12,8 @@
   <li class="nav-item">
     <a class="nav-link{% if current == 'history' %} active{% endif %}"{% if current == 'history' %} aria-current="true"{% endif %} href="/tables/{{ table_id }}/history">History</a>
   </li>
+  <li class="nav-item">
+    <a class="nav-link{% if current == 'import' %} active{% endif %}"{% if current == 'import' %} aria-current="true"{% endif %} href="/tables/{{ table_id }}/import">Import</a>
+  </li>
 </ul>
 {%- endmacro %}

laketower/templates/tables/history.html

@@ -2,6 +2,11 @@
 {% import 'tables/_macros.html' as table_macros %}
 
 {% block body %}
+{% if error %}
+<div class="alert alert-danger" role="alert">
+  {{ error.message }}
+</div>
+{% else %}
 {{ table_macros.table_nav(table_id, 'history') }}
 
 <div class="row">
@@ -39,4 +44,5 @@
       {% endfor %}
   </div>
 </div>
+{% endif %}
 {% endblock %}

laketower/templates/tables/import.html

@@ -0,0 +1,71 @@
+{% extends "_base.html" %}
+{% import 'tables/_macros.html' as table_macros %}
+
+{% block body %}
+{{ table_macros.table_nav(table_id, 'import') }}
+
+{% if message %}
+{% if message.type == 'success' %}{% set alert_type = 'success' %}
+{% elif message.type == 'error' %}{% set alert_type = 'danger' %}
+{% else %}{% set alert_type = 'primary' %}
+{% endif %}
+
+<div class="alert alert-{{ alert_type }}" role="alert">
+  {{ message.body }}
+</div>
+{% endif %}
+
+<div class="row justify-content-center">
+  <div class="col-12 col-md-8 col-lg-4">
+    <form action="{{ request.url.path }}" method="post" enctype="multipart/form-data">
+      <div class="mb-3">
+        <label for="import-file-input" class="form-label">Input file</label>
+        <input id="import-file-input" class="form-control" name="input_file" type="file" accept=".csv" required>
+      </div>
+
+      <div class="mb-3">
+        <label class="form-label">Mode</label>
+
+        <div class="form-check">
+          <input id="import-mode-append" class="form-check-input" name="mode" type="radio" value="append" checked>
+          <label for="import-mode-append" class="form-check-label">Append</label>
+        </div>
+        <div class="form-check">
+          <input id="import-mode-overwrite" class="form-check-input" name="mode" type="radio" value="overwrite">
+          <label for="import-mode-overwrite" class="form-check-label">Overwrite</label>
+        </div>
+      </div>
+
+      <div class="mb-3">
+        <label for="import-file-format" class="form-label">File format</label>
+        <select id="import-file-format" class="form-select" name="file_format">
+          <option value="csv" selected>CSV</option>
+        </select>
+      </div>
+
+      <div class="mb-3">
+        <label for="import-delimiter" class="form-label">Delimiter</label>
+        <input id="import-delimiter" class="form-control" name="delimiter" value="," required>
+      </div>
+
+      <div class="mb-3">
+        <label for="import-encoding" class="form-label">Encoding</label>
+        <select id="import-encoding" class="form-select" name="encoding">
+          <option value="utf-8" selected>UTF-8</option>
+          <option value="utf-16">UTF-16</option>
+          <option value="utf-32">UTF-32</option>
+          <option value="latin-1">Latin-1</option>
+        </select>
+      </div>
+
+      <div class="mb-3">
+        <div class="d-flex justify-content-end">
+          <button type="submit" class="btn btn-primary">
+            <i class="bi-upload" aria-hidden="true"></i> Import Data
+          </button>
+        </div>
+      </div>
+    </form>
+  </div>
+</div>
+{% endblock %}

laketower/templates/tables/index.html

@@ -2,6 +2,11 @@
 {% import 'tables/_macros.html' as table_macros %}
 
 {% block body %}
+{% if error %}
+<div class="alert alert-danger" role="alert">
+  {{ error.message }}
+</div>
+{% else %}
 {{ table_macros.table_nav(table_id, 'overview') }}
 
 <div class="row row-cols-1 row-cols-md-2 g-4">
@@ -81,4 +86,5 @@
     </div>
   </div>
 </div>
+{% endif %}
 {% endblock %}

laketower/templates/tables/query.html

@@ -25,6 +25,12 @@
       {{ error.message }}
     </div>
     {% else %}
+    <div class="d-flex justify-content-between align-items-center mb-2">
+      <h3>Results</h3>
+      <a href="/tables/query/csv?sql={{ sql_query | urlencode }}" class="btn btn-outline-secondary btn-sm">
+        <i class="bi-download" aria-hidden="true"></i> Export CSV
+      </a>
+    </div>
     <div class="table-responsive">
       <table class="table table-sm table-bordered table-striped table-hover">
         <thead>

laketower/templates/tables/statistics.html

@@ -2,6 +2,11 @@
 {% import 'tables/_macros.html' as table_macros %}
 
 {% block body %}
+{% if error %}
+<div class="alert alert-danger" role="alert">
+  {{ error.message }}
+</div>
+{% else %}
 {{ table_macros.table_nav(table_id, 'statistics') }}
 
 <div class="row">
@@ -53,4 +58,5 @@
     </div>
   </div>
 </div>
+{% endif %}
 {% endblock %}

laketower/templates/tables/view.html

@@ -2,6 +2,11 @@
 {% import 'tables/_macros.html' as table_macros %}
 
 {% block body %}
+{% if error %}
+<div class="alert alert-danger" role="alert">
+  {{ error.message }}
+</div>
+{% else %}
 {{ table_macros.table_nav(table_id, 'view') }}
 
 <div class="row">
@@ -93,4 +98,5 @@
     </div>
   </div>
 </div>
+{% endif %}
 {% endblock %}

laketower/web.py

@@ -3,17 +3,21 @@ from pathlib import Path
 from typing import Annotated
 
 import pydantic_settings
-from fastapi import APIRouter, FastAPI, Query, Request
-from fastapi.responses import HTMLResponse
+from fastapi import APIRouter, FastAPI, File, Form, Query, Request, UploadFile
+from fastapi.responses import HTMLResponse, Response
 from fastapi.staticfiles import StaticFiles
 from fastapi.templating import Jinja2Templates
 
 from laketower.config import Config, load_yaml_config
 from laketower.tables import (
     DEFAULT_LIMIT,
+    ImportFileFormatEnum,
+    ImportModeEnum,
     execute_query,
     generate_table_statistics_query,
     generate_table_query,
+    import_file_to_table,
+    load_datasets,
     load_table,
 )
 
@@ -55,10 +59,7 @@ def index(request: Request) -> HTMLResponse:
 @router.get("/tables/query", response_class=HTMLResponse)
 def get_tables_query(request: Request, sql: str) -> HTMLResponse:
     config: Config = request.app.state.config
-    tables_dataset = {
-        table_config.name: load_table(table_config).dataset()
-        for table_config in config.tables
-    }
+    tables_dataset = load_datasets(config.tables)
 
     try:
         results = execute_query(tables_dataset, sql)
@@ -80,13 +81,36 @@ def get_tables_query(request: Request, sql: str) -> HTMLResponse:
     )
 
 
+@router.get("/tables/query/csv")
+def export_tables_query_csv(request: Request, sql: str) -> Response:
+    config: Config = request.app.state.config
+    tables_dataset = load_datasets(config.tables)
+
+    results = execute_query(tables_dataset, sql)
+    csv_content = results.to_csv(header=True, index=False, sep=",")
+
+    return Response(
+        content=csv_content,
+        media_type="text/csv",
+        headers={"Content-Disposition": "attachment; filename=query_results.csv"},
+    )
+
+
 @router.get("/tables/{table_id}", response_class=HTMLResponse)
 def get_table_index(request: Request, table_id: str) -> HTMLResponse:
     config: Config = request.app.state.config
     table_config = next(
         filter(lambda table_config: table_config.name == table_id, config.tables)
     )
-    table = load_table(table_config)
+    try:
+        table = load_table(table_config)
+        table_metadata = table.metadata()
+        table_schema = table.schema()
+        error = None
+    except ValueError as e:
+        error = {"message": str(e)}
+        table_metadata = None
+        table_schema = None
 
     return templates.TemplateResponse(
         request=request,
@@ -95,8 +119,9 @@ def get_table_index(request: Request, table_id: str) -> HTMLResponse:
             "tables": config.tables,
             "queries": config.queries,
             "table_id": table_id,
-            "table_metadata": table.metadata(),
-            "table_schema": table.schema(),
+            "table_metadata": table_metadata,
+            "table_schema": table_schema,
+            "error": error,
         },
     )
 
@@ -107,7 +132,13 @@ def get_table_history(request: Request, table_id: str) -> HTMLResponse:
     table_config = next(
         filter(lambda table_config: table_config.name == table_id, config.tables)
     )
-    table = load_table(table_config)
+    try:
+        table = load_table(table_config)
+        table_history = table.history()
+        error = None
+    except ValueError as e:
+        error = {"message": str(e)}
+        table_history = None
 
     return templates.TemplateResponse(
         request=request,
@@ -116,7 +147,8 @@ def get_table_history(request: Request, table_id: str) -> HTMLResponse:
             "tables": config.tables,
             "queries": config.queries,
             "table_id": table_id,
-            "table_history": table.history(),
+            "table_history": table_history,
+            "error": error,
         },
     )
 
@@ -131,12 +163,18 @@ def get_table_statistics(
     table_config = next(
         filter(lambda table_config: table_config.name == table_id, config.tables)
     )
-    table = load_table(table_config)
-    table_name = table_config.name
-    table_metadata = table.metadata()
-    table_dataset = table.dataset(version=version)
-    sql_query = generate_table_statistics_query(table_name)
-    query_results = execute_query({table_name: table_dataset}, sql_query)
+    try:
+        table = load_table(table_config)
+        table_name = table_config.name
+        table_metadata = table.metadata()
+        table_dataset = table.dataset(version=version)
+        sql_query = generate_table_statistics_query(table_name)
+        query_results = execute_query({table_name: table_dataset}, sql_query)
+        error = None
+    except ValueError as e:
+        error = {"message": str(e)}
+        table_metadata = None
+        query_results = None
 
     return templates.TemplateResponse(
         request=request,
@@ -147,6 +185,7 @@ def get_table_statistics(
             "table_id": table_id,
             "table_metadata": table_metadata,
             "table_results": query_results,
+            "error": error,
         },
     )
 
@@ -165,14 +204,21 @@ def get_table_view(
     table_config = next(
         filter(lambda table_config: table_config.name == table_id, config.tables)
     )
-    table = load_table(table_config)
-    table_name = table_config.name
-    table_metadata = table.metadata()
-    table_dataset = table.dataset(version=version)
-    sql_query = generate_table_query(
-        table_name, limit=limit, cols=cols, sort_asc=sort_asc, sort_desc=sort_desc
-    )
-    results = execute_query({table_name: table_dataset}, sql_query)
+    try:
+        table = load_table(table_config)
+        table_name = table_config.name
+        table_metadata = table.metadata()
+        table_dataset = table.dataset(version=version)
+        sql_query = generate_table_query(
+            table_name, limit=limit, cols=cols, sort_asc=sort_asc, sort_desc=sort_desc
+        )
+        results = execute_query({table_name: table_dataset}, sql_query)
+        error = None
+    except ValueError as e:
+        error = {"message": str(e)}
+        table_metadata = None
+        sql_query = None
+        results = None
 
     return templates.TemplateResponse(
         request=request,
@@ -185,6 +231,78 @@ def get_table_view(
             "table_results": results,
             "sql_query": sql_query,
             "default_limit": DEFAULT_LIMIT,
+            "error": error,
+        },
+    )
+
+
+@router.get("/tables/{table_id}/import", response_class=HTMLResponse)
+def get_table_import(
+    request: Request,
+    table_id: str,
+) -> HTMLResponse:
+    config: Config = request.app.state.config
+    table_config = next(
+        filter(lambda table_config: table_config.name == table_id, config.tables)
+    )
+    try:
+        table = load_table(table_config)
+        table_metadata = table.metadata()
+        message = None
+    except ValueError as e:
+        message = {"type": "error", "body": str(e)}
+        table_metadata = None
+
+    return templates.TemplateResponse(
+        request=request,
+        name="tables/import.html",
+        context={
+            "tables": config.tables,
+            "queries": config.queries,
+            "table_id": table_id,
+            "table_metadata": table_metadata,
+            "message": message,
+        },
+    )
+
+
+@router.post("/tables/{table_id}/import", response_class=HTMLResponse)
+def post_table_import(
+    request: Request,
+    table_id: str,
+    input_file: Annotated[UploadFile, File()],
+    mode: Annotated[ImportModeEnum, Form()],
+    file_format: Annotated[ImportFileFormatEnum, Form()],
+    delimiter: Annotated[str, Form()],
+    encoding: Annotated[str, Form()],
+) -> HTMLResponse:
+    config: Config = request.app.state.config
+    table_config = next(
+        filter(lambda table_config: table_config.name == table_id, config.tables)
+    )
+    try:
+        table = load_table(table_config)
+        table_metadata = table.metadata()
+        rows_imported = import_file_to_table(
+            table_config, input_file.file, mode, file_format, delimiter, encoding
+        )
+        message = {
+            "type": "success",
+            "body": f"Successfully imported {rows_imported} rows",
+        }
+    except Exception as e:
+        message = {"type": "error", "body": str(e)}
+        table_metadata = None
+
+    return templates.TemplateResponse(
+        request=request,
+        name="tables/import.html",
+        context={
+            "tables": config.tables,
+            "queries": config.queries,
+            "table_id": table_id,
+            "table_metadata": table_metadata,
+            "message": message,
         },
     )
 
@@ -195,10 +313,7 @@ def get_query_view(request: Request, query_id: str) -> HTMLResponse:
     query_config = next(
         filter(lambda query_config: query_config.name == query_id, config.queries)
     )
-    tables_dataset = {
-        table_config.name: load_table(table_config).dataset()
-        for table_config in config.tables
-    }
+    tables_dataset = load_datasets(config.tables)
 
     try:
         results = execute_query(tables_dataset, query_config.sql)