laketower 0.5.0__py3-none-any.whl → 0.6.0__py3-none-any.whl

laketower/tables.py

@@ -1,5 +1,6 @@
+import enum
 from datetime import datetime, timezone
-from typing import Any, Protocol
+from typing import Any, BinaryIO, Protocol, TextIO
 
 import deltalake
 import duckdb
@@ -17,6 +18,15 @@ from laketower.config import ConfigTable, TableFormats
 DEFAULT_LIMIT = 10
 
 
+class ImportModeEnum(str, enum.Enum):
+    append = "append"
+    overwrite = "overwrite"
+
+
+class ImportFileFormatEnum(str, enum.Enum):
+    csv = "csv"
+
+
 class TableMetadata(pydantic.BaseModel):
     table_format: TableFormats
     name: str | None = None
@@ -43,17 +53,112 @@ class TableHistory(pydantic.BaseModel):
 
 
 class TableProtocol(Protocol):  # pragma: no cover
+    @classmethod
+    def is_valid(cls, table_config: ConfigTable) -> bool: ...
+    def __init__(self, table_config: ConfigTable) -> None: ...
     def metadata(self) -> TableMetadata: ...
     def schema(self) -> pa.Schema: ...
     def history(self) -> TableHistory: ...
     def dataset(self, version: int | str | None = None) -> padataset.Dataset: ...
+    def import_data(
+        self, data: pd.DataFrame, mode: ImportModeEnum = ImportModeEnum.append
+    ) -> None: ...
 
 
 class DeltaTable:
     def __init__(self, table_config: ConfigTable):
         super().__init__()
         self.table_config = table_config
-        self._impl = deltalake.DeltaTable(table_config.uri)
+        storage_options = self._generate_storage_options(table_config)
+        self._impl = deltalake.DeltaTable(
+            table_config.uri, storage_options=storage_options
+        )
+
+    @classmethod
+    def _generate_storage_options(
+        cls, table_config: ConfigTable
+    ) -> dict[str, str] | None:
+        # documentation from `object-store` Rust crate:
+        # - s3: https://docs.rs/object_store/latest/object_store/aws/enum.AmazonS3ConfigKey.html
+        # - adls: https://docs.rs/object_store/latest/object_store/azure/enum.AzureConfigKey.html
+        storage_options = None
+        conn_s3 = (
+            table_config.connection.s3
+            if table_config.connection and table_config.connection.s3
+            else None
+        )
+        conn_adls = (
+            table_config.connection.adls
+            if table_config.connection and table_config.connection.adls
+            else None
+        )
+        if conn_s3:
+            storage_options = (
+                {
+                    "aws_access_key_id": conn_s3.s3_access_key_id,
+                    "aws_secret_access_key": conn_s3.s3_secret_access_key.get_secret_value(),
+                    "aws_allow_http": str(conn_s3.s3_allow_http).lower(),
+                }
+                | ({"aws_region": conn_s3.s3_region} if conn_s3.s3_region else {})
+                | (
+                    {"aws_endpoint_url": str(conn_s3.s3_endpoint_url).rstrip("/")}
+                    if conn_s3.s3_endpoint_url
+                    else {}
+                )
+            )
+        elif conn_adls:
+            storage_options = (
+                {
+                    "azure_storage_account_name": conn_adls.adls_account_name,
+                    "azure_use_azure_cli": str(conn_adls.use_azure_cli).lower(),
+                }
+                | (
+                    {
+                        "azure_storage_access_key": conn_adls.adls_access_key.get_secret_value()
+                    }
+                    if conn_adls.adls_access_key
+                    else {}
+                )
+                | (
+                    {"azure_storage_sas_key": conn_adls.adls_sas_key.get_secret_value()}
+                    if conn_adls.adls_sas_key
+                    else {}
+                )
+                | (
+                    {"azure_storage_tenant_id": conn_adls.adls_tenant_id}
+                    if conn_adls.adls_tenant_id
+                    else {}
+                )
+                | (
+                    {"azure_storage_client_id": conn_adls.adls_client_id}
+                    if conn_adls.adls_client_id
+                    else {}
+                )
+                | (
+                    {
+                        "azure_storage_client_secret": conn_adls.adls_client_secret.get_secret_value()
+                    }
+                    if conn_adls.adls_client_secret
+                    else {}
+                )
+                | (
+                    {
+                        "azure_msi_endpoint": str(conn_adls.azure_msi_endpoint).rstrip(
+                            "/"
+                        )
+                    }
+                    if conn_adls.azure_msi_endpoint
+                    else {}
+                )
+            )
+        return storage_options
+
+    @classmethod
+    def is_valid(cls, table_config: ConfigTable) -> bool:
+        storage_options = cls._generate_storage_options(table_config)
+        return deltalake.DeltaTable.is_deltatable(
+            table_config.uri, storage_options=storage_options
+        )
 
     def metadata(self) -> TableMetadata:
         metadata = self._impl.metadata()
@@ -72,7 +177,7 @@ class DeltaTable:
         )
 
     def schema(self) -> pa.Schema:
-        return self._impl.schema().to_pyarrow()
+        return pa.schema(self._impl.schema().to_arrow())  # type: ignore[arg-type]
 
     def history(self) -> TableHistory:
         delta_history = self._impl.history()
@@ -96,10 +201,32 @@ class DeltaTable:
             self._impl.load_as_version(version)
         return self._impl.to_pyarrow_dataset()
 
+    def import_data(
+        self, data: pd.DataFrame, mode: ImportModeEnum = ImportModeEnum.append
+    ) -> None:
+        deltalake.write_deltalake(
+            self.table_config.uri, data, mode=mode.value, schema_mode="merge"
+        )
+
 
 def load_table(table_config: ConfigTable) -> TableProtocol:
-    format_handler = {TableFormats.delta: DeltaTable}
-    return format_handler[table_config.table_format](table_config)
+    format_handler: dict[TableFormats, type[TableProtocol]] = {
+        TableFormats.delta: DeltaTable
+    }
+    table_handler = format_handler[table_config.table_format]
+    if not table_handler.is_valid(table_config):
+        raise ValueError(f"Invalid table: {table_config.uri}")
+    return table_handler(table_config)
+
+
+def load_datasets(table_configs: list[ConfigTable]) -> dict[str, padataset.Dataset]:
+    tables_dataset = {}
+    for table_config in table_configs:
+        try:
+            tables_dataset[table_config.name] = load_table(table_config).dataset()
+        except ValueError:
+            pass
+    return tables_dataset
 
 
 def generate_table_query(
@@ -110,21 +237,26 @@ def generate_table_query(
     sort_desc: str | None = None,
 ) -> str:
     query_expr = (
-        sqlglot.select(*(cols or ["*"])).from_(table_name).limit(limit or DEFAULT_LIMIT)
+        sqlglot.select(*([f'"{col}"' for col in cols] if cols else ["*"]))
+        .from_(f'"{table_name}"')
+        .limit(limit or DEFAULT_LIMIT)
     )
     if sort_asc:
         query_expr = query_expr.order_by(f"{sort_asc} asc")
     elif sort_desc:
         query_expr = query_expr.order_by(f"{sort_desc} desc")
-    return sqlglot.Generator(dialect=sqlglot.dialects.duckdb.DuckDB).generate(
-        query_expr
-    )
+    return query_expr.sql(dialect=sqlglot.dialects.duckdb.DuckDB, identify="always")
 
 
 def generate_table_statistics_query(table_name: str) -> str:
-    return (
-        f"SELECT column_name, count, avg, std, min, max FROM (SUMMARIZE {table_name})"  # nosec B608
+    summarize_expr = sqlglot.expressions.Summarize(
+        this=sqlglot.expressions.Table(this=f'"{table_name}"')
     )
+    subquery_expr = sqlglot.expressions.Subquery(this=summarize_expr)
+    query_expr = sqlglot.select(
+        "column_name", "count", "avg", "std", "min", "max"
+    ).from_(subquery_expr)
+    return query_expr.sql(dialect=sqlglot.dialects.duckdb.DuckDB, identify="always")
 
 
 def execute_query(
@@ -133,9 +265,31 @@ def execute_query(
     try:
         conn = duckdb.connect()
         for table_name, table_dataset in tables_datasets.items():
+            # ATTACH IF NOT EXISTS ':memory:' AS {catalog.name};
+            # CREATE SCHEMA IF NOT EXISTS {catalog.name}.{database.name};
+            # USE {catalog.name}.{database.name};
+            # CREATE VIEW IF NOT EXISTS {table.name} AS FROM {table.name}_dataset;
+
             view_name = f"{table_name}_view"
             conn.register(view_name, table_dataset)
-            conn.execute(f"create table {table_name} as select * from {view_name}")  # nosec B608
+            conn.execute(f'create table "{table_name}" as select * from "{view_name}"')  # nosec B608
         return conn.execute(sql_query).df()
     except duckdb.Error as e:
         raise ValueError(str(e)) from e
+
+
+def import_file_to_table(
+    table_config: ConfigTable,
+    file_path: BinaryIO | TextIO,
+    mode: ImportModeEnum = ImportModeEnum.append,
+    file_format: ImportFileFormatEnum = ImportFileFormatEnum.csv,
+    delimiter: str = ",",
+    encoding: str = "utf-8",
+) -> int:
+    file_format_handler = {
+        ImportFileFormatEnum.csv: lambda f, d, e: pd.read_csv(f, sep=d, encoding=e)
+    }
+    table = load_table(table_config)
+    df = file_format_handler[file_format](file_path, delimiter, encoding)
+    table.import_data(df, mode=mode)
+    return len(df)

laketower/templates/queries/view.html

@@ -34,6 +34,12 @@
       {{ error.message }}
     </div>
     {% else %}
+    <div class="d-flex justify-content-between align-items-center mb-2">
+      <h3>Results</h3>
+      <a href="/tables/query/csv?sql={{ query.sql | urlencode }}" class="btn btn-outline-secondary btn-sm">
+        <i class="bi-download" aria-hidden="true"></i> Export CSV
+      </a>
+    </div>
     <div class="table-responsive">
       <table class="table table-sm table-bordered table-striped table-hover">
         <thead>

laketower/templates/tables/_macros.html

@@ -12,5 +12,8 @@
   <li class="nav-item">
     <a class="nav-link{% if current == 'history' %} active{% endif %}"{% if current == 'history' %} aria-current="true"{% endif %} href="/tables/{{ table_id }}/history">History</a>
   </li>
+  <li class="nav-item">
+    <a class="nav-link{% if current == 'import' %} active{% endif %}"{% if current == 'import' %} aria-current="true"{% endif %} href="/tables/{{ table_id }}/import">Import</a>
+  </li>
 </ul>
 {%- endmacro %}

laketower/templates/tables/history.html

@@ -2,6 +2,11 @@
 {% import 'tables/_macros.html' as table_macros %}
 
 {% block body %}
+{% if error %}
+<div class="alert alert-danger" role="alert">
+  {{ error.message }}
+</div>
+{% else %}
 {{ table_macros.table_nav(table_id, 'history') }}
 
 <div class="row">
@@ -39,4 +44,5 @@
       {% endfor %}
   </div>
 </div>
+{% endif %}
 {% endblock %}

laketower/templates/tables/import.html

@@ -0,0 +1,71 @@
+{% extends "_base.html" %}
+{% import 'tables/_macros.html' as table_macros %}
+
+{% block body %}
+{{ table_macros.table_nav(table_id, 'import') }}
+
+{% if message %}
+{% if message.type == 'success' %}{% set alert_type = 'success' %}
+{% elif message.type == 'error' %}{% set alert_type = 'danger' %}
+{% else %}{% set alert_type = 'primary' %}
+{% endif %}
+
+<div class="alert alert-{{ alert_type }}" role="alert">
+  {{ message.body }}
+</div>
+{% endif %}
+
+<div class="row justify-content-center">
+  <div class="col-12 col-md-8 col-lg-4">
+    <form action="{{ request.url.path }}" method="post" enctype="multipart/form-data">
+      <div class="mb-3">
+        <label for="import-file-input" class="form-label">Input file</label>
+        <input id="import-file-input" class="form-control" name="input_file" type="file" accept=".csv" required>
+      </div>
+
+      <div class="mb-3">
+        <label class="form-label">Mode</label>
+
+        <div class="form-check">
+          <input id="import-mode-append" class="form-check-input" name="mode" type="radio" value="append" checked>
+          <label for="import-mode-append" class="form-check-label">Append</label>
+        </div>
+        <div class="form-check">
+          <input id="import-mode-overwrite" class="form-check-input" name="mode" type="radio" value="overwrite">
+          <label for="import-mode-overwrite" class="form-check-label">Overwrite</label>
+        </div>
+      </div>
+
+      <div class="mb-3">
+        <label for="import-file-format" class="form-label">File format</label>
+        <select id="import-file-format" class="form-select" name="file_format">
+          <option value="csv" selected>CSV</option>
+        </select>
+      </div>
+
+      <div class="mb-3">
+        <label for="import-delimiter" class="form-label">Delimiter</label>
+        <input id="import-delimiter" class="form-control" name="delimiter" value="," required>
+      </div>
+
+      <div class="mb-3">
+        <label for="import-encoding" class="form-label">Encoding</label>
+        <select id="import-encoding" class="form-select" name="encoding">
+          <option value="utf-8" selected>UTF-8</option>
+          <option value="utf-16">UTF-16</option>
+          <option value="utf-32">UTF-32</option>
+          <option value="latin-1">Latin-1</option>
+        </select>
+      </div>
+
+      <div class="mb-3">
+        <div class="d-flex justify-content-end">
+          <button type="submit" class="btn btn-primary">
+            <i class="bi-upload" aria-hidden="true"></i> Import Data
+          </button>
+        </div>
+      </div>
+    </form>
+  </div>
+</div>
+{% endblock %}

laketower/templates/tables/index.html

@@ -2,6 +2,11 @@
 {% import 'tables/_macros.html' as table_macros %}
 
 {% block body %}
+{% if error %}
+<div class="alert alert-danger" role="alert">
+  {{ error.message }}
+</div>
+{% else %}
 {{ table_macros.table_nav(table_id, 'overview') }}
 
 <div class="row row-cols-1 row-cols-md-2 g-4">
@@ -81,4 +86,5 @@
     </div>
   </div>
 </div>
+{% endif %}
 {% endblock %}

laketower/templates/tables/query.html

@@ -25,6 +25,12 @@
       {{ error.message }}
     </div>
     {% else %}
+    <div class="d-flex justify-content-between align-items-center mb-2">
+      <h3>Results</h3>
+      <a href="/tables/query/csv?sql={{ sql_query | urlencode }}" class="btn btn-outline-secondary btn-sm">
+        <i class="bi-download" aria-hidden="true"></i> Export CSV
+      </a>
+    </div>
     <div class="table-responsive">
       <table class="table table-sm table-bordered table-striped table-hover">
         <thead>

laketower/templates/tables/statistics.html

@@ -2,6 +2,11 @@
 {% import 'tables/_macros.html' as table_macros %}
 
 {% block body %}
+{% if error %}
+<div class="alert alert-danger" role="alert">
+  {{ error.message }}
+</div>
+{% else %}
 {{ table_macros.table_nav(table_id, 'statistics') }}
 
 <div class="row">
@@ -53,4 +58,5 @@
     </div>
   </div>
 </div>
+{% endif %}
 {% endblock %}

laketower/templates/tables/view.html

@@ -2,6 +2,11 @@
 {% import 'tables/_macros.html' as table_macros %}
 
 {% block body %}
+{% if error %}
+<div class="alert alert-danger" role="alert">
+  {{ error.message }}
+</div>
+{% else %}
 {{ table_macros.table_nav(table_id, 'view') }}
 
 <div class="row">
@@ -93,4 +98,5 @@
     </div>
   </div>
 </div>
+{% endif %}
 {% endblock %}