laketower 0.1.0__py3-none-any.whl → 0.3.0__py3-none-any.whl

laketower/__about__.py

@@ -1 +1 @@
-__version__ = "0.1.0"
+__version__ = "0.3.0"

laketower/cli.py

@@ -1,178 +1,22 @@
-from __future__ import annotations
-
 import argparse
-import enum
-from datetime import datetime, timezone
+import os
 from pathlib import Path
-from typing import Any
 
-import deltalake
-import duckdb
-import pandas as pd
-import pyarrow as pa
-import pydantic
+import rich.jupyter
 import rich.panel
 import rich.table
 import rich.text
 import rich.tree
-import sqlglot
-import sqlglot.dialects
-import sqlglot.dialects.duckdb
-import sqlglot.generator
-import yaml
+import uvicorn
 
 from laketower.__about__ import __version__
+from laketower.config import load_yaml_config
+from laketower.tables import execute_query, generate_table_query, load_table
 
 
-class TableFormats(str, enum.Enum):
-    delta = "delta"
-
-
-class ConfigTable(pydantic.BaseModel):
-    name: str
-    uri: str
-    table_format: TableFormats = pydantic.Field(alias="format")
-
-    @pydantic.model_validator(mode="after")
-    def check_table(self) -> "ConfigTable":
-        def check_delta_table(table_uri: str) -> None:
-            if not deltalake.DeltaTable.is_deltatable(table_uri):
-                raise ValueError(f"{table_uri} is not a valid Delta table")
-
-        format_check = {TableFormats.delta: check_delta_table}
-        format_check[self.table_format](self.uri)
-
-        return self
-
-
-class ConfigQuery(pydantic.BaseModel):
-    name: str
-    sql: str
-
-
-class ConfigDashboard(pydantic.BaseModel):
-    name: str
-
-
-class Config(pydantic.BaseModel):
-    tables: list[ConfigTable] = []
-
-
-def load_yaml_config(config_path: Path) -> Config:
-    config_dict = yaml.safe_load(config_path.read_text())
-    return Config.model_validate(config_dict)
-
-
-class TableMetadata(pydantic.BaseModel):
-    table_format: TableFormats
-    name: str
-    description: str
-    uri: str
-    id: str
-    version: int
-    created_at: datetime
-    partitions: list[str]
-    configuration: dict[str, str]
-
-
-class TableRevision(pydantic.BaseModel):
-    version: int
-    timestamp: datetime
-    client_version: str
-    operation: str
-    operation_parameters: dict[str, Any]
-    operation_metrics: dict[str, Any]
-
-
-class TableHistory(pydantic.BaseModel):
-    revisions: list[TableRevision]
-
-
-def load_table_metadata(table_config: ConfigTable) -> TableMetadata:
-    def load_delta_table_metadata(table_config: ConfigTable) -> TableMetadata:
-        delta_table = deltalake.DeltaTable(table_config.uri)
-        metadata = delta_table.metadata()
-        return TableMetadata(
-            table_format=table_config.table_format,
-            name=metadata.name,
-            description=metadata.description,
-            uri=delta_table.table_uri,
-            id=str(metadata.id),
-            version=delta_table.version(),
-            created_at=datetime.fromtimestamp(
-                metadata.created_time / 1000, tz=timezone.utc
-            ),
-            partitions=metadata.partition_columns,
-            configuration=metadata.configuration,
-        )
-
-    format_handler = {TableFormats.delta: load_delta_table_metadata}
-    return format_handler[table_config.table_format](table_config)
-
-
-def load_table_schema(table_config: ConfigTable) -> pa.Schema:
-    def load_delta_table_schema(table_config: ConfigTable) -> pa.Schema:
-        delta_table = deltalake.DeltaTable(table_config.uri)
-        return delta_table.schema().to_pyarrow()
-
-    format_handler = {TableFormats.delta: load_delta_table_schema}
-    return format_handler[table_config.table_format](table_config)
-
-
-def load_table_history(table_config: ConfigTable) -> TableHistory:
-    def load_delta_table_history(table_config: ConfigTable) -> TableHistory:
-        delta_table = deltalake.DeltaTable(table_config.uri)
-        delta_history = delta_table.history()
-        revisions = [
-            TableRevision(
-                version=event["version"],
-                timestamp=datetime.fromtimestamp(
-                    event["timestamp"] / 1000, tz=timezone.utc
-                ),
-                client_version=event["clientVersion"],
-                operation=event["operation"],
-                operation_parameters=event["operationParameters"],
-                operation_metrics=event.get("operationMetrics") or {},
-            )
-            for event in delta_history
-        ]
-        return TableHistory(revisions=revisions)
-
-    format_handler = {TableFormats.delta: load_delta_table_history}
-    return format_handler[table_config.table_format](table_config)
-
-
-def load_table_dataset(table_config: ConfigTable) -> pa.dataset.Dataset:
-    def load_delta_table_metadata(table_config: ConfigTable) -> pa.dataset.Dataset:
-        delta_table = deltalake.DeltaTable(table_config.uri)
-        return delta_table.to_pyarrow_dataset()
-
-    format_handler = {TableFormats.delta: load_delta_table_metadata}
-    return format_handler[table_config.table_format](table_config)
-
-
-def execute_query_table(table_config: ConfigTable, sql_query: str) -> pd.DataFrame:
-    table_dataset = load_table_dataset(table_config)
-    table_name = table_config.name
-    view_name = f"{table_name}_view"
-    conn = duckdb.connect()
-    conn.register(view_name, table_dataset)
-    conn.execute(f"create table {table_name} as select * from {view_name}")  # nosec B608
-    return conn.execute(sql_query).df()
-
-
-def execute_query(tables_config: list[ConfigTable], sql_query: str) -> pd.DataFrame:
-    try:
-        conn = duckdb.connect()
-        for table_config in tables_config:
-            table_dataset = load_table_dataset(table_config)
-            table_name = table_config.name
-            view_name = f"{table_name}_view"
-            conn.register(view_name, table_dataset)
-            conn.execute(f"create table {table_name} as select * from {view_name}")  # nosec B608
-        return conn.execute(sql_query).df()
-    except duckdb.Error as e:
-        raise ValueError(str(e)) from e
+def run_web(config_path: Path, reload: bool) -> None:  # pragma: no cover
+    os.environ["LAKETOWER_CONFIG_PATH"] = str(config_path.absolute())
+    uvicorn.run("laketower.web:create_app", factory=True, reload=reload)
 
 
 def validate_config(config_path: Path) -> None:
@@ -200,7 +44,8 @@ def list_tables(config_path: Path) -> None:
 def table_metadata(config_path: Path, table_name: str) -> None:
     config = load_yaml_config(config_path)
     table_config = next(filter(lambda x: x.name == table_name, config.tables))
-    metadata = load_table_metadata(table_config)
+    table = load_table(table_config)
+    metadata = table.metadata()
 
     tree = rich.tree.Tree(table_name)
     tree.add(f"name: {metadata.name}")
@@ -219,7 +64,8 @@ def table_metadata(config_path: Path, table_name: str) -> None:
 def table_schema(config_path: Path, table_name: str) -> None:
     config = load_yaml_config(config_path)
     table_config = next(filter(lambda x: x.name == table_name, config.tables))
-    schema = load_table_schema(table_config)
+    table = load_table(table_config)
+    schema = table.schema()
 
     tree = rich.tree.Tree(table_name)
     for field in schema:
@@ -232,7 +78,8 @@ def table_schema(config_path: Path, table_name: str) -> None:
 def table_history(config_path: Path, table_name: str) -> None:
     config = load_yaml_config(config_path)
     table_config = next(filter(lambda x: x.name == table_name, config.tables))
-    history = load_table_history(table_config)
+    table = load_table(table_config)
+    history = table.history()
 
     tree = rich.tree.Tree(table_name)
     for rev in history.revisions:
@@ -257,22 +104,21 @@ def view_table(
     cols: list[str] | None = None,
     sort_asc: str | None = None,
     sort_desc: str | None = None,
+    version: int | None = None,
 ) -> None:
     config = load_yaml_config(config_path)
     table_config = next(filter(lambda x: x.name == table_name, config.tables))
+    table = load_table(table_config)
+    table_dataset = table.dataset(version=version)
+    sql_query = generate_table_query(
+        table_name, limit=limit, cols=cols, sort_asc=sort_asc, sort_desc=sort_desc
+    )
+    results = execute_query({table_name: table_dataset}, sql_query)
 
-    query_expr = sqlglot.select(*(cols or ["*"])).from_(table_name).limit(limit or 10)
-    if sort_asc:
-        query_expr = query_expr.order_by(f"{sort_asc} asc")
-    elif sort_desc:
-        query_expr = query_expr.order_by(f"{sort_desc} desc")
-    sql_query = sqlglot.Generator(dialect=sqlglot.dialects.DuckDB).generate(query_expr)
-
-    results = execute_query_table(table_config, sql_query)
     out = rich.table.Table()
     for column in results.columns:
         out.add_column(column)
-    for value_list in results.values.tolist():
+    for value_list in results.to_numpy().tolist():
         row = [str(x) for x in value_list]
         out.add_row(*row)
 
@@ -282,10 +128,14 @@ def view_table(
 
 def query_table(config_path: Path, sql_query: str) -> None:
     config = load_yaml_config(config_path)
+    tables_dataset = {
+        table_config.name: load_table(table_config).dataset()
+        for table_config in config.tables
+    }
 
     out: rich.jupyter.JupyterMixin
     try:
-        results = execute_query(config.tables, sql_query)
+        results = execute_query(tables_dataset, sql_query)
         out = rich.table.Table()
         for column in results.columns:
             out.add_column(column)
@@ -300,7 +150,9 @@ def query_table(config_path: Path, sql_query: str) -> None:
 
 
 def cli() -> None:
-    parser = argparse.ArgumentParser("laketower")
+    parser = argparse.ArgumentParser(
+        "laketower", formatter_class=argparse.ArgumentDefaultsHelpFormatter
+    )
     parser.add_argument("--version", action="version", version=__version__)
     parser.add_argument(
         "--config",
@@ -311,6 +163,17 @@ def cli() -> None:
     )
     subparsers = parser.add_subparsers(title="commands", required=True)
 
+    parser_web = subparsers.add_parser(
+        "web", help="Launch the web application", add_help=True
+    )
+    parser_web.add_argument(
+        "--reload",
+        help="Reload the web server on changes",
+        action="store_true",
+        required=False,
+    )
+    parser_web.set_defaults(func=lambda x: run_web(x.config, x.reload))
+
     parser_config = subparsers.add_parser(
         "config", help="Work with configuration", add_help=True
     )
@@ -364,9 +227,12 @@ def cli() -> None:
     parser_tables_view_sort_group.add_argument(
         "--sort-desc", help="Sort by given column in descending order"
     )
+    parser_tables_view.add_argument(
+        "--version", type=int, help="Time-travel to table revision number"
+    )
     parser_tables_view.set_defaults(
         func=lambda x: view_table(
-            x.config, x.table, x.limit, x.cols, x.sort_asc, x.sort_desc
+            x.config, x.table, x.limit, x.cols, x.sort_asc, x.sort_desc, x.version
         )
     )
 

laketower/config.py

@@ -0,0 +1,45 @@
+import enum
+from pathlib import Path
+
+import deltalake
+import pydantic
+import yaml
+
+
+class TableFormats(str, enum.Enum):
+    delta = "delta"
+
+
+class ConfigTable(pydantic.BaseModel):
+    name: str
+    uri: str
+    table_format: TableFormats = pydantic.Field(alias="format")
+
+    @pydantic.model_validator(mode="after")
+    def check_table(self) -> "ConfigTable":
+        def check_delta_table(table_uri: str) -> None:
+            if not deltalake.DeltaTable.is_deltatable(table_uri):
+                raise ValueError(f"{table_uri} is not a valid Delta table")
+
+        format_check = {TableFormats.delta: check_delta_table}
+        format_check[self.table_format](self.uri)
+
+        return self
+
+
+class ConfigQuery(pydantic.BaseModel):
+    name: str
+    sql: str
+
+
+class ConfigDashboard(pydantic.BaseModel):
+    name: str
+
+
+class Config(pydantic.BaseModel):
+    tables: list[ConfigTable] = []
+
+
+def load_yaml_config(config_path: Path) -> Config:
+    config_dict = yaml.safe_load(config_path.read_text())
+    return Config.model_validate(config_dict)

laketower/tables.py

@@ -0,0 +1,134 @@
+from datetime import datetime, timezone
+from typing import Any, Protocol
+
+import deltalake
+import duckdb
+import pandas as pd
+import pyarrow as pa
+import pyarrow.dataset as padataset
+import pydantic
+import sqlglot
+import sqlglot.dialects.duckdb
+
+from laketower.config import ConfigTable, TableFormats
+
+
+DEFAULT_LIMIT = 10
+
+
+class TableMetadata(pydantic.BaseModel):
+    table_format: TableFormats
+    name: str | None = None
+    description: str | None = None
+    uri: str
+    id: str
+    version: int
+    created_at: datetime
+    partitions: list[str]
+    configuration: dict[str, str]
+
+
+class TableRevision(pydantic.BaseModel):
+    version: int
+    timestamp: datetime
+    client_version: str | None = None
+    operation: str
+    operation_parameters: dict[str, Any]
+    operation_metrics: dict[str, Any]
+
+
+class TableHistory(pydantic.BaseModel):
+    revisions: list[TableRevision]
+
+
+class TableProtocol(Protocol):  # pragma: no cover
+    def metadata(self) -> TableMetadata: ...
+    def schema(self) -> pa.Schema: ...
+    def history(self) -> TableHistory: ...
+    def dataset(self, version: int | str | None = None) -> padataset.Dataset: ...
+
+
+class DeltaTable:
+    def __init__(self, table_config: ConfigTable):
+        super().__init__()
+        self.table_config = table_config
+        self._impl = deltalake.DeltaTable(table_config.uri)
+
+    def metadata(self) -> TableMetadata:
+        metadata = self._impl.metadata()
+        return TableMetadata(
+            table_format=self.table_config.table_format,
+            name=metadata.name,
+            description=metadata.description,
+            uri=self._impl.table_uri,
+            id=str(metadata.id),
+            version=self._impl.version(),
+            created_at=datetime.fromtimestamp(
+                metadata.created_time / 1000, tz=timezone.utc
+            ),
+            partitions=metadata.partition_columns,
+            configuration=metadata.configuration,
+        )
+
+    def schema(self) -> pa.Schema:
+        return self._impl.schema().to_pyarrow()
+
+    def history(self) -> TableHistory:
+        delta_history = self._impl.history()
+        revisions = [
+            TableRevision(
+                version=event["version"],
+                timestamp=datetime.fromtimestamp(
+                    event["timestamp"] / 1000, tz=timezone.utc
+                ),
+                client_version=event.get("clientVersion") or event.get("engineInfo"),
+                operation=event["operation"],
+                operation_parameters=event["operationParameters"],
+                operation_metrics=event.get("operationMetrics") or {},
+            )
+            for event in delta_history
+        ]
+        return TableHistory(revisions=revisions)
+
+    def dataset(self, version: int | str | None = None) -> padataset.Dataset:
+        if version is not None:
+            self._impl.load_as_version(version)
+        return self._impl.to_pyarrow_dataset()
+
+
+def load_table(table_config: ConfigTable) -> TableProtocol:
+    format_handler = {TableFormats.delta: DeltaTable}
+    return format_handler[table_config.table_format](table_config)
+
+
+def generate_table_query(
+    table_name: str,
+    limit: int | None = None,
+    cols: list[str] | None = None,
+    sort_asc: str | None = None,
+    sort_desc: str | None = None,
+) -> str:
+    query_expr = (
+        sqlglot.select(*(cols or ["*"])).from_(table_name).limit(limit or DEFAULT_LIMIT)
+    )
+    if sort_asc:
+        query_expr = query_expr.order_by(f"{sort_asc} asc")
+    elif sort_desc:
+        query_expr = query_expr.order_by(f"{sort_desc} desc")
+    return sqlglot.Generator(dialect=sqlglot.dialects.duckdb.DuckDB).generate(
+        query_expr
+    )
+
+
+def execute_query(
+    tables_datasets: dict[str, padataset.Dataset], sql_query: str
+) -> pd.DataFrame:
+    try:
+        conn = duckdb.connect()
+        for table_name, table_dataset in tables_datasets.items():
+            view_name = f"{table_name}_view"
+            conn.register(view_name, table_dataset)
+            conn.execute(f"create table {table_name} as select * from {view_name}")  # nosec B608
+        return conn.execute(sql_query).df()
+    except duckdb.Error as e:
+        raise ValueError(str(e)) from e

laketower/templates/_base.html

@@ -0,0 +1,72 @@
+<!DOCTYPE html>
+<html data-bs-theme="dark" data-bs-core="modern">
+
+<head>
+  <meta charset="utf-8">
+  <meta name="viewport" content="width=device-width, initial-scale=1">
+  <title>Laketower</title>
+  <link href="https://cdn.jsdelivr.net/npm/bootstrap-icons@1.11.3/font/bootstrap-icons.min.css" rel="stylesheet">
+  <link href="https://cdn.jsdelivr.net/npm/halfmoon@2.0.2/css/halfmoon.min.css" rel="stylesheet"
+    integrity="sha256-RjeFzczeuZHCyS+Gvz+kleETzBF/o84ZRHukze/yv6o=" crossorigin="anonymous">
+  <link href="https://cdn.jsdelivr.net/npm/halfmoon@2.0.2/css/cores/halfmoon.modern.css" rel="stylesheet"
+    integrity="sha256-DD6elX+jPmbFYPsGvzodUv2+9FHkxHlVtQi0/RJVULs=" crossorigin="anonymous">
+</head>
+
+<body class="ps-md-sbwidth">
+  <header>
+    <nav class="sidebar offcanvas-start offcanvas-md" tabindex="-1" id="sidebar">
+      <div class="offcanvas-header border-bottom">
+        <a class="sidebar-brand" href="/">
+          Laketower
+        </a>
+        <button type="button" class="btn-close d-md-none" data-bs-dismiss="offcanvas" aria-label="Close" data-bs-target="#sidebar"></button>
+      </div>
+      <div class="offcanvas-body">
+        <ul class="sidebar-nav">
+          <li>
+            <h6 class="sidebar-header">Tables</h6>
+          </li>
+          <li><hr class="sidebar-divider"></li>
+          {% for table in tables %}
+          {% set table_url = '/tables/' + table.name %}
+          <li class="nav-item">
+            <a class="text-truncate nav-link{% if request.url.path.startswith(table_url) %} active{% endif %}" href="{{ table_url }}" aria-current="page">
+              <i class="bi-table" aria-hidden="true"></i>
+              {{ table.name }}
+            </a>
+          </li>
+          {% endfor %}
+        </ul>
+      </div>
+    </nav>
+
+    <nav class="navbar navbar-expand-md sticky-top" style="border-bottom: var(--bs-border-width) solid var(--bs-content-border-color); min-height: 60px;">
+      <div class="container-md justify-content-start">
+        <button type="button" data-bs-toggle="offcanvas" data-bs-target="#sidebar" class="btn btn-secondary d-md-none">
+          <i class="bi-layout-sidebar" aria-hidden="true"></i>
+        </button>
+
+        <a href="/" class="navbar-brand d-flex align-items-center d-md-none ms-auto ms-md-0">
+					Laketower
+				</a>
+      </div>
+    </nav>
+  </header>
+
+  <main>
+    <div class="container py-3 py-sm-4 px-3 px-sm-4">
+      {% block body %}{% endblock %}
+    </div>
+  </main>
+
+  <footer></footer>
+
+  <script src="https://cdn.jsdelivr.net/npm/@popperjs/core@2.11.8/dist/umd/popper.min.js"
+    integrity="sha384-I7E8VVD/ismYTF4hNIPjVp/Zjvgyol6VFvRkX/vR+Vc4jQkC+hVqc2pM8ODewa9r"
+    crossorigin="anonymous"></script>
+  <script src="https://cdn.jsdelivr.net/npm/bootstrap@5.3.3/dist/js/bootstrap.min.js"
+    integrity="sha384-0pUGZvbkm6XF6gxjEnlmuGrJXVbNuzT9qBBavbLwCsOGabYfZo0T0to5eqruptLy"
+    crossorigin="anonymous"></script>
+</body>
+
+</html>

laketower/templates/index.html

@@ -0,0 +1,4 @@
+{% extends "_base.html" %}
+
+{% block body %}
+{% endblock %}

laketower/templates/tables/_macros.html

@@ -0,0 +1,13 @@
+{% macro table_nav(table_id, current) -%}
+<ul class="nav nav-pills justify-content-center mb-3">
+  <li class="nav-item">
+    <a class="nav-link{% if current == 'overview' %} active{% endif %}"{% if current == 'overview' %} aria-current="true"{% endif %} href="/tables/{{ table_id }}">Overview</a>
+  </li>
+  <li class="nav-item">
+    <a class="nav-link{% if current == 'view' %} active{% endif %}"{% if current == 'view' %} aria-current="true"{% endif %} href="/tables/{{ table_id }}/view">Data</a>
+  </li>
+  <li class="nav-item">
+    <a class="nav-link{% if current == 'history' %} active{% endif %}"{% if current == 'history' %} aria-current="true"{% endif %} href="/tables/{{ table_id }}/history">History</a>
+  </li>
+</ul>
+{%- endmacro %}

laketower/templates/tables/history.html

@@ -0,0 +1,42 @@
+{% extends "_base.html" %}
+{% import 'tables/_macros.html' as table_macros %}
+
+{% block body %}
+{{ table_macros.table_nav(table_id, 'history') }}
+
+<div class="row">
+  <div class="col">
+    <div class="accordion" id="accordion-history">
+      {% for revision in table_history.revisions %}
+      <div class="accordion-item">
+        <h2 class="accordion-header">
+          <button class="accordion-button{% if not loop.first %} collapsed{% endif %}" type="button" data-bs-toggle="collapse" data-bs-target="#collapse-{{ loop.index }}" aria-expanded="{% if loop.first %}true{% else %}false{% endif %}" aria-controls="collapse-{{ loop.index }}">
+            version: {{ revision.version }}
+          </button>
+        </h2>
+        <div id="collapse-{{ loop.index }}" class="accordion-collapse collapse{% if loop.first %} show{% endif %}" data-bs-parent="#accordion-history">
+          <div class="accordion-body">
+            <ul>
+              <li>timestamp: {{ revision.timestamp }}</li>
+              <li>operation: {{ revision.operation }}</li>
+              <li>operation parameters</li>
+              <ul>
+                {% for param_key, param_val in revision.operation_parameters.items() %}
+                <li>{{ param_key }}: {{ param_val }}</li>
+                {% endfor %}
+              </ul>
+              <li>operation metrics</li>
+              <ul>
+                {% for metric_key, metric_val in revision.operation_metrics.items() %}
+                <li>{{ metric_key }}: {{ metric_val }}</li>
+                {% endfor %}
+              </ul>
+              <li>client version: {{ revision.client_version }}</li>
+            </ul>
+          </div>
+        </div>
+      </div>
+      {% endfor %}
+  </div>
+</div>
+{% endblock %}

laketower/templates/tables/index.html

@@ -0,0 +1,84 @@
+{% extends "_base.html" %}
+{% import 'tables/_macros.html' as table_macros %}
+
+{% block body %}
+{{ table_macros.table_nav(table_id, 'overview') }}
+
+<div class="row row-cols-1 row-cols-md-2 g-4">
+  <div class="col">
+    <div class="card h-100">
+      <div class="card-header">Schema</div>
+      <div class="card-body">
+        <div class="table-responsive">
+          <table class="table">
+            <thead>
+              <th>Column</th>
+              <th>Type</th>
+              <th>Nullable</th>
+            </thead>
+            <tbody>
+              {% for field in table_schema %}
+              <tr>
+                <td>{{ field.name }}</td>
+                <td><code>{{ field.type }}</code></td>
+                <td>{% if field.nullable %}True{% else %}False{% endif %}</td>
+              </tr>
+              {% endfor %}
+            </tbody>
+          </table>
+        </div>
+      </div>
+    </div>
+  </div>
+
+  <div class="col">
+    <div class="card h-100">
+      <div class="card-header">Metadata</div>
+      <div class="card-body">
+        <div class="table-responsive">
+          <table class="table">
+            <tbody>
+              <tr>
+                <th>URI</th>
+                <td>{{ table_metadata.uri }}</td>
+              </tr>
+              <tr>
+                <th>Format</th>
+                <td>{{ table_metadata.table_format.value }}</td>
+              </tr>
+              <tr>
+                <th>ID</th>
+                <td>{{ table_metadata.id }}</td>
+              </tr>
+              <tr>
+                <th>Version</th>
+                <td>{{ table_metadata.version }}</td>
+              </tr>
+              <tr>
+                <th>Name</th>
+                <td>{{ table_metadata.name }}</td>
+              </tr>
+              <tr>
+                <th>Description</th>
+                <td>{{ table_metadata.description }}</td>
+              </tr>
+              <tr>
+                <th>Created at</th>
+                <td>{{ table_metadata.created_at }}</td>
+              </tr>
+              <tr>
+                <th>Partitions</th>
+                <td>{{ table_metadata.partitions }}</td>
+              </tr>
+              <tr>
+                <th>Configuration</th>
+                <td>{{ table_metadata.configuration }}</td>
+              </tr>
+            </tbody>
+          </table>
+        </div>
+      </div>
+    </div>
+  </div>
+</div>
+{% endblock %}

laketower/templates/tables/query.html

@@ -0,0 +1,47 @@
+{% extends "_base.html" %}
+{% import 'tables/_macros.html' as table_macros %}
+
+{% block body %}
+<div class="row">
+  <div class="col">
+    <form>
+      <div class="mb-3">
+        <textarea name="sql" rows="5" class="form-control">{{ sql_query }}</textarea>
+      </div>
+
+      <div class="mb-3">
+        <div class="d-flex justify-content-end">
+          <button type="submit" class="btn btn-primary">Execute</button>
+        </div>
+      </div>
+    </form>
+
+    {% if error %}
+    <div class="alert alert-danger" role="alert">
+      {{ error.message }}
+    </div>
+    {% else %}
+    <div class="table-responsive">
+      <table class="table table-sm table-bordered table-striped table-hover">
+        <thead>
+          <tr>
+            {% for column in table_results.columns %}
+            <th>{{ column }}</th>
+            {% endfor %}
+          </tr>
+        </thead>
+        <tbody class="table-group-divider">
+          {% for row in table_results.to_numpy().tolist() %}
+          <tr>
+            {% for col in row %}
+            <td>{{ col }}</td>
+            {% endfor %}
+          </tr>
+          {% endfor %}
+        </tbody>
+      </table>
+    </div>
+    {% endif %}
+  </div>
+</div>
+{% endblock %}

laketower/templates/tables/view.html

@@ -0,0 +1,96 @@
+{% extends "_base.html" %}
+{% import 'tables/_macros.html' as table_macros %}
+
+{% block body %}
+{{ table_macros.table_nav(table_id, 'view') }}
+
+<div class="row">
+  <div class="col">
+    <div class="table-responsive">
+      <table class="table table-sm table-bordered table-striped table-hover">
+        <thead>
+          <tr>
+            {% for column in table_results.columns %}
+            <th>
+              {{ column }}
+              {% if column == request.query_params.sort_asc %}
+              <a href="{{ request | current_path_with_args([('sort_asc', None), ('sort_desc', column)]) }}" style="text-decoration: none;">
+                <i class="bi-sort-up" aria-hidden="true"></i>
+              </a>
+              {% elif column == request.query_params.sort_desc %}
+              <a href="{{ request | current_path_with_args([('sort_asc', column), ('sort_desc', None)]) }}" style="text-decoration: none;">
+                <i class="bi-sort-down" aria-hidden="true"></i>
+              </a>
+              {% else %}
+              <a href="{{ request | current_path_with_args([('sort_asc', column), ('sort_desc', None)]) }}" class="text-decoration-none">
+                <i class="bi-arrow-down-up" aria-hidden="true"></i>
+              </a>
+              {% endif %}
+              {% set other_cols = table_results.columns | list | reject('equalto', column) | list %}
+              {% set cols_args = [] %}
+              {% for col in other_cols %}
+                {% set tmp = cols_args.append(('cols', col)) %}
+              {% endfor %}
+              <a href='{{ request | current_path_with_args(cols_args) }}' class='text-decoration-none'>
+                <i class='bi-eye-slash' aria-hidden='true'></i>
+              </a>
+            </th>
+            {% endfor %}
+          </tr>
+        </thead>
+        <tbody class="table-group-divider">
+          {% for row in table_results.to_numpy().tolist() %}
+          <tr>
+            {% for col in row %}
+            <td>{{ col }}</td>
+            {% endfor %}
+          </tr>
+          {% endfor %}
+        </tbody>
+      </table>
+
+      <div class="d-flex justify-content-between">
+        <div class="row">
+          <form class="col" action="{{ request.url.path }}" method="get">
+            {% for param_name, param_val in request.query_params.multi_items() %}
+            {% if param_name != 'limit' %}
+            <input type="hidden" name="{{ param_name }}" value="{{ param_val }}">
+            {% endif %}
+            {% endfor %}
+
+            <div class="input-group">
+              <input id="limit-input" name="limit" type="number" class="form-control" min="1" max="10000" value="{{ request.query_params.limit or default_limit }}">
+              <button type="submit" class="btn btn-primary">Limit</button>
+            </div>
+          </form>
+
+          <form class="col" ="{{ request.url.path }}" method="get">
+            {% for param_name, param_val in request.query_params.multi_items() %}
+            {% if param_name != 'version' %}
+            <input type="hidden" name="{{ param_name }}" value="{{ param_val }}">
+            {% endif %}
+            {% endfor %}
+
+            <div class="input-group">
+              <input
+                id="version-input"
+                name="version"
+                type="number"
+                class="form-control"
+                min="0"
+                max="{{ table_metadata.version }}"
+                value="{{ request.query_params.version or table_metadata.version }}"
+              >
+              <button type="submit" class="btn btn-primary">Version</button>
+            </div>
+          </form>
+        </div>
+
+        <a class="btn btn-primary" href="/tables/query?sql={{ sql_query }}" role="button">
+          <i class="bi-code" aria-hidden="true"></i> SQL Query
+        </a>
+      </div>
+    </div>
+  </div>
+</div>
+{% endblock %}

laketower/web.py

@@ -0,0 +1,167 @@
+import urllib.parse
+from pathlib import Path
+from typing import Annotated
+
+import pydantic_settings
+from fastapi import APIRouter, FastAPI, Query, Request
+from fastapi.responses import HTMLResponse
+from fastapi.staticfiles import StaticFiles
+from fastapi.templating import Jinja2Templates
+
+from laketower.config import Config, load_yaml_config
+from laketower.tables import (
+    DEFAULT_LIMIT,
+    execute_query,
+    generate_table_query,
+    load_table,
+)
+
+
+class Settings(pydantic_settings.BaseSettings):
+    laketower_config_path: Path
+
+
+def current_path_with_args(request: Request, args: list[tuple[str, str]]) -> str:
+    keys_to_update = set(arg[0] for arg in args)
+    query_params = request.query_params.multi_items()
+    new_query_params = list(
+        filter(lambda param: param[0] not in keys_to_update, query_params)
+    )
+    new_query_params.extend((k, v) for k, v in args if v is not None)
+    query_string = urllib.parse.urlencode(new_query_params)
+    return f"{request.url.path}?{query_string}"
+
+
+templates = Jinja2Templates(directory=Path(__file__).parent / "templates")
+templates.env.filters["current_path_with_args"] = current_path_with_args
+
+router = APIRouter()
+
+
+@router.get("/", response_class=HTMLResponse)
+def index(request: Request) -> HTMLResponse:
+    config: Config = request.app.state.config
+    return templates.TemplateResponse(
+        request=request,
+        name="index.html",
+        context={"tables": config.tables},
+    )
+
+
+@router.get("/tables/query", response_class=HTMLResponse)
+def get_tables_query(request: Request, sql: str) -> HTMLResponse:
+    config: Config = request.app.state.config
+    tables_dataset = {
+        table_config.name: load_table(table_config).dataset()
+        for table_config in config.tables
+    }
+
+    try:
+        results = execute_query(tables_dataset, sql)
+        error = None
+    except ValueError as e:
+        error = {"message": str(e)}
+        results = None
+
+    return templates.TemplateResponse(
+        request=request,
+        name="tables/query.html",
+        context={
+            "tables": config.tables,
+            "table_results": results,
+            "sql_query": sql,
+            "error": error,
+        },
+    )
+
+
+@router.get("/tables/{table_id}", response_class=HTMLResponse)
+def get_table_index(request: Request, table_id: str) -> HTMLResponse:
+    config: Config = request.app.state.config
+    table_config = next(
+        filter(lambda table_config: table_config.name == table_id, config.tables)
+    )
+    table = load_table(table_config)
+
+    return templates.TemplateResponse(
+        request=request,
+        name="tables/index.html",
+        context={
+            "tables": config.tables,
+            "table_id": table_id,
+            "table_metadata": table.metadata(),
+            "table_schema": table.schema(),
+        },
+    )
+
+
+@router.get("/tables/{table_id}/history", response_class=HTMLResponse)
+def get_table_history(request: Request, table_id: str) -> HTMLResponse:
+    config: Config = request.app.state.config
+    table_config = next(
+        filter(lambda table_config: table_config.name == table_id, config.tables)
+    )
+    table = load_table(table_config)
+
+    return templates.TemplateResponse(
+        request=request,
+        name="tables/history.html",
+        context={
+            "tables": config.tables,
+            "table_id": table_id,
+            "table_history": table.history(),
+        },
+    )
+
+
+@router.get("/tables/{table_id}/view", response_class=HTMLResponse)
+def get_table_view(
+    request: Request,
+    table_id: str,
+    limit: int | None = None,
+    cols: Annotated[list[str] | None, Query()] = None,
+    sort_asc: str | None = None,
+    sort_desc: str | None = None,
+    version: int | None = None,
+) -> HTMLResponse:
+    config: Config = request.app.state.config
+    table_config = next(
+        filter(lambda table_config: table_config.name == table_id, config.tables)
+    )
+    table = load_table(table_config)
+    table_name = table_config.name
+    table_metadata = table.metadata()
+    table_dataset = table.dataset(version=version)
+    sql_query = generate_table_query(
+        table_name, limit=limit, cols=cols, sort_asc=sort_asc, sort_desc=sort_desc
+    )
+    results = execute_query({table_name: table_dataset}, sql_query)
+
+    return templates.TemplateResponse(
+        request=request,
+        name="tables/view.html",
+        context={
+            "tables": config.tables,
+            "table_id": table_id,
+            "table_metadata": table_metadata,
+            "table_results": results,
+            "sql_query": sql_query,
+            "default_limit": DEFAULT_LIMIT,
+        },
+    )
+
+
+def create_app() -> FastAPI:
+    settings = Settings()  # type: ignore[call-arg]
+    config = load_yaml_config(settings.laketower_config_path)
+
+    app = FastAPI(title="laketower")
+    app.mount(
+        "/static",
+        StaticFiles(directory=Path(__file__).parent / "static"),
+        name="static",
+    )
+    app.include_router(router)
+    app.state.config = config
+
+    return app

{laketower-0.1.0.dist-info → laketower-0.3.0.dist-info}/METADATA

@@ -1,24 +1,39 @@
 Metadata-Version: 2.4
 Name: laketower
-Version: 0.1.0
+Version: 0.3.0
 Summary: Oversee your lakehouse
+Project-URL: Repository, https://github.com/datalpia/laketower
+Project-URL: Issues, https://github.com/datalpia/laketower/issues
+Project-URL: Changelog, https://github.com/datalpia/laketower/blob/master/CHANGELOG.md
 Author-email: Romain Clement <git@romain-clement.net>
 License: AGPL-3.0-or-later
 License-File: LICENSE.md
+Keywords: data,delta-lake,lakehouse,sql
 Classifier: Development Status :: 2 - Pre-Alpha
 Classifier: Intended Audience :: Developers
+Classifier: Intended Audience :: End Users/Desktop
 Classifier: Intended Audience :: Information Technology
+Classifier: Intended Audience :: Other Audience
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Topic :: Database
 Classifier: Topic :: Software Development
 Classifier: Topic :: Utilities
-Requires-Python: <3.14,>=3.9
+Requires-Python: <3.14,>=3.10
 Requires-Dist: deltalake
 Requires-Dist: duckdb
+Requires-Dist: fastapi
+Requires-Dist: jinja2>=3
 Requires-Dist: pandas
-Requires-Dist: pyarrow<19
-Requires-Dist: pydantic
+Requires-Dist: pyarrow!=19.0.0
+Requires-Dist: pydantic-settings>=2
+Requires-Dist: pydantic>=2
 Requires-Dist: pyyaml
 Requires-Dist: rich
 Requires-Dist: sqlglot
+Requires-Dist: uvicorn
 Description-Content-Type: text/markdown
 
 # Laketower
@@ -26,8 +41,9 @@ Description-Content-Type: text/markdown
 > Oversee your lakehouse
 
 [![PyPI](https://img.shields.io/pypi/v/laketower.svg)](https://pypi.org/project/laketower/)
+[![Python Versions](https://img.shields.io/pypi/pyversions/laketower?logo=python&logoColor=white)](https://pypi.org/project/laketower/)
 [![CI/CD](https://github.com/datalpia/laketower/actions/workflows/ci-cd.yml/badge.svg)](https://github.com/datalpia/laketower/actions/workflows/ci-cd.yml)
-[![License](https://img.shields.io/github/license/datalpia/laketower)](https://github.com/datalpia/laketower/blob/main/LICENSE)
+[![License](https://img.shields.io/github/license/datalpia/laketower)](https://github.com/datalpia/laketower/blob/main/LICENSE.md)
 
 Utility application to explore and manage tables in your data lakehouse, especially tailored for data pipelines local development.
 
@@ -40,6 +56,7 @@ Utility application to explore and manage tables in your data lakehouse, especia
 - View table content with a simple query builder
 - Query all registered tables with DuckDB SQL dialect
 - Static and versionable YAML configuration
+- Web application
 - CLI application
 
 ## Installation
@@ -90,21 +107,30 @@ tables:
     format: delta
 ```
 
+### Web Application
+
+The easiest way to get started is to launch the Laketower web application:
+
+```bash
+$ laketower -c demo/laketower.yml web
+```
+
 ### CLI
 
 Laketower provides a CLI interface:
 
 ```bash
 $ laketower --help
-usage: laketower [-h] [--version] [--config CONFIG] {config,tables} ...
+usage: laketower [-h] [--version] [--config CONFIG] {web,config,tables} ...
 
 options:
   -h, --help           show this help message and exit
   --version            show program's version number and exit
-  --config, -c CONFIG  Path to the Laketower YAML configuration file
+  --config, -c CONFIG  Path to the Laketower YAML configuration file (default: laketower.yml)
 
 commands:
-  {config,tables}
+  {web,config,tables}
+    web                Launch the web application
     config             Work with configuration
     tables             Work with tables
 ```
@@ -228,6 +254,7 @@ Optional arguments:
 - `--sort-asc <col>`: sort by a column name in ascending order
 - `--sort-desc <col>`: sort by a column name in descending order
 - `--limit <num>` (default 10): limit the number of rows
+- `--version`: time-travel to table revision number
 
 ```bash
 $ laketower -c demo/laketower.yml tables view weather
@@ -262,6 +289,26 @@ $ laketower -c demo/laketower.yml tables view weather --cols time city temperatu
 └───────────────────────────┴──────────┴───────────────────┘
 ```
 
+```bash
+$ laketower -c demo/laketower.yml tables view weather --version 1
+
+┏━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━┓
+┃ time                      ┃ city     ┃ temperature_2m    ┃ relative_humidity_2m ┃ wind_speed_10m     ┃
+┡━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━┩
+│ 2025-01-26 01:00:00+01:00 │ Grenoble │ 7.0               │ 87.0                 │ 8.899999618530273  │
+│ 2025-01-26 02:00:00+01:00 │ Grenoble │ 6.099999904632568 │ 87.0                 │ 6.199999809265137  │
+│ 2025-01-26 03:00:00+01:00 │ Grenoble │ 6.0               │ 86.0                 │ 2.700000047683716  │
+│ 2025-01-26 04:00:00+01:00 │ Grenoble │ 6.099999904632568 │ 82.0                 │ 3.0999999046325684 │
+│ 2025-01-26 05:00:00+01:00 │ Grenoble │ 5.5               │ 87.0                 │ 3.299999952316284  │
+│ 2025-01-26 06:00:00+01:00 │ Grenoble │ 5.199999809265137 │ 91.0                 │ 2.200000047683716  │
+│ 2025-01-26 07:00:00+01:00 │ Grenoble │ 4.800000190734863 │ 86.0                 │ 3.0                │
+│ 2025-01-26 08:00:00+01:00 │ Grenoble │ 4.900000095367432 │ 83.0                 │ 1.100000023841858  │
+│ 2025-01-26 09:00:00+01:00 │ Grenoble │ 4.0               │ 92.0                 │ 3.0999999046325684 │
+│ 2025-01-26 10:00:00+01:00 │ Grenoble │ 5.0               │ 86.0                 │ 6.400000095367432  │
+└───────────────────────────┴──────────┴───────────────────┴──────────────────────┴────────────────────┘
+```
+
+
 #### Query all registered tables
 
 Query any registered tables using DuckDB SQL dialect!

laketower-0.3.0.dist-info/RECORD

@@ -0,0 +1,20 @@
+laketower/__about__.py,sha256=VrXpHDu3erkzwl_WXrqINBm9xWkcyUy53IQOj042dOs,22
+laketower/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+laketower/__main__.py,sha256=czKxJKG8OfncnxWmpaOWx7b1JBwFnZNQi7wKSTncB4M,108
+laketower/cli.py,sha256=_2DM_TrrgZ4qOJhMz0f3g5sKrCwqMe0rIpzQTAwy4p8,8991
+laketower/config.py,sha256=cJ7KKWd2Pv9T_MbpK7QxqD8PdPZ9I38i3-tofcU0KeI,1049
+laketower/tables.py,sha256=1hEkegorMPG-aZ69UbvWcL4QwOgGYe7DvMc1u-i5pUI,4192
+laketower/web.py,sha256=YnwpjiqizmhkoXGBm8ylATwJniIpKTl6RJDB9YMAIz0,5035
+laketower/static/.gitkeep,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+laketower/templates/_base.html,sha256=i60vd7da_iKm4o9DulPWD1Io8dbljNYVf9hcEGHAegI,2927
+laketower/templates/index.html,sha256=dLF2Og0qgzBkvGyVRidRNzTv0u4o97ifOx1jVeig8Kg,59
+laketower/templates/tables/_macros.html,sha256=O-D57cTZDyWOCOQxzM1ZJkrSXdMJ7bhKy_znSsN8FX8,740
+laketower/templates/tables/history.html,sha256=yAW0xw9_Uxp0QZYKje6qhcbpeznxI3fb740hfNyILZ8,1740
+laketower/templates/tables/index.html,sha256=oY13l_p8qozlLONanLpga1WhEo4oTP92pRf9sBSuFZI,2394
+laketower/templates/tables/query.html,sha256=9NFWJDE50AE-P6BPRw9M4TTNqPCjysWsSRHhcw_sqvc,1206
+laketower/templates/tables/view.html,sha256=sFCQlEzIODc-M6VuHIqGI5PnPkGPaYPg21WAzQg_af0,3860
+laketower-0.3.0.dist-info/METADATA,sha256=TcdpEndo5GaQWcFphFBj8oHN6zILFAMeQ9SkoXaWUCk,14963
+laketower-0.3.0.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+laketower-0.3.0.dist-info/entry_points.txt,sha256=OL_4klopvyEzasJOFJ-sKu54lv24Jvomni32h1WVUjk,48
+laketower-0.3.0.dist-info/licenses/LICENSE.md,sha256=hIahDEOTzuHCU5J2nd07LWwkLW7Hko4UFO__ffsvB-8,34523
+laketower-0.3.0.dist-info/RECORD,,

laketower-0.1.0.dist-info/RECORD

@@ -1,9 +0,0 @@
-laketower/__about__.py,sha256=kUR5RAFc7HCeiqdlX36dZOHkUI5wI6V_43RpEcD8b-0,22
-laketower/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-laketower/__main__.py,sha256=czKxJKG8OfncnxWmpaOWx7b1JBwFnZNQi7wKSTncB4M,108
-laketower/cli.py,sha256=W0fmtYwprapBCvIyZS12Sdfxs82WWznLEdftX6kJbRE,13385
-laketower-0.1.0.dist-info/METADATA,sha256=JGVuSka5mnQJg2PVbtvNxGMEeVKtBOC86qCt5SDSw5o,11511
-laketower-0.1.0.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-laketower-0.1.0.dist-info/entry_points.txt,sha256=OL_4klopvyEzasJOFJ-sKu54lv24Jvomni32h1WVUjk,48
-laketower-0.1.0.dist-info/licenses/LICENSE.md,sha256=hIahDEOTzuHCU5J2nd07LWwkLW7Hko4UFO__ffsvB-8,34523
-laketower-0.1.0.dist-info/RECORD,,