laketower 0.3.0__py3-none-any.whl → 0.4.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of laketower might be problematic. Click here for more details.
- laketower/__about__.py +1 -1
- laketower/cli.py +86 -1
- laketower/config.py +2 -0
- laketower/tables.py +7 -0
- laketower/templates/_base.html +33 -2
- laketower/templates/queries/view.html +48 -0
- laketower/templates/tables/_macros.html +3 -0
- laketower/templates/tables/query.html +1 -1
- laketower/templates/tables/statistics.html +56 -0
- laketower/web.py +70 -1
- {laketower-0.3.0.dist-info → laketower-0.4.0.dist-info}/METADATA +110 -10
- laketower-0.4.0.dist-info/RECORD +22 -0
- laketower-0.3.0.dist-info/RECORD +0 -20
- {laketower-0.3.0.dist-info → laketower-0.4.0.dist-info}/WHEEL +0 -0
- {laketower-0.3.0.dist-info → laketower-0.4.0.dist-info}/entry_points.txt +0 -0
- {laketower-0.3.0.dist-info → laketower-0.4.0.dist-info}/licenses/LICENSE.md +0 -0
laketower/__about__.py
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
__version__ = "0.
|
|
1
|
+
__version__ = "0.4.0"
|
laketower/cli.py
CHANGED
|
@@ -11,7 +11,12 @@ import uvicorn
|
|
|
11
11
|
|
|
12
12
|
from laketower.__about__ import __version__
|
|
13
13
|
from laketower.config import load_yaml_config
|
|
14
|
-
from laketower.tables import
|
|
14
|
+
from laketower.tables import (
|
|
15
|
+
execute_query,
|
|
16
|
+
generate_table_query,
|
|
17
|
+
generate_table_statistics_query,
|
|
18
|
+
load_table,
|
|
19
|
+
)
|
|
15
20
|
|
|
16
21
|
|
|
17
22
|
def run_web(config_path: Path, reload: bool) -> None: # pragma: no cover
|
|
@@ -97,6 +102,27 @@ def table_history(config_path: Path, table_name: str) -> None:
|
|
|
97
102
|
console.print(tree, markup=False)
|
|
98
103
|
|
|
99
104
|
|
|
105
|
+
def table_statistics(
|
|
106
|
+
config_path: Path, table_name: str, version: int | None = None
|
|
107
|
+
) -> None:
|
|
108
|
+
config = load_yaml_config(config_path)
|
|
109
|
+
table_config = next(filter(lambda x: x.name == table_name, config.tables))
|
|
110
|
+
table = load_table(table_config)
|
|
111
|
+
table_dataset = table.dataset(version=version)
|
|
112
|
+
sql_query = generate_table_statistics_query(table_name)
|
|
113
|
+
results = execute_query({table_name: table_dataset}, sql_query)
|
|
114
|
+
|
|
115
|
+
out = rich.table.Table()
|
|
116
|
+
for column in results.columns:
|
|
117
|
+
out.add_column(column)
|
|
118
|
+
for value_list in results.to_numpy().tolist():
|
|
119
|
+
row = [str(x) for x in value_list]
|
|
120
|
+
out.add_row(*row)
|
|
121
|
+
|
|
122
|
+
console = rich.get_console()
|
|
123
|
+
console.print(out, markup=False) # disable markup to allow bracket characters
|
|
124
|
+
|
|
125
|
+
|
|
100
126
|
def view_table(
|
|
101
127
|
config_path: Path,
|
|
102
128
|
table_name: str,
|
|
@@ -149,6 +175,40 @@ def query_table(config_path: Path, sql_query: str) -> None:
|
|
|
149
175
|
console.print(out)
|
|
150
176
|
|
|
151
177
|
|
|
178
|
+
def list_queries(config_path: Path) -> None:
|
|
179
|
+
config = load_yaml_config(config_path)
|
|
180
|
+
tree = rich.tree.Tree("queries")
|
|
181
|
+
for query in config.queries:
|
|
182
|
+
tree.add(query.name)
|
|
183
|
+
console = rich.get_console()
|
|
184
|
+
console.print(tree)
|
|
185
|
+
|
|
186
|
+
|
|
187
|
+
def view_query(config_path: Path, query_name: str) -> None:
|
|
188
|
+
config = load_yaml_config(config_path)
|
|
189
|
+
query_config = next(filter(lambda x: x.name == query_name, config.queries))
|
|
190
|
+
sql_query = query_config.sql
|
|
191
|
+
tables_dataset = {
|
|
192
|
+
table_config.name: load_table(table_config).dataset()
|
|
193
|
+
for table_config in config.tables
|
|
194
|
+
}
|
|
195
|
+
|
|
196
|
+
out: rich.jupyter.JupyterMixin
|
|
197
|
+
try:
|
|
198
|
+
results = execute_query(tables_dataset, sql_query)
|
|
199
|
+
out = rich.table.Table()
|
|
200
|
+
for column in results.columns:
|
|
201
|
+
out.add_column(column)
|
|
202
|
+
for value_list in results.values.tolist():
|
|
203
|
+
row = [str(x) for x in value_list]
|
|
204
|
+
out.add_row(*row)
|
|
205
|
+
except ValueError as e:
|
|
206
|
+
out = rich.panel.Panel.fit(f"[red]{e}")
|
|
207
|
+
|
|
208
|
+
console = rich.get_console()
|
|
209
|
+
console.print(out)
|
|
210
|
+
|
|
211
|
+
|
|
152
212
|
def cli() -> None:
|
|
153
213
|
parser = argparse.ArgumentParser(
|
|
154
214
|
"laketower", formatter_class=argparse.ArgumentDefaultsHelpFormatter
|
|
@@ -212,6 +272,17 @@ def cli() -> None:
|
|
|
212
272
|
parser_tables_history.add_argument("table", help="Name of the table")
|
|
213
273
|
parser_tables_history.set_defaults(func=lambda x: table_history(x.config, x.table))
|
|
214
274
|
|
|
275
|
+
parser_tables_statistics = subsparsers_tables.add_parser(
|
|
276
|
+
"statistics", help="Display summary statistics of a given table schema"
|
|
277
|
+
)
|
|
278
|
+
parser_tables_statistics.add_argument("table", help="Name of the table")
|
|
279
|
+
parser_tables_statistics.add_argument(
|
|
280
|
+
"--version", type=int, help="Time-travel to table revision number"
|
|
281
|
+
)
|
|
282
|
+
parser_tables_statistics.set_defaults(
|
|
283
|
+
func=lambda x: table_statistics(x.config, x.table, x.version)
|
|
284
|
+
)
|
|
285
|
+
|
|
215
286
|
parser_tables_view = subsparsers_tables.add_parser(
|
|
216
287
|
"view", help="View a given table"
|
|
217
288
|
)
|
|
@@ -242,5 +313,19 @@ def cli() -> None:
|
|
|
242
313
|
parser_tables_query.add_argument("sql", help="SQL query to execute")
|
|
243
314
|
parser_tables_query.set_defaults(func=lambda x: query_table(x.config, x.sql))
|
|
244
315
|
|
|
316
|
+
parser_queries = subparsers.add_parser("queries", help="Work with queries")
|
|
317
|
+
subsparsers_queries = parser_queries.add_subparsers(required=True)
|
|
318
|
+
|
|
319
|
+
parser_queries_list = subsparsers_queries.add_parser(
|
|
320
|
+
"list", help="List all registered queries"
|
|
321
|
+
)
|
|
322
|
+
parser_queries_list.set_defaults(func=lambda x: list_queries(x.config))
|
|
323
|
+
|
|
324
|
+
parser_queries_view = subsparsers_queries.add_parser(
|
|
325
|
+
"view", help="View a given query"
|
|
326
|
+
)
|
|
327
|
+
parser_queries_view.add_argument("query", help="Name of the query")
|
|
328
|
+
parser_queries_view.set_defaults(func=lambda x: view_query(x.config, x.query))
|
|
329
|
+
|
|
245
330
|
args = parser.parse_args()
|
|
246
331
|
args.func(args)
|
laketower/config.py
CHANGED
|
@@ -29,6 +29,7 @@ class ConfigTable(pydantic.BaseModel):
|
|
|
29
29
|
|
|
30
30
|
class ConfigQuery(pydantic.BaseModel):
|
|
31
31
|
name: str
|
|
32
|
+
title: str
|
|
32
33
|
sql: str
|
|
33
34
|
|
|
34
35
|
|
|
@@ -38,6 +39,7 @@ class ConfigDashboard(pydantic.BaseModel):
|
|
|
38
39
|
|
|
39
40
|
class Config(pydantic.BaseModel):
|
|
40
41
|
tables: list[ConfigTable] = []
|
|
42
|
+
queries: list[ConfigQuery] = []
|
|
41
43
|
|
|
42
44
|
|
|
43
45
|
def load_yaml_config(config_path: Path) -> Config:
|
laketower/tables.py
CHANGED
|
@@ -9,6 +9,7 @@ import pyarrow.dataset as padataset
|
|
|
9
9
|
import pydantic
|
|
10
10
|
import sqlglot
|
|
11
11
|
import sqlglot.dialects.duckdb
|
|
12
|
+
import sqlglot.expressions
|
|
12
13
|
|
|
13
14
|
from laketower.config import ConfigTable, TableFormats
|
|
14
15
|
|
|
@@ -120,6 +121,12 @@ def generate_table_query(
|
|
|
120
121
|
)
|
|
121
122
|
|
|
122
123
|
|
|
124
|
+
def generate_table_statistics_query(table_name: str) -> str:
|
|
125
|
+
return (
|
|
126
|
+
f"SELECT column_name, count, avg, std, min, max FROM (SUMMARIZE {table_name})" # nosec B608
|
|
127
|
+
)
|
|
128
|
+
|
|
129
|
+
|
|
123
130
|
def execute_query(
|
|
124
131
|
tables_datasets: dict[str, padataset.Dataset], sql_query: str
|
|
125
132
|
) -> pd.DataFrame:
|
laketower/templates/_base.html
CHANGED
|
@@ -19,7 +19,14 @@
|
|
|
19
19
|
<a class="sidebar-brand" href="/">
|
|
20
20
|
Laketower
|
|
21
21
|
</a>
|
|
22
|
-
<button
|
|
22
|
+
<button
|
|
23
|
+
type="button"
|
|
24
|
+
class="btn-close d-md-none"
|
|
25
|
+
data-bs-dismiss="offcanvas"
|
|
26
|
+
aria-label="Close"
|
|
27
|
+
data-bs-target="#sidebar"
|
|
28
|
+
>
|
|
29
|
+
</button>
|
|
23
30
|
</div>
|
|
24
31
|
<div class="offcanvas-body">
|
|
25
32
|
<ul class="sidebar-nav">
|
|
@@ -30,12 +37,36 @@
|
|
|
30
37
|
{% for table in tables %}
|
|
31
38
|
{% set table_url = '/tables/' + table.name %}
|
|
32
39
|
<li class="nav-item">
|
|
33
|
-
<a
|
|
40
|
+
<a
|
|
41
|
+
class="text-truncate nav-link{% if request.url.path.startswith(table_url) %} active{% endif %}"
|
|
42
|
+
href="{{ table_url }}"
|
|
43
|
+
aria-current="page"
|
|
44
|
+
>
|
|
34
45
|
<i class="bi-table" aria-hidden="true"></i>
|
|
35
46
|
{{ table.name }}
|
|
36
47
|
</a>
|
|
37
48
|
</li>
|
|
38
49
|
{% endfor %}
|
|
50
|
+
|
|
51
|
+
<li><hr class="sidebar-divider"></li>
|
|
52
|
+
|
|
53
|
+
<li>
|
|
54
|
+
<h6 class="sidebar-header">Queries</h6>
|
|
55
|
+
</li>
|
|
56
|
+
<li><hr class="sidebar-divider"></li>
|
|
57
|
+
{% for query in queries %}
|
|
58
|
+
{% set query_url = '/queries/' + query.name + '/view' %}
|
|
59
|
+
<li class="nav-item">
|
|
60
|
+
<a
|
|
61
|
+
class="text-truncate nav-link{% if request.url.path.startswith(query_url) %} active{% endif %}"
|
|
62
|
+
href="{{ query_url }}"
|
|
63
|
+
aria-current="page"
|
|
64
|
+
>
|
|
65
|
+
<i class="bi-code-square" aria-hidden="true"></i>
|
|
66
|
+
{{ query.title }}
|
|
67
|
+
</a>
|
|
68
|
+
</li>
|
|
69
|
+
{% endfor %}
|
|
39
70
|
</ul>
|
|
40
71
|
</div>
|
|
41
72
|
</nav>
|
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
{% extends "_base.html" %}
|
|
2
|
+
|
|
3
|
+
{% block body %}
|
|
4
|
+
<div class="row">
|
|
5
|
+
<div class="col">
|
|
6
|
+
<h2 class="mb-3">{{ query.title }}</h2>
|
|
7
|
+
|
|
8
|
+
<form action="{{ request.url.path }}" method="get">
|
|
9
|
+
<div class="mb-3">
|
|
10
|
+
<textarea disabled name="sql" rows="5" class="form-control">{{ query.sql }}</textarea>
|
|
11
|
+
</div>
|
|
12
|
+
|
|
13
|
+
<div class="mb-3">
|
|
14
|
+
<div class="d-flex justify-content-end">
|
|
15
|
+
<button type="submit" class="btn btn-primary">Execute</button>
|
|
16
|
+
</div>
|
|
17
|
+
</div>
|
|
18
|
+
</form>
|
|
19
|
+
|
|
20
|
+
{% if error %}
|
|
21
|
+
<div class="alert alert-danger" role="alert">
|
|
22
|
+
{{ error.message }}
|
|
23
|
+
</div>
|
|
24
|
+
{% else %}
|
|
25
|
+
<div class="table-responsive">
|
|
26
|
+
<table class="table table-sm table-bordered table-striped table-hover">
|
|
27
|
+
<thead>
|
|
28
|
+
<tr>
|
|
29
|
+
{% for column in query_results.columns %}
|
|
30
|
+
<th>{{ column }}</th>
|
|
31
|
+
{% endfor %}
|
|
32
|
+
</tr>
|
|
33
|
+
</thead>
|
|
34
|
+
<tbody class="table-group-divider">
|
|
35
|
+
{% for row in query_results.to_numpy().tolist() %}
|
|
36
|
+
<tr>
|
|
37
|
+
{% for col in row %}
|
|
38
|
+
<td>{{ col }}</td>
|
|
39
|
+
{% endfor %}
|
|
40
|
+
</tr>
|
|
41
|
+
{% endfor %}
|
|
42
|
+
</tbody>
|
|
43
|
+
</table>
|
|
44
|
+
</div>
|
|
45
|
+
{% endif %}
|
|
46
|
+
</div>
|
|
47
|
+
</div>
|
|
48
|
+
{% endblock %}
|
|
@@ -6,6 +6,9 @@
|
|
|
6
6
|
<li class="nav-item">
|
|
7
7
|
<a class="nav-link{% if current == 'view' %} active{% endif %}"{% if current == 'view' %} aria-current="true"{% endif %} href="/tables/{{ table_id }}/view">Data</a>
|
|
8
8
|
</li>
|
|
9
|
+
<li class="nav-item">
|
|
10
|
+
<a class="nav-link{% if current == 'statistics' %} active{% endif %}"{% if current == 'statistics' %} aria-current="true"{% endif %} href="/tables/{{ table_id }}/statistics">Statistics</a>
|
|
11
|
+
</li>
|
|
9
12
|
<li class="nav-item">
|
|
10
13
|
<a class="nav-link{% if current == 'history' %} active{% endif %}"{% if current == 'history' %} aria-current="true"{% endif %} href="/tables/{{ table_id }}/history">History</a>
|
|
11
14
|
</li>
|
|
@@ -0,0 +1,56 @@
|
|
|
1
|
+
{% extends "_base.html" %}
|
|
2
|
+
{% import 'tables/_macros.html' as table_macros %}
|
|
3
|
+
|
|
4
|
+
{% block body %}
|
|
5
|
+
{{ table_macros.table_nav(table_id, 'statistics') }}
|
|
6
|
+
|
|
7
|
+
<div class="row">
|
|
8
|
+
<div class="col">
|
|
9
|
+
<div class="table-responsive">
|
|
10
|
+
<table class="table table-sm table-bordered table-striped table-hover">
|
|
11
|
+
<thead>
|
|
12
|
+
<tr>
|
|
13
|
+
{% for column in table_results.columns %}
|
|
14
|
+
<th>{{ column }}</th>
|
|
15
|
+
{% endfor %}
|
|
16
|
+
</tr>
|
|
17
|
+
</thead>
|
|
18
|
+
<tbody class="table-group-divider">
|
|
19
|
+
{% for row in table_results.to_numpy().tolist() %}
|
|
20
|
+
<tr>
|
|
21
|
+
{% for col in row %}
|
|
22
|
+
<td>{{ col }}</td>
|
|
23
|
+
{% endfor %}
|
|
24
|
+
</tr>
|
|
25
|
+
{% endfor %}
|
|
26
|
+
</tbody>
|
|
27
|
+
</table>
|
|
28
|
+
|
|
29
|
+
<div class="d-flex justify-content-between">
|
|
30
|
+
<div class="row">
|
|
31
|
+
<form class="col" ="{{ request.url.path }}" method="get">
|
|
32
|
+
{% for param_name, param_val in request.query_params.multi_items() %}
|
|
33
|
+
{% if param_name != 'version' %}
|
|
34
|
+
<input type="hidden" name="{{ param_name }}" value="{{ param_val }}">
|
|
35
|
+
{% endif %}
|
|
36
|
+
{% endfor %}
|
|
37
|
+
|
|
38
|
+
<div class="input-group">
|
|
39
|
+
<input
|
|
40
|
+
id="version-input"
|
|
41
|
+
name="version"
|
|
42
|
+
type="number"
|
|
43
|
+
class="form-control"
|
|
44
|
+
min="0"
|
|
45
|
+
max="{{ table_metadata.version }}"
|
|
46
|
+
value="{{ request.query_params.version or table_metadata.version }}"
|
|
47
|
+
>
|
|
48
|
+
<button type="submit" class="btn btn-primary">Version</button>
|
|
49
|
+
</div>
|
|
50
|
+
</form>
|
|
51
|
+
</div>
|
|
52
|
+
</div>
|
|
53
|
+
</div>
|
|
54
|
+
</div>
|
|
55
|
+
</div>
|
|
56
|
+
{% endblock %}
|
laketower/web.py
CHANGED
|
@@ -12,6 +12,7 @@ from laketower.config import Config, load_yaml_config
|
|
|
12
12
|
from laketower.tables import (
|
|
13
13
|
DEFAULT_LIMIT,
|
|
14
14
|
execute_query,
|
|
15
|
+
generate_table_statistics_query,
|
|
15
16
|
generate_table_query,
|
|
16
17
|
load_table,
|
|
17
18
|
)
|
|
@@ -44,7 +45,10 @@ def index(request: Request) -> HTMLResponse:
|
|
|
44
45
|
return templates.TemplateResponse(
|
|
45
46
|
request=request,
|
|
46
47
|
name="index.html",
|
|
47
|
-
context={
|
|
48
|
+
context={
|
|
49
|
+
"tables": config.tables,
|
|
50
|
+
"queries": config.queries,
|
|
51
|
+
},
|
|
48
52
|
)
|
|
49
53
|
|
|
50
54
|
|
|
@@ -68,6 +72,7 @@ def get_tables_query(request: Request, sql: str) -> HTMLResponse:
|
|
|
68
72
|
name="tables/query.html",
|
|
69
73
|
context={
|
|
70
74
|
"tables": config.tables,
|
|
75
|
+
"queries": config.queries,
|
|
71
76
|
"table_results": results,
|
|
72
77
|
"sql_query": sql,
|
|
73
78
|
"error": error,
|
|
@@ -88,6 +93,7 @@ def get_table_index(request: Request, table_id: str) -> HTMLResponse:
|
|
|
88
93
|
name="tables/index.html",
|
|
89
94
|
context={
|
|
90
95
|
"tables": config.tables,
|
|
96
|
+
"queries": config.queries,
|
|
91
97
|
"table_id": table_id,
|
|
92
98
|
"table_metadata": table.metadata(),
|
|
93
99
|
"table_schema": table.schema(),
|
|
@@ -108,12 +114,43 @@ def get_table_history(request: Request, table_id: str) -> HTMLResponse:
|
|
|
108
114
|
name="tables/history.html",
|
|
109
115
|
context={
|
|
110
116
|
"tables": config.tables,
|
|
117
|
+
"queries": config.queries,
|
|
111
118
|
"table_id": table_id,
|
|
112
119
|
"table_history": table.history(),
|
|
113
120
|
},
|
|
114
121
|
)
|
|
115
122
|
|
|
116
123
|
|
|
124
|
+
@router.get("/tables/{table_id}/statistics", response_class=HTMLResponse)
|
|
125
|
+
def get_table_statistics(
|
|
126
|
+
request: Request,
|
|
127
|
+
table_id: str,
|
|
128
|
+
version: int | None = None,
|
|
129
|
+
) -> HTMLResponse:
|
|
130
|
+
config: Config = request.app.state.config
|
|
131
|
+
table_config = next(
|
|
132
|
+
filter(lambda table_config: table_config.name == table_id, config.tables)
|
|
133
|
+
)
|
|
134
|
+
table = load_table(table_config)
|
|
135
|
+
table_name = table_config.name
|
|
136
|
+
table_metadata = table.metadata()
|
|
137
|
+
table_dataset = table.dataset(version=version)
|
|
138
|
+
sql_query = generate_table_statistics_query(table_name)
|
|
139
|
+
query_results = execute_query({table_name: table_dataset}, sql_query)
|
|
140
|
+
|
|
141
|
+
return templates.TemplateResponse(
|
|
142
|
+
request=request,
|
|
143
|
+
name="tables/statistics.html",
|
|
144
|
+
context={
|
|
145
|
+
"tables": config.tables,
|
|
146
|
+
"queries": config.queries,
|
|
147
|
+
"table_id": table_id,
|
|
148
|
+
"table_metadata": table_metadata,
|
|
149
|
+
"table_results": query_results,
|
|
150
|
+
},
|
|
151
|
+
)
|
|
152
|
+
|
|
153
|
+
|
|
117
154
|
@router.get("/tables/{table_id}/view", response_class=HTMLResponse)
|
|
118
155
|
def get_table_view(
|
|
119
156
|
request: Request,
|
|
@@ -142,6 +179,7 @@ def get_table_view(
|
|
|
142
179
|
name="tables/view.html",
|
|
143
180
|
context={
|
|
144
181
|
"tables": config.tables,
|
|
182
|
+
"queries": config.queries,
|
|
145
183
|
"table_id": table_id,
|
|
146
184
|
"table_metadata": table_metadata,
|
|
147
185
|
"table_results": results,
|
|
@@ -151,6 +189,37 @@ def get_table_view(
|
|
|
151
189
|
)
|
|
152
190
|
|
|
153
191
|
|
|
192
|
+
@router.get("/queries/{query_id}/view", response_class=HTMLResponse)
|
|
193
|
+
def get_query_view(request: Request, query_id: str) -> HTMLResponse:
|
|
194
|
+
config: Config = request.app.state.config
|
|
195
|
+
query_config = next(
|
|
196
|
+
filter(lambda query_config: query_config.name == query_id, config.queries)
|
|
197
|
+
)
|
|
198
|
+
tables_dataset = {
|
|
199
|
+
table_config.name: load_table(table_config).dataset()
|
|
200
|
+
for table_config in config.tables
|
|
201
|
+
}
|
|
202
|
+
|
|
203
|
+
try:
|
|
204
|
+
results = execute_query(tables_dataset, query_config.sql)
|
|
205
|
+
error = None
|
|
206
|
+
except ValueError as e:
|
|
207
|
+
error = {"message": str(e)}
|
|
208
|
+
results = None
|
|
209
|
+
|
|
210
|
+
return templates.TemplateResponse(
|
|
211
|
+
request=request,
|
|
212
|
+
name="queries/view.html",
|
|
213
|
+
context={
|
|
214
|
+
"tables": config.tables,
|
|
215
|
+
"queries": config.queries,
|
|
216
|
+
"query": query_config,
|
|
217
|
+
"query_results": results,
|
|
218
|
+
"error": error,
|
|
219
|
+
},
|
|
220
|
+
)
|
|
221
|
+
|
|
222
|
+
|
|
154
223
|
def create_app() -> FastAPI:
|
|
155
224
|
settings = Settings() # type: ignore[call-arg]
|
|
156
225
|
config = load_yaml_config(settings.laketower_config_path)
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: laketower
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.4.0
|
|
4
4
|
Summary: Oversee your lakehouse
|
|
5
5
|
Project-URL: Repository, https://github.com/datalpia/laketower
|
|
6
6
|
Project-URL: Issues, https://github.com/datalpia/laketower/issues
|
|
@@ -53,8 +53,10 @@ Utility application to explore and manage tables in your data lakehouse, especia
|
|
|
53
53
|
- Inspect table metadata
|
|
54
54
|
- Inspect table schema
|
|
55
55
|
- Inspect table history
|
|
56
|
+
- Get table statistics
|
|
56
57
|
- View table content with a simple query builder
|
|
57
58
|
- Query all registered tables with DuckDB SQL dialect
|
|
59
|
+
- Execute saved queries
|
|
58
60
|
- Static and versionable YAML configuration
|
|
59
61
|
- Web application
|
|
60
62
|
- CLI application
|
|
@@ -105,6 +107,30 @@ tables:
|
|
|
105
107
|
- name: weather
|
|
106
108
|
uri: demo/weather
|
|
107
109
|
format: delta
|
|
110
|
+
|
|
111
|
+
queries:
|
|
112
|
+
- name: all_data
|
|
113
|
+
title: All data
|
|
114
|
+
sql: |
|
|
115
|
+
select
|
|
116
|
+
sample_table.*,
|
|
117
|
+
weather.*
|
|
118
|
+
from
|
|
119
|
+
sample_table,
|
|
120
|
+
weather
|
|
121
|
+
limit 10
|
|
122
|
+
- name: daily_avg_temperature
|
|
123
|
+
title: Daily average temperature
|
|
124
|
+
sql: |
|
|
125
|
+
select
|
|
126
|
+
date_trunc('day', time) as day,
|
|
127
|
+
round(avg(temperature_2m)) as avg_temperature
|
|
128
|
+
from
|
|
129
|
+
weather
|
|
130
|
+
group by
|
|
131
|
+
day
|
|
132
|
+
order by
|
|
133
|
+
day asc
|
|
108
134
|
```
|
|
109
135
|
|
|
110
136
|
### Web Application
|
|
@@ -121,18 +147,20 @@ Laketower provides a CLI interface:
|
|
|
121
147
|
|
|
122
148
|
```bash
|
|
123
149
|
$ laketower --help
|
|
124
|
-
|
|
150
|
+
|
|
151
|
+
usage: laketower [-h] [--version] [--config CONFIG] {web,config,tables,queries} ...
|
|
125
152
|
|
|
126
153
|
options:
|
|
127
|
-
-h, --help
|
|
128
|
-
--version
|
|
129
|
-
--config, -c CONFIG
|
|
154
|
+
-h, --help show this help message and exit
|
|
155
|
+
--version show program's version number and exit
|
|
156
|
+
--config, -c CONFIG Path to the Laketower YAML configuration file (default: laketower.yml)
|
|
130
157
|
|
|
131
158
|
commands:
|
|
132
|
-
{web,config,tables}
|
|
133
|
-
web
|
|
134
|
-
config
|
|
135
|
-
tables
|
|
159
|
+
{web,config,tables,queries}
|
|
160
|
+
web Launch the web application
|
|
161
|
+
config Work with configuration
|
|
162
|
+
tables Work with tables
|
|
163
|
+
queries Work with queries
|
|
136
164
|
```
|
|
137
165
|
|
|
138
166
|
By default, a YAML configuration file named `laketower.yml` will be looked for.
|
|
@@ -245,6 +273,40 @@ weather
|
|
|
245
273
|
└── operation metrics
|
|
246
274
|
```
|
|
247
275
|
|
|
276
|
+
#### Get statistics of a given table
|
|
277
|
+
|
|
278
|
+
Get basic statistics on all columns of a given table:
|
|
279
|
+
|
|
280
|
+
```bash
|
|
281
|
+
$ laketower -c demo/laketower.yml tables statistics weather
|
|
282
|
+
|
|
283
|
+
┏━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━┳━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━┓
|
|
284
|
+
┃ column_name ┃ count ┃ avg ┃ std ┃ min ┃ max ┃
|
|
285
|
+
┡━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━╇━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━┩
|
|
286
|
+
│ time │ 576 │ None │ None │ 2025-01-26 01:00:00+01 │ 2025-02-12 00:00:00+01 │
|
|
287
|
+
│ city │ 576 │ None │ None │ Grenoble │ Grenoble │
|
|
288
|
+
│ temperature_2m │ 576 │ 5.2623263956047595 │ 3.326529069892729 │ 0.0 │ 15.1 │
|
|
289
|
+
│ relative_humidity_2m │ 576 │ 78.76909722222223 │ 15.701802163559918 │ 29.0 │ 100.0 │
|
|
290
|
+
│ wind_speed_10m │ 576 │ 7.535763886032833 │ 10.00898058743763 │ 0.0 │ 42.4 │
|
|
291
|
+
└──────────────────────┴───────┴────────────────────┴────────────────────┴────────────────────────┴────────────────────────┘
|
|
292
|
+
```
|
|
293
|
+
|
|
294
|
+
Specifying a table version yields according results:
|
|
295
|
+
|
|
296
|
+
```bash
|
|
297
|
+
$ laketower -c demo/laketower.yml tables statistics --version 0 weather
|
|
298
|
+
|
|
299
|
+
┏━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━┳━━━━━━┳━━━━━━┳━━━━━━┳━━━━━━┓
|
|
300
|
+
┃ column_name ┃ count ┃ avg ┃ std ┃ min ┃ max ┃
|
|
301
|
+
┡━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━╇━━━━━━╇━━━━━━╇━━━━━━╇━━━━━━┩
|
|
302
|
+
│ time │ 0 │ None │ None │ None │ None │
|
|
303
|
+
│ city │ 0 │ None │ None │ None │ None │
|
|
304
|
+
│ temperature_2m │ 0 │ None │ None │ None │ None │
|
|
305
|
+
│ relative_humidity_2m │ 0 │ None │ None │ None │ None │
|
|
306
|
+
│ wind_speed_10m │ 0 │ None │ None │ None │ None │
|
|
307
|
+
└──────────────────────┴───────┴──────┴──────┴──────┴──────┘
|
|
308
|
+
```
|
|
309
|
+
|
|
248
310
|
#### View a given table
|
|
249
311
|
|
|
250
312
|
Using a simple query builder, the content of a table can be displayed.
|
|
@@ -308,7 +370,6 @@ $ laketower -c demo/laketower.yml tables view weather --version 1
|
|
|
308
370
|
└───────────────────────────┴──────────┴───────────────────┴──────────────────────┴────────────────────┘
|
|
309
371
|
```
|
|
310
372
|
|
|
311
|
-
|
|
312
373
|
#### Query all registered tables
|
|
313
374
|
|
|
314
375
|
Query any registered tables using DuckDB SQL dialect!
|
|
@@ -325,6 +386,45 @@ $ laketower -c demo/laketower.yml tables query "select date_trunc('day', time) a
|
|
|
325
386
|
└───────────────────────────┴────────────────────┘
|
|
326
387
|
```
|
|
327
388
|
|
|
389
|
+
#### List saved queries
|
|
390
|
+
|
|
391
|
+
```bash
|
|
392
|
+
$ laketower -c demo/laketower.yml queries list
|
|
393
|
+
|
|
394
|
+
queries
|
|
395
|
+
├── all_data
|
|
396
|
+
└── daily_avg_temperature
|
|
397
|
+
```
|
|
398
|
+
|
|
399
|
+
#### Execute saved queries
|
|
400
|
+
|
|
401
|
+
```bash
|
|
402
|
+
$ laketower -c demo/laketower.yml queries view daily_avg_temperature
|
|
403
|
+
|
|
404
|
+
┏━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━┓
|
|
405
|
+
┃ day ┃ avg_temperature ┃
|
|
406
|
+
┡━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━┩
|
|
407
|
+
│ 2025-01-26 00:00:00+01:00 │ 8.0 │
|
|
408
|
+
│ 2025-01-27 00:00:00+01:00 │ 13.0 │
|
|
409
|
+
│ 2025-01-28 00:00:00+01:00 │ 7.0 │
|
|
410
|
+
│ 2025-01-29 00:00:00+01:00 │ 8.0 │
|
|
411
|
+
│ 2025-01-30 00:00:00+01:00 │ 9.0 │
|
|
412
|
+
│ 2025-01-31 00:00:00+01:00 │ 6.0 │
|
|
413
|
+
│ 2025-02-01 00:00:00+01:00 │ 4.0 │
|
|
414
|
+
│ 2025-02-02 00:00:00+01:00 │ 4.0 │
|
|
415
|
+
│ 2025-02-03 00:00:00+01:00 │ 4.0 │
|
|
416
|
+
│ 2025-02-04 00:00:00+01:00 │ 3.0 │
|
|
417
|
+
│ 2025-02-05 00:00:00+01:00 │ 3.0 │
|
|
418
|
+
│ 2025-02-06 00:00:00+01:00 │ 2.0 │
|
|
419
|
+
│ 2025-02-07 00:00:00+01:00 │ 6.0 │
|
|
420
|
+
│ 2025-02-08 00:00:00+01:00 │ 7.0 │
|
|
421
|
+
│ 2025-02-09 00:00:00+01:00 │ 5.0 │
|
|
422
|
+
│ 2025-02-10 00:00:00+01:00 │ 2.0 │
|
|
423
|
+
│ 2025-02-11 00:00:00+01:00 │ 5.0 │
|
|
424
|
+
│ 2025-02-12 00:00:00+01:00 │ 5.0 │
|
|
425
|
+
└───────────────────────────┴─────────────────┘
|
|
426
|
+
```
|
|
427
|
+
|
|
328
428
|
## License
|
|
329
429
|
|
|
330
430
|
Licensed under [GNU Affero General Public License v3.0 (AGPLv3)](LICENSE.md)
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
laketower/__about__.py,sha256=42STGor_9nKYXumfeV5tiyD_M8VdcddX7CEexmibPBk,22
|
|
2
|
+
laketower/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
3
|
+
laketower/__main__.py,sha256=czKxJKG8OfncnxWmpaOWx7b1JBwFnZNQi7wKSTncB4M,108
|
|
4
|
+
laketower/cli.py,sha256=U4gI12egcOs51wxjmQlU70XhA2QGcowc0AmTYpUKEFE,11962
|
|
5
|
+
laketower/config.py,sha256=NdUDF7lr2hEW9Gujp0OpkOKcDP46ju1y_r0IM4Hrx2M,1100
|
|
6
|
+
laketower/tables.py,sha256=QwqoK73Q9pDRzyuoN9pwmwP_WWj3Rg2qPJaIcCdnJbw,4402
|
|
7
|
+
laketower/web.py,sha256=5NMKj26aVz3cKnUAe-3sLDJ_4Ue3u0VXhATrDQ8GVF8,7205
|
|
8
|
+
laketower/static/.gitkeep,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
9
|
+
laketower/templates/_base.html,sha256=S-8kjAfYBx3Btb4FwzM2qyfkGYrOBHhpvCWR32mCvOw,3729
|
|
10
|
+
laketower/templates/index.html,sha256=dLF2Og0qgzBkvGyVRidRNzTv0u4o97ifOx1jVeig8Kg,59
|
|
11
|
+
laketower/templates/queries/view.html,sha256=uYFtlCQ8Pde1bFN17mbryMSu7UqqS1xbrKo8NOs0Tto,1254
|
|
12
|
+
laketower/templates/tables/_macros.html,sha256=fnj_8nBco0iS6mlBmGmfT2PZhI2Y82yP0cm8tRxchpU,965
|
|
13
|
+
laketower/templates/tables/history.html,sha256=yAW0xw9_Uxp0QZYKje6qhcbpeznxI3fb740hfNyILZ8,1740
|
|
14
|
+
laketower/templates/tables/index.html,sha256=oY13l_p8qozlLONanLpga1WhEo4oTP92pRf9sBSuFZI,2394
|
|
15
|
+
laketower/templates/tables/query.html,sha256=chqylXlOhbRALxirebhQc8iYlkLz4cjjRtXGlWb-fXY,1251
|
|
16
|
+
laketower/templates/tables/statistics.html,sha256=rgIOuF2PlHo2jvcYDAnxa5ObNortwyALlrURpM7qxMw,1714
|
|
17
|
+
laketower/templates/tables/view.html,sha256=sFCQlEzIODc-M6VuHIqGI5PnPkGPaYPg21WAzQg_af0,3860
|
|
18
|
+
laketower-0.4.0.dist-info/METADATA,sha256=fWkOW0A76eKHpgxXWqW62f1nWGYYIA8IuBeFEfj4mFM,20486
|
|
19
|
+
laketower-0.4.0.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
|
|
20
|
+
laketower-0.4.0.dist-info/entry_points.txt,sha256=OL_4klopvyEzasJOFJ-sKu54lv24Jvomni32h1WVUjk,48
|
|
21
|
+
laketower-0.4.0.dist-info/licenses/LICENSE.md,sha256=hIahDEOTzuHCU5J2nd07LWwkLW7Hko4UFO__ffsvB-8,34523
|
|
22
|
+
laketower-0.4.0.dist-info/RECORD,,
|
laketower-0.3.0.dist-info/RECORD
DELETED
|
@@ -1,20 +0,0 @@
|
|
|
1
|
-
laketower/__about__.py,sha256=VrXpHDu3erkzwl_WXrqINBm9xWkcyUy53IQOj042dOs,22
|
|
2
|
-
laketower/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
3
|
-
laketower/__main__.py,sha256=czKxJKG8OfncnxWmpaOWx7b1JBwFnZNQi7wKSTncB4M,108
|
|
4
|
-
laketower/cli.py,sha256=_2DM_TrrgZ4qOJhMz0f3g5sKrCwqMe0rIpzQTAwy4p8,8991
|
|
5
|
-
laketower/config.py,sha256=cJ7KKWd2Pv9T_MbpK7QxqD8PdPZ9I38i3-tofcU0KeI,1049
|
|
6
|
-
laketower/tables.py,sha256=1hEkegorMPG-aZ69UbvWcL4QwOgGYe7DvMc1u-i5pUI,4192
|
|
7
|
-
laketower/web.py,sha256=YnwpjiqizmhkoXGBm8ylATwJniIpKTl6RJDB9YMAIz0,5035
|
|
8
|
-
laketower/static/.gitkeep,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
9
|
-
laketower/templates/_base.html,sha256=i60vd7da_iKm4o9DulPWD1Io8dbljNYVf9hcEGHAegI,2927
|
|
10
|
-
laketower/templates/index.html,sha256=dLF2Og0qgzBkvGyVRidRNzTv0u4o97ifOx1jVeig8Kg,59
|
|
11
|
-
laketower/templates/tables/_macros.html,sha256=O-D57cTZDyWOCOQxzM1ZJkrSXdMJ7bhKy_znSsN8FX8,740
|
|
12
|
-
laketower/templates/tables/history.html,sha256=yAW0xw9_Uxp0QZYKje6qhcbpeznxI3fb740hfNyILZ8,1740
|
|
13
|
-
laketower/templates/tables/index.html,sha256=oY13l_p8qozlLONanLpga1WhEo4oTP92pRf9sBSuFZI,2394
|
|
14
|
-
laketower/templates/tables/query.html,sha256=9NFWJDE50AE-P6BPRw9M4TTNqPCjysWsSRHhcw_sqvc,1206
|
|
15
|
-
laketower/templates/tables/view.html,sha256=sFCQlEzIODc-M6VuHIqGI5PnPkGPaYPg21WAzQg_af0,3860
|
|
16
|
-
laketower-0.3.0.dist-info/METADATA,sha256=TcdpEndo5GaQWcFphFBj8oHN6zILFAMeQ9SkoXaWUCk,14963
|
|
17
|
-
laketower-0.3.0.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
|
|
18
|
-
laketower-0.3.0.dist-info/entry_points.txt,sha256=OL_4klopvyEzasJOFJ-sKu54lv24Jvomni32h1WVUjk,48
|
|
19
|
-
laketower-0.3.0.dist-info/licenses/LICENSE.md,sha256=hIahDEOTzuHCU5J2nd07LWwkLW7Hko4UFO__ffsvB-8,34523
|
|
20
|
-
laketower-0.3.0.dist-info/RECORD,,
|
|
File without changes
|
|
File without changes
|
|
File without changes
|