laketower 0.2.0__tar.gz → 0.4.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of laketower might be problematic. Click here for more details.
- {laketower-0.2.0 → laketower-0.4.0}/.github/workflows/ci-cd.yml +1 -1
- {laketower-0.2.0 → laketower-0.4.0}/CHANGELOG.md +25 -1
- {laketower-0.2.0 → laketower-0.4.0}/PKG-INFO +111 -12
- {laketower-0.2.0 → laketower-0.4.0}/README.md +109 -9
- laketower-0.4.0/demo/laketower.yml +31 -0
- laketower-0.4.0/laketower/__about__.py +1 -0
- {laketower-0.2.0 → laketower-0.4.0}/laketower/cli.py +86 -3
- {laketower-0.2.0 → laketower-0.4.0}/laketower/config.py +2 -0
- {laketower-0.2.0 → laketower-0.4.0}/laketower/tables.py +11 -6
- {laketower-0.2.0 → laketower-0.4.0}/laketower/templates/_base.html +33 -2
- laketower-0.4.0/laketower/templates/queries/view.html +48 -0
- {laketower-0.2.0 → laketower-0.4.0}/laketower/templates/tables/_macros.html +3 -0
- {laketower-0.2.0 → laketower-0.4.0}/laketower/templates/tables/query.html +7 -1
- laketower-0.4.0/laketower/templates/tables/statistics.html +56 -0
- {laketower-0.2.0 → laketower-0.4.0}/laketower/web.py +84 -8
- {laketower-0.2.0 → laketower-0.4.0}/pyproject.toml +14 -14
- laketower-0.4.0/renovate.json +11 -0
- {laketower-0.2.0 → laketower-0.4.0}/tests/conftest.py +8 -1
- {laketower-0.2.0 → laketower-0.4.0}/tests/test_cli.py +163 -3
- {laketower-0.2.0 → laketower-0.4.0}/tests/test_web.py +122 -5
- {laketower-0.2.0 → laketower-0.4.0}/uv.lock +110 -292
- laketower-0.2.0/demo/laketower.yml +0 -7
- laketower-0.2.0/laketower/__about__.py +0 -1
- {laketower-0.2.0 → laketower-0.4.0}/.gitignore +0 -0
- {laketower-0.2.0 → laketower-0.4.0}/.python-version +0 -0
- {laketower-0.2.0 → laketower-0.4.0}/LICENSE.md +0 -0
- {laketower-0.2.0 → laketower-0.4.0}/demo/generate.py +0 -0
- {laketower-0.2.0 → laketower-0.4.0}/demo/sample_table/_delta_log/00000000000000000000.json +0 -0
- {laketower-0.2.0 → laketower-0.4.0}/demo/sample_table/_delta_log/00000000000000000001.json +0 -0
- {laketower-0.2.0 → laketower-0.4.0}/demo/sample_table/_delta_log/00000000000000000002.json +0 -0
- {laketower-0.2.0 → laketower-0.4.0}/demo/sample_table/_delta_log/00000000000000000003.json +0 -0
- {laketower-0.2.0 → laketower-0.4.0}/demo/sample_table/part-00001-1a31a393-6db6-4d1a-bf4e-81ea061ff8cd-c000.snappy.parquet +0 -0
- {laketower-0.2.0 → laketower-0.4.0}/demo/sample_table/part-00001-5af77102-9207-4c89-aaf6-37e1f815ec26-c000.snappy.parquet +0 -0
- {laketower-0.2.0 → laketower-0.4.0}/demo/sample_table/part-00001-b11bab55-43d0-4d05-ae88-5b9481ae57db-c000.snappy.parquet +0 -0
- {laketower-0.2.0 → laketower-0.4.0}/demo/weather/_delta_log/00000000000000000000.json +0 -0
- {laketower-0.2.0 → laketower-0.4.0}/demo/weather/_delta_log/00000000000000000001.json +0 -0
- {laketower-0.2.0 → laketower-0.4.0}/demo/weather/_delta_log/00000000000000000002.json +0 -0
- {laketower-0.2.0 → laketower-0.4.0}/demo/weather/part-00001-2323b963-be56-44e0-8c10-e237e7e6d4b9-c000.snappy.parquet +0 -0
- {laketower-0.2.0 → laketower-0.4.0}/demo/weather/part-00001-6360cbf8-f8a9-475f-8729-6f20b4ca64a9-c000.snappy.parquet +0 -0
- {laketower-0.2.0 → laketower-0.4.0}/laketower/__init__.py +0 -0
- {laketower-0.2.0 → laketower-0.4.0}/laketower/__main__.py +0 -0
- {laketower-0.2.0 → laketower-0.4.0}/laketower/static/.gitkeep +0 -0
- {laketower-0.2.0 → laketower-0.4.0}/laketower/templates/index.html +0 -0
- {laketower-0.2.0 → laketower-0.4.0}/laketower/templates/tables/history.html +0 -0
- {laketower-0.2.0 → laketower-0.4.0}/laketower/templates/tables/index.html +0 -0
- {laketower-0.2.0 → laketower-0.4.0}/laketower/templates/tables/view.html +0 -0
- {laketower-0.2.0 → laketower-0.4.0}/tasks.py +0 -0
- {laketower-0.2.0 → laketower-0.4.0}/tests/__init__.py +0 -0
|
@@ -7,6 +7,28 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
|
|
|
7
7
|
|
|
8
8
|
## [Unreleased]
|
|
9
9
|
|
|
10
|
+
## [0.4.0] - 2025-03-01
|
|
11
|
+
Introducing new features:
|
|
12
|
+
- Display tables statistics
|
|
13
|
+
- List and execute pre-defined queries
|
|
14
|
+
|
|
15
|
+
### Added
|
|
16
|
+
- web: add queries view page
|
|
17
|
+
- web: add table statistics page with version control
|
|
18
|
+
- cli: add queries view command
|
|
19
|
+
- cli: add queries list command
|
|
20
|
+
- cli: add table statistics command
|
|
21
|
+
|
|
22
|
+
## [0.3.0] - 2025-02-27
|
|
23
|
+
Minor release with fixes and dropped Python 3.9 support.
|
|
24
|
+
|
|
25
|
+
### BREAKING CHANGES
|
|
26
|
+
- deps: drop support for python 3.9
|
|
27
|
+
|
|
28
|
+
### Fixed
|
|
29
|
+
- web: handle invalid tables sql query
|
|
30
|
+
- web: truncate long table names in sidebar
|
|
31
|
+
|
|
10
32
|
## [0.2.0] - 2025-02-25
|
|
11
33
|
Introducing the Laketower web application!
|
|
12
34
|
|
|
@@ -37,6 +59,8 @@ Initial release of `laketower`.
|
|
|
37
59
|
- View a given table with simple query builder
|
|
38
60
|
- Query all registered tables with DuckDB SQL dialect
|
|
39
61
|
|
|
40
|
-
[Unreleased]: https://github.com/datalpia/laketower/compare/0.
|
|
62
|
+
[Unreleased]: https://github.com/datalpia/laketower/compare/0.4.0...HEAD
|
|
63
|
+
[0.4.0]: https://github.com/datalpia/laketower/compare/0.3.0...0.4.0
|
|
64
|
+
[0.3.0]: https://github.com/datalpia/laketower/compare/0.2.0...0.3.0
|
|
41
65
|
[0.2.0]: https://github.com/datalpia/laketower/compare/0.1.0...0.2.0
|
|
42
66
|
[0.1.0]: https://github.com/datalpia/laketower/releases/tag/0.1.0
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: laketower
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.4.0
|
|
4
4
|
Summary: Oversee your lakehouse
|
|
5
5
|
Project-URL: Repository, https://github.com/datalpia/laketower
|
|
6
6
|
Project-URL: Issues, https://github.com/datalpia/laketower/issues
|
|
@@ -14,7 +14,6 @@ Classifier: Intended Audience :: Developers
|
|
|
14
14
|
Classifier: Intended Audience :: End Users/Desktop
|
|
15
15
|
Classifier: Intended Audience :: Information Technology
|
|
16
16
|
Classifier: Intended Audience :: Other Audience
|
|
17
|
-
Classifier: Programming Language :: Python :: 3.9
|
|
18
17
|
Classifier: Programming Language :: Python :: 3.10
|
|
19
18
|
Classifier: Programming Language :: Python :: 3.11
|
|
20
19
|
Classifier: Programming Language :: Python :: 3.12
|
|
@@ -22,7 +21,7 @@ Classifier: Programming Language :: Python :: 3.13
|
|
|
22
21
|
Classifier: Topic :: Database
|
|
23
22
|
Classifier: Topic :: Software Development
|
|
24
23
|
Classifier: Topic :: Utilities
|
|
25
|
-
Requires-Python: <3.14,>=3.
|
|
24
|
+
Requires-Python: <3.14,>=3.10
|
|
26
25
|
Requires-Dist: deltalake
|
|
27
26
|
Requires-Dist: duckdb
|
|
28
27
|
Requires-Dist: fastapi
|
|
@@ -54,8 +53,10 @@ Utility application to explore and manage tables in your data lakehouse, especia
|
|
|
54
53
|
- Inspect table metadata
|
|
55
54
|
- Inspect table schema
|
|
56
55
|
- Inspect table history
|
|
56
|
+
- Get table statistics
|
|
57
57
|
- View table content with a simple query builder
|
|
58
58
|
- Query all registered tables with DuckDB SQL dialect
|
|
59
|
+
- Execute saved queries
|
|
59
60
|
- Static and versionable YAML configuration
|
|
60
61
|
- Web application
|
|
61
62
|
- CLI application
|
|
@@ -106,6 +107,30 @@ tables:
|
|
|
106
107
|
- name: weather
|
|
107
108
|
uri: demo/weather
|
|
108
109
|
format: delta
|
|
110
|
+
|
|
111
|
+
queries:
|
|
112
|
+
- name: all_data
|
|
113
|
+
title: All data
|
|
114
|
+
sql: |
|
|
115
|
+
select
|
|
116
|
+
sample_table.*,
|
|
117
|
+
weather.*
|
|
118
|
+
from
|
|
119
|
+
sample_table,
|
|
120
|
+
weather
|
|
121
|
+
limit 10
|
|
122
|
+
- name: daily_avg_temperature
|
|
123
|
+
title: Daily average temperature
|
|
124
|
+
sql: |
|
|
125
|
+
select
|
|
126
|
+
date_trunc('day', time) as day,
|
|
127
|
+
round(avg(temperature_2m)) as avg_temperature
|
|
128
|
+
from
|
|
129
|
+
weather
|
|
130
|
+
group by
|
|
131
|
+
day
|
|
132
|
+
order by
|
|
133
|
+
day asc
|
|
109
134
|
```
|
|
110
135
|
|
|
111
136
|
### Web Application
|
|
@@ -122,18 +147,20 @@ Laketower provides a CLI interface:
|
|
|
122
147
|
|
|
123
148
|
```bash
|
|
124
149
|
$ laketower --help
|
|
125
|
-
|
|
150
|
+
|
|
151
|
+
usage: laketower [-h] [--version] [--config CONFIG] {web,config,tables,queries} ...
|
|
126
152
|
|
|
127
153
|
options:
|
|
128
|
-
-h, --help
|
|
129
|
-
--version
|
|
130
|
-
--config, -c CONFIG
|
|
154
|
+
-h, --help show this help message and exit
|
|
155
|
+
--version show program's version number and exit
|
|
156
|
+
--config, -c CONFIG Path to the Laketower YAML configuration file (default: laketower.yml)
|
|
131
157
|
|
|
132
158
|
commands:
|
|
133
|
-
{web,config,tables}
|
|
134
|
-
web
|
|
135
|
-
config
|
|
136
|
-
tables
|
|
159
|
+
{web,config,tables,queries}
|
|
160
|
+
web Launch the web application
|
|
161
|
+
config Work with configuration
|
|
162
|
+
tables Work with tables
|
|
163
|
+
queries Work with queries
|
|
137
164
|
```
|
|
138
165
|
|
|
139
166
|
By default, a YAML configuration file named `laketower.yml` will be looked for.
|
|
@@ -246,6 +273,40 @@ weather
|
|
|
246
273
|
└── operation metrics
|
|
247
274
|
```
|
|
248
275
|
|
|
276
|
+
#### Get statistics of a given table
|
|
277
|
+
|
|
278
|
+
Get basic statistics on all columns of a given table:
|
|
279
|
+
|
|
280
|
+
```bash
|
|
281
|
+
$ laketower -c demo/laketower.yml tables statistics weather
|
|
282
|
+
|
|
283
|
+
┏━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━┳━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━┓
|
|
284
|
+
┃ column_name ┃ count ┃ avg ┃ std ┃ min ┃ max ┃
|
|
285
|
+
┡━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━╇━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━┩
|
|
286
|
+
│ time │ 576 │ None │ None │ 2025-01-26 01:00:00+01 │ 2025-02-12 00:00:00+01 │
|
|
287
|
+
│ city │ 576 │ None │ None │ Grenoble │ Grenoble │
|
|
288
|
+
│ temperature_2m │ 576 │ 5.2623263956047595 │ 3.326529069892729 │ 0.0 │ 15.1 │
|
|
289
|
+
│ relative_humidity_2m │ 576 │ 78.76909722222223 │ 15.701802163559918 │ 29.0 │ 100.0 │
|
|
290
|
+
│ wind_speed_10m │ 576 │ 7.535763886032833 │ 10.00898058743763 │ 0.0 │ 42.4 │
|
|
291
|
+
└──────────────────────┴───────┴────────────────────┴────────────────────┴────────────────────────┴────────────────────────┘
|
|
292
|
+
```
|
|
293
|
+
|
|
294
|
+
Specifying a table version yields according results:
|
|
295
|
+
|
|
296
|
+
```bash
|
|
297
|
+
$ laketower -c demo/laketower.yml tables statistics --version 0 weather
|
|
298
|
+
|
|
299
|
+
┏━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━┳━━━━━━┳━━━━━━┳━━━━━━┳━━━━━━┓
|
|
300
|
+
┃ column_name ┃ count ┃ avg ┃ std ┃ min ┃ max ┃
|
|
301
|
+
┡━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━╇━━━━━━╇━━━━━━╇━━━━━━╇━━━━━━┩
|
|
302
|
+
│ time │ 0 │ None │ None │ None │ None │
|
|
303
|
+
│ city │ 0 │ None │ None │ None │ None │
|
|
304
|
+
│ temperature_2m │ 0 │ None │ None │ None │ None │
|
|
305
|
+
│ relative_humidity_2m │ 0 │ None │ None │ None │ None │
|
|
306
|
+
│ wind_speed_10m │ 0 │ None │ None │ None │ None │
|
|
307
|
+
└──────────────────────┴───────┴──────┴──────┴──────┴──────┘
|
|
308
|
+
```
|
|
309
|
+
|
|
249
310
|
#### View a given table
|
|
250
311
|
|
|
251
312
|
Using a simple query builder, the content of a table can be displayed.
|
|
@@ -309,7 +370,6 @@ $ laketower -c demo/laketower.yml tables view weather --version 1
|
|
|
309
370
|
└───────────────────────────┴──────────┴───────────────────┴──────────────────────┴────────────────────┘
|
|
310
371
|
```
|
|
311
372
|
|
|
312
|
-
|
|
313
373
|
#### Query all registered tables
|
|
314
374
|
|
|
315
375
|
Query any registered tables using DuckDB SQL dialect!
|
|
@@ -326,6 +386,45 @@ $ laketower -c demo/laketower.yml tables query "select date_trunc('day', time) a
|
|
|
326
386
|
└───────────────────────────┴────────────────────┘
|
|
327
387
|
```
|
|
328
388
|
|
|
389
|
+
#### List saved queries
|
|
390
|
+
|
|
391
|
+
```bash
|
|
392
|
+
$ laketower -c demo/laketower.yml queries list
|
|
393
|
+
|
|
394
|
+
queries
|
|
395
|
+
├── all_data
|
|
396
|
+
└── daily_avg_temperature
|
|
397
|
+
```
|
|
398
|
+
|
|
399
|
+
#### Execute saved queries
|
|
400
|
+
|
|
401
|
+
```bash
|
|
402
|
+
$ laketower -c demo/laketower.yml queries view daily_avg_temperature
|
|
403
|
+
|
|
404
|
+
┏━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━┓
|
|
405
|
+
┃ day ┃ avg_temperature ┃
|
|
406
|
+
┡━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━┩
|
|
407
|
+
│ 2025-01-26 00:00:00+01:00 │ 8.0 │
|
|
408
|
+
│ 2025-01-27 00:00:00+01:00 │ 13.0 │
|
|
409
|
+
│ 2025-01-28 00:00:00+01:00 │ 7.0 │
|
|
410
|
+
│ 2025-01-29 00:00:00+01:00 │ 8.0 │
|
|
411
|
+
│ 2025-01-30 00:00:00+01:00 │ 9.0 │
|
|
412
|
+
│ 2025-01-31 00:00:00+01:00 │ 6.0 │
|
|
413
|
+
│ 2025-02-01 00:00:00+01:00 │ 4.0 │
|
|
414
|
+
│ 2025-02-02 00:00:00+01:00 │ 4.0 │
|
|
415
|
+
│ 2025-02-03 00:00:00+01:00 │ 4.0 │
|
|
416
|
+
│ 2025-02-04 00:00:00+01:00 │ 3.0 │
|
|
417
|
+
│ 2025-02-05 00:00:00+01:00 │ 3.0 │
|
|
418
|
+
│ 2025-02-06 00:00:00+01:00 │ 2.0 │
|
|
419
|
+
│ 2025-02-07 00:00:00+01:00 │ 6.0 │
|
|
420
|
+
│ 2025-02-08 00:00:00+01:00 │ 7.0 │
|
|
421
|
+
│ 2025-02-09 00:00:00+01:00 │ 5.0 │
|
|
422
|
+
│ 2025-02-10 00:00:00+01:00 │ 2.0 │
|
|
423
|
+
│ 2025-02-11 00:00:00+01:00 │ 5.0 │
|
|
424
|
+
│ 2025-02-12 00:00:00+01:00 │ 5.0 │
|
|
425
|
+
└───────────────────────────┴─────────────────┘
|
|
426
|
+
```
|
|
427
|
+
|
|
329
428
|
## License
|
|
330
429
|
|
|
331
430
|
Licensed under [GNU Affero General Public License v3.0 (AGPLv3)](LICENSE.md)
|
|
@@ -15,8 +15,10 @@ Utility application to explore and manage tables in your data lakehouse, especia
|
|
|
15
15
|
- Inspect table metadata
|
|
16
16
|
- Inspect table schema
|
|
17
17
|
- Inspect table history
|
|
18
|
+
- Get table statistics
|
|
18
19
|
- View table content with a simple query builder
|
|
19
20
|
- Query all registered tables with DuckDB SQL dialect
|
|
21
|
+
- Execute saved queries
|
|
20
22
|
- Static and versionable YAML configuration
|
|
21
23
|
- Web application
|
|
22
24
|
- CLI application
|
|
@@ -67,6 +69,30 @@ tables:
|
|
|
67
69
|
- name: weather
|
|
68
70
|
uri: demo/weather
|
|
69
71
|
format: delta
|
|
72
|
+
|
|
73
|
+
queries:
|
|
74
|
+
- name: all_data
|
|
75
|
+
title: All data
|
|
76
|
+
sql: |
|
|
77
|
+
select
|
|
78
|
+
sample_table.*,
|
|
79
|
+
weather.*
|
|
80
|
+
from
|
|
81
|
+
sample_table,
|
|
82
|
+
weather
|
|
83
|
+
limit 10
|
|
84
|
+
- name: daily_avg_temperature
|
|
85
|
+
title: Daily average temperature
|
|
86
|
+
sql: |
|
|
87
|
+
select
|
|
88
|
+
date_trunc('day', time) as day,
|
|
89
|
+
round(avg(temperature_2m)) as avg_temperature
|
|
90
|
+
from
|
|
91
|
+
weather
|
|
92
|
+
group by
|
|
93
|
+
day
|
|
94
|
+
order by
|
|
95
|
+
day asc
|
|
70
96
|
```
|
|
71
97
|
|
|
72
98
|
### Web Application
|
|
@@ -83,18 +109,20 @@ Laketower provides a CLI interface:
|
|
|
83
109
|
|
|
84
110
|
```bash
|
|
85
111
|
$ laketower --help
|
|
86
|
-
|
|
112
|
+
|
|
113
|
+
usage: laketower [-h] [--version] [--config CONFIG] {web,config,tables,queries} ...
|
|
87
114
|
|
|
88
115
|
options:
|
|
89
|
-
-h, --help
|
|
90
|
-
--version
|
|
91
|
-
--config, -c CONFIG
|
|
116
|
+
-h, --help show this help message and exit
|
|
117
|
+
--version show program's version number and exit
|
|
118
|
+
--config, -c CONFIG Path to the Laketower YAML configuration file (default: laketower.yml)
|
|
92
119
|
|
|
93
120
|
commands:
|
|
94
|
-
{web,config,tables}
|
|
95
|
-
web
|
|
96
|
-
config
|
|
97
|
-
tables
|
|
121
|
+
{web,config,tables,queries}
|
|
122
|
+
web Launch the web application
|
|
123
|
+
config Work with configuration
|
|
124
|
+
tables Work with tables
|
|
125
|
+
queries Work with queries
|
|
98
126
|
```
|
|
99
127
|
|
|
100
128
|
By default, a YAML configuration file named `laketower.yml` will be looked for.
|
|
@@ -207,6 +235,40 @@ weather
|
|
|
207
235
|
└── operation metrics
|
|
208
236
|
```
|
|
209
237
|
|
|
238
|
+
#### Get statistics of a given table
|
|
239
|
+
|
|
240
|
+
Get basic statistics on all columns of a given table:
|
|
241
|
+
|
|
242
|
+
```bash
|
|
243
|
+
$ laketower -c demo/laketower.yml tables statistics weather
|
|
244
|
+
|
|
245
|
+
┏━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━┳━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━┓
|
|
246
|
+
┃ column_name ┃ count ┃ avg ┃ std ┃ min ┃ max ┃
|
|
247
|
+
┡━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━╇━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━┩
|
|
248
|
+
│ time │ 576 │ None │ None │ 2025-01-26 01:00:00+01 │ 2025-02-12 00:00:00+01 │
|
|
249
|
+
│ city │ 576 │ None │ None │ Grenoble │ Grenoble │
|
|
250
|
+
│ temperature_2m │ 576 │ 5.2623263956047595 │ 3.326529069892729 │ 0.0 │ 15.1 │
|
|
251
|
+
│ relative_humidity_2m │ 576 │ 78.76909722222223 │ 15.701802163559918 │ 29.0 │ 100.0 │
|
|
252
|
+
│ wind_speed_10m │ 576 │ 7.535763886032833 │ 10.00898058743763 │ 0.0 │ 42.4 │
|
|
253
|
+
└──────────────────────┴───────┴────────────────────┴────────────────────┴────────────────────────┴────────────────────────┘
|
|
254
|
+
```
|
|
255
|
+
|
|
256
|
+
Specifying a table version yields according results:
|
|
257
|
+
|
|
258
|
+
```bash
|
|
259
|
+
$ laketower -c demo/laketower.yml tables statistics --version 0 weather
|
|
260
|
+
|
|
261
|
+
┏━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━┳━━━━━━┳━━━━━━┳━━━━━━┳━━━━━━┓
|
|
262
|
+
┃ column_name ┃ count ┃ avg ┃ std ┃ min ┃ max ┃
|
|
263
|
+
┡━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━╇━━━━━━╇━━━━━━╇━━━━━━╇━━━━━━┩
|
|
264
|
+
│ time │ 0 │ None │ None │ None │ None │
|
|
265
|
+
│ city │ 0 │ None │ None │ None │ None │
|
|
266
|
+
│ temperature_2m │ 0 │ None │ None │ None │ None │
|
|
267
|
+
│ relative_humidity_2m │ 0 │ None │ None │ None │ None │
|
|
268
|
+
│ wind_speed_10m │ 0 │ None │ None │ None │ None │
|
|
269
|
+
└──────────────────────┴───────┴──────┴──────┴──────┴──────┘
|
|
270
|
+
```
|
|
271
|
+
|
|
210
272
|
#### View a given table
|
|
211
273
|
|
|
212
274
|
Using a simple query builder, the content of a table can be displayed.
|
|
@@ -270,7 +332,6 @@ $ laketower -c demo/laketower.yml tables view weather --version 1
|
|
|
270
332
|
└───────────────────────────┴──────────┴───────────────────┴──────────────────────┴────────────────────┘
|
|
271
333
|
```
|
|
272
334
|
|
|
273
|
-
|
|
274
335
|
#### Query all registered tables
|
|
275
336
|
|
|
276
337
|
Query any registered tables using DuckDB SQL dialect!
|
|
@@ -287,6 +348,45 @@ $ laketower -c demo/laketower.yml tables query "select date_trunc('day', time) a
|
|
|
287
348
|
└───────────────────────────┴────────────────────┘
|
|
288
349
|
```
|
|
289
350
|
|
|
351
|
+
#### List saved queries
|
|
352
|
+
|
|
353
|
+
```bash
|
|
354
|
+
$ laketower -c demo/laketower.yml queries list
|
|
355
|
+
|
|
356
|
+
queries
|
|
357
|
+
├── all_data
|
|
358
|
+
└── daily_avg_temperature
|
|
359
|
+
```
|
|
360
|
+
|
|
361
|
+
#### Execute saved queries
|
|
362
|
+
|
|
363
|
+
```bash
|
|
364
|
+
$ laketower -c demo/laketower.yml queries view daily_avg_temperature
|
|
365
|
+
|
|
366
|
+
┏━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━┓
|
|
367
|
+
┃ day ┃ avg_temperature ┃
|
|
368
|
+
┡━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━┩
|
|
369
|
+
│ 2025-01-26 00:00:00+01:00 │ 8.0 │
|
|
370
|
+
│ 2025-01-27 00:00:00+01:00 │ 13.0 │
|
|
371
|
+
│ 2025-01-28 00:00:00+01:00 │ 7.0 │
|
|
372
|
+
│ 2025-01-29 00:00:00+01:00 │ 8.0 │
|
|
373
|
+
│ 2025-01-30 00:00:00+01:00 │ 9.0 │
|
|
374
|
+
│ 2025-01-31 00:00:00+01:00 │ 6.0 │
|
|
375
|
+
│ 2025-02-01 00:00:00+01:00 │ 4.0 │
|
|
376
|
+
│ 2025-02-02 00:00:00+01:00 │ 4.0 │
|
|
377
|
+
│ 2025-02-03 00:00:00+01:00 │ 4.0 │
|
|
378
|
+
│ 2025-02-04 00:00:00+01:00 │ 3.0 │
|
|
379
|
+
│ 2025-02-05 00:00:00+01:00 │ 3.0 │
|
|
380
|
+
│ 2025-02-06 00:00:00+01:00 │ 2.0 │
|
|
381
|
+
│ 2025-02-07 00:00:00+01:00 │ 6.0 │
|
|
382
|
+
│ 2025-02-08 00:00:00+01:00 │ 7.0 │
|
|
383
|
+
│ 2025-02-09 00:00:00+01:00 │ 5.0 │
|
|
384
|
+
│ 2025-02-10 00:00:00+01:00 │ 2.0 │
|
|
385
|
+
│ 2025-02-11 00:00:00+01:00 │ 5.0 │
|
|
386
|
+
│ 2025-02-12 00:00:00+01:00 │ 5.0 │
|
|
387
|
+
└───────────────────────────┴─────────────────┘
|
|
388
|
+
```
|
|
389
|
+
|
|
290
390
|
## License
|
|
291
391
|
|
|
292
392
|
Licensed under [GNU Affero General Public License v3.0 (AGPLv3)](LICENSE.md)
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
tables:
|
|
2
|
+
- name: sample_table
|
|
3
|
+
uri: demo/sample_table
|
|
4
|
+
format: delta
|
|
5
|
+
- name: weather
|
|
6
|
+
uri: demo/weather
|
|
7
|
+
format: delta
|
|
8
|
+
|
|
9
|
+
queries:
|
|
10
|
+
- name: all_data
|
|
11
|
+
title: All data
|
|
12
|
+
sql: |
|
|
13
|
+
select
|
|
14
|
+
sample_table.*,
|
|
15
|
+
weather.*
|
|
16
|
+
from
|
|
17
|
+
sample_table,
|
|
18
|
+
weather
|
|
19
|
+
limit 10
|
|
20
|
+
- name: daily_avg_temperature
|
|
21
|
+
title: Daily average temperature
|
|
22
|
+
sql: |
|
|
23
|
+
select
|
|
24
|
+
date_trunc('day', time) as day,
|
|
25
|
+
round(avg(temperature_2m)) as avg_temperature
|
|
26
|
+
from
|
|
27
|
+
weather
|
|
28
|
+
group by
|
|
29
|
+
day
|
|
30
|
+
order by
|
|
31
|
+
day asc
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
__version__ = "0.4.0"
|
|
@@ -1,5 +1,3 @@
|
|
|
1
|
-
from __future__ import annotations
|
|
2
|
-
|
|
3
1
|
import argparse
|
|
4
2
|
import os
|
|
5
3
|
from pathlib import Path
|
|
@@ -13,7 +11,12 @@ import uvicorn
|
|
|
13
11
|
|
|
14
12
|
from laketower.__about__ import __version__
|
|
15
13
|
from laketower.config import load_yaml_config
|
|
16
|
-
from laketower.tables import
|
|
14
|
+
from laketower.tables import (
|
|
15
|
+
execute_query,
|
|
16
|
+
generate_table_query,
|
|
17
|
+
generate_table_statistics_query,
|
|
18
|
+
load_table,
|
|
19
|
+
)
|
|
17
20
|
|
|
18
21
|
|
|
19
22
|
def run_web(config_path: Path, reload: bool) -> None: # pragma: no cover
|
|
@@ -99,6 +102,27 @@ def table_history(config_path: Path, table_name: str) -> None:
|
|
|
99
102
|
console.print(tree, markup=False)
|
|
100
103
|
|
|
101
104
|
|
|
105
|
+
def table_statistics(
|
|
106
|
+
config_path: Path, table_name: str, version: int | None = None
|
|
107
|
+
) -> None:
|
|
108
|
+
config = load_yaml_config(config_path)
|
|
109
|
+
table_config = next(filter(lambda x: x.name == table_name, config.tables))
|
|
110
|
+
table = load_table(table_config)
|
|
111
|
+
table_dataset = table.dataset(version=version)
|
|
112
|
+
sql_query = generate_table_statistics_query(table_name)
|
|
113
|
+
results = execute_query({table_name: table_dataset}, sql_query)
|
|
114
|
+
|
|
115
|
+
out = rich.table.Table()
|
|
116
|
+
for column in results.columns:
|
|
117
|
+
out.add_column(column)
|
|
118
|
+
for value_list in results.to_numpy().tolist():
|
|
119
|
+
row = [str(x) for x in value_list]
|
|
120
|
+
out.add_row(*row)
|
|
121
|
+
|
|
122
|
+
console = rich.get_console()
|
|
123
|
+
console.print(out, markup=False) # disable markup to allow bracket characters
|
|
124
|
+
|
|
125
|
+
|
|
102
126
|
def view_table(
|
|
103
127
|
config_path: Path,
|
|
104
128
|
table_name: str,
|
|
@@ -151,6 +175,40 @@ def query_table(config_path: Path, sql_query: str) -> None:
|
|
|
151
175
|
console.print(out)
|
|
152
176
|
|
|
153
177
|
|
|
178
|
+
def list_queries(config_path: Path) -> None:
|
|
179
|
+
config = load_yaml_config(config_path)
|
|
180
|
+
tree = rich.tree.Tree("queries")
|
|
181
|
+
for query in config.queries:
|
|
182
|
+
tree.add(query.name)
|
|
183
|
+
console = rich.get_console()
|
|
184
|
+
console.print(tree)
|
|
185
|
+
|
|
186
|
+
|
|
187
|
+
def view_query(config_path: Path, query_name: str) -> None:
|
|
188
|
+
config = load_yaml_config(config_path)
|
|
189
|
+
query_config = next(filter(lambda x: x.name == query_name, config.queries))
|
|
190
|
+
sql_query = query_config.sql
|
|
191
|
+
tables_dataset = {
|
|
192
|
+
table_config.name: load_table(table_config).dataset()
|
|
193
|
+
for table_config in config.tables
|
|
194
|
+
}
|
|
195
|
+
|
|
196
|
+
out: rich.jupyter.JupyterMixin
|
|
197
|
+
try:
|
|
198
|
+
results = execute_query(tables_dataset, sql_query)
|
|
199
|
+
out = rich.table.Table()
|
|
200
|
+
for column in results.columns:
|
|
201
|
+
out.add_column(column)
|
|
202
|
+
for value_list in results.values.tolist():
|
|
203
|
+
row = [str(x) for x in value_list]
|
|
204
|
+
out.add_row(*row)
|
|
205
|
+
except ValueError as e:
|
|
206
|
+
out = rich.panel.Panel.fit(f"[red]{e}")
|
|
207
|
+
|
|
208
|
+
console = rich.get_console()
|
|
209
|
+
console.print(out)
|
|
210
|
+
|
|
211
|
+
|
|
154
212
|
def cli() -> None:
|
|
155
213
|
parser = argparse.ArgumentParser(
|
|
156
214
|
"laketower", formatter_class=argparse.ArgumentDefaultsHelpFormatter
|
|
@@ -214,6 +272,17 @@ def cli() -> None:
|
|
|
214
272
|
parser_tables_history.add_argument("table", help="Name of the table")
|
|
215
273
|
parser_tables_history.set_defaults(func=lambda x: table_history(x.config, x.table))
|
|
216
274
|
|
|
275
|
+
parser_tables_statistics = subsparsers_tables.add_parser(
|
|
276
|
+
"statistics", help="Display summary statistics of a given table schema"
|
|
277
|
+
)
|
|
278
|
+
parser_tables_statistics.add_argument("table", help="Name of the table")
|
|
279
|
+
parser_tables_statistics.add_argument(
|
|
280
|
+
"--version", type=int, help="Time-travel to table revision number"
|
|
281
|
+
)
|
|
282
|
+
parser_tables_statistics.set_defaults(
|
|
283
|
+
func=lambda x: table_statistics(x.config, x.table, x.version)
|
|
284
|
+
)
|
|
285
|
+
|
|
217
286
|
parser_tables_view = subsparsers_tables.add_parser(
|
|
218
287
|
"view", help="View a given table"
|
|
219
288
|
)
|
|
@@ -244,5 +313,19 @@ def cli() -> None:
|
|
|
244
313
|
parser_tables_query.add_argument("sql", help="SQL query to execute")
|
|
245
314
|
parser_tables_query.set_defaults(func=lambda x: query_table(x.config, x.sql))
|
|
246
315
|
|
|
316
|
+
parser_queries = subparsers.add_parser("queries", help="Work with queries")
|
|
317
|
+
subsparsers_queries = parser_queries.add_subparsers(required=True)
|
|
318
|
+
|
|
319
|
+
parser_queries_list = subsparsers_queries.add_parser(
|
|
320
|
+
"list", help="List all registered queries"
|
|
321
|
+
)
|
|
322
|
+
parser_queries_list.set_defaults(func=lambda x: list_queries(x.config))
|
|
323
|
+
|
|
324
|
+
parser_queries_view = subsparsers_queries.add_parser(
|
|
325
|
+
"view", help="View a given query"
|
|
326
|
+
)
|
|
327
|
+
parser_queries_view.add_argument("query", help="Name of the query")
|
|
328
|
+
parser_queries_view.set_defaults(func=lambda x: view_query(x.config, x.query))
|
|
329
|
+
|
|
247
330
|
args = parser.parse_args()
|
|
248
331
|
args.func(args)
|
|
@@ -29,6 +29,7 @@ class ConfigTable(pydantic.BaseModel):
|
|
|
29
29
|
|
|
30
30
|
class ConfigQuery(pydantic.BaseModel):
|
|
31
31
|
name: str
|
|
32
|
+
title: str
|
|
32
33
|
sql: str
|
|
33
34
|
|
|
34
35
|
|
|
@@ -38,6 +39,7 @@ class ConfigDashboard(pydantic.BaseModel):
|
|
|
38
39
|
|
|
39
40
|
class Config(pydantic.BaseModel):
|
|
40
41
|
tables: list[ConfigTable] = []
|
|
42
|
+
queries: list[ConfigQuery] = []
|
|
41
43
|
|
|
42
44
|
|
|
43
45
|
def load_yaml_config(config_path: Path) -> Config:
|
|
@@ -1,7 +1,5 @@
|
|
|
1
|
-
from __future__ import annotations
|
|
2
|
-
|
|
3
1
|
from datetime import datetime, timezone
|
|
4
|
-
from typing import Any,
|
|
2
|
+
from typing import Any, Protocol
|
|
5
3
|
|
|
6
4
|
import deltalake
|
|
7
5
|
import duckdb
|
|
@@ -11,6 +9,7 @@ import pyarrow.dataset as padataset
|
|
|
11
9
|
import pydantic
|
|
12
10
|
import sqlglot
|
|
13
11
|
import sqlglot.dialects.duckdb
|
|
12
|
+
import sqlglot.expressions
|
|
14
13
|
|
|
15
14
|
from laketower.config import ConfigTable, TableFormats
|
|
16
15
|
|
|
@@ -20,8 +19,8 @@ DEFAULT_LIMIT = 10
|
|
|
20
19
|
|
|
21
20
|
class TableMetadata(pydantic.BaseModel):
|
|
22
21
|
table_format: TableFormats
|
|
23
|
-
name:
|
|
24
|
-
description:
|
|
22
|
+
name: str | None = None
|
|
23
|
+
description: str | None = None
|
|
25
24
|
uri: str
|
|
26
25
|
id: str
|
|
27
26
|
version: int
|
|
@@ -33,7 +32,7 @@ class TableMetadata(pydantic.BaseModel):
|
|
|
33
32
|
class TableRevision(pydantic.BaseModel):
|
|
34
33
|
version: int
|
|
35
34
|
timestamp: datetime
|
|
36
|
-
client_version:
|
|
35
|
+
client_version: str | None = None
|
|
37
36
|
operation: str
|
|
38
37
|
operation_parameters: dict[str, Any]
|
|
39
38
|
operation_metrics: dict[str, Any]
|
|
@@ -122,6 +121,12 @@ def generate_table_query(
|
|
|
122
121
|
)
|
|
123
122
|
|
|
124
123
|
|
|
124
|
+
def generate_table_statistics_query(table_name: str) -> str:
|
|
125
|
+
return (
|
|
126
|
+
f"SELECT column_name, count, avg, std, min, max FROM (SUMMARIZE {table_name})" # nosec B608
|
|
127
|
+
)
|
|
128
|
+
|
|
129
|
+
|
|
125
130
|
def execute_query(
|
|
126
131
|
tables_datasets: dict[str, padataset.Dataset], sql_query: str
|
|
127
132
|
) -> pd.DataFrame:
|
|
@@ -19,7 +19,14 @@
|
|
|
19
19
|
<a class="sidebar-brand" href="/">
|
|
20
20
|
Laketower
|
|
21
21
|
</a>
|
|
22
|
-
<button
|
|
22
|
+
<button
|
|
23
|
+
type="button"
|
|
24
|
+
class="btn-close d-md-none"
|
|
25
|
+
data-bs-dismiss="offcanvas"
|
|
26
|
+
aria-label="Close"
|
|
27
|
+
data-bs-target="#sidebar"
|
|
28
|
+
>
|
|
29
|
+
</button>
|
|
23
30
|
</div>
|
|
24
31
|
<div class="offcanvas-body">
|
|
25
32
|
<ul class="sidebar-nav">
|
|
@@ -30,12 +37,36 @@
|
|
|
30
37
|
{% for table in tables %}
|
|
31
38
|
{% set table_url = '/tables/' + table.name %}
|
|
32
39
|
<li class="nav-item">
|
|
33
|
-
<a
|
|
40
|
+
<a
|
|
41
|
+
class="text-truncate nav-link{% if request.url.path.startswith(table_url) %} active{% endif %}"
|
|
42
|
+
href="{{ table_url }}"
|
|
43
|
+
aria-current="page"
|
|
44
|
+
>
|
|
34
45
|
<i class="bi-table" aria-hidden="true"></i>
|
|
35
46
|
{{ table.name }}
|
|
36
47
|
</a>
|
|
37
48
|
</li>
|
|
38
49
|
{% endfor %}
|
|
50
|
+
|
|
51
|
+
<li><hr class="sidebar-divider"></li>
|
|
52
|
+
|
|
53
|
+
<li>
|
|
54
|
+
<h6 class="sidebar-header">Queries</h6>
|
|
55
|
+
</li>
|
|
56
|
+
<li><hr class="sidebar-divider"></li>
|
|
57
|
+
{% for query in queries %}
|
|
58
|
+
{% set query_url = '/queries/' + query.name + '/view' %}
|
|
59
|
+
<li class="nav-item">
|
|
60
|
+
<a
|
|
61
|
+
class="text-truncate nav-link{% if request.url.path.startswith(query_url) %} active{% endif %}"
|
|
62
|
+
href="{{ query_url }}"
|
|
63
|
+
aria-current="page"
|
|
64
|
+
>
|
|
65
|
+
<i class="bi-code-square" aria-hidden="true"></i>
|
|
66
|
+
{{ query.title }}
|
|
67
|
+
</a>
|
|
68
|
+
</li>
|
|
69
|
+
{% endfor %}
|
|
39
70
|
</ul>
|
|
40
71
|
</div>
|
|
41
72
|
</nav>
|