laketower 0.3.0__tar.gz → 0.4.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of laketower might be problematic. Click here for more details.
- {laketower-0.3.0 → laketower-0.4.1}/CHANGELOG.md +25 -1
- {laketower-0.3.0 → laketower-0.4.1}/PKG-INFO +110 -10
- {laketower-0.3.0 → laketower-0.4.1}/README.md +109 -9
- laketower-0.4.1/demo/laketower.yml +31 -0
- laketower-0.4.1/laketower/__about__.py +1 -0
- {laketower-0.3.0 → laketower-0.4.1}/laketower/cli.py +86 -1
- {laketower-0.3.0 → laketower-0.4.1}/laketower/config.py +2 -0
- {laketower-0.3.0 → laketower-0.4.1}/laketower/tables.py +7 -0
- {laketower-0.3.0 → laketower-0.4.1}/laketower/templates/_base.html +33 -2
- laketower-0.4.1/laketower/templates/queries/view.html +60 -0
- {laketower-0.3.0 → laketower-0.4.1}/laketower/templates/tables/_macros.html +3 -0
- {laketower-0.3.0 → laketower-0.4.1}/laketower/templates/tables/query.html +6 -2
- laketower-0.4.1/laketower/templates/tables/statistics.html +56 -0
- {laketower-0.3.0 → laketower-0.4.1}/laketower/templates/tables/view.html +1 -1
- {laketower-0.3.0 → laketower-0.4.1}/laketower/web.py +70 -1
- {laketower-0.3.0 → laketower-0.4.1}/pyproject.toml +3 -2
- {laketower-0.3.0 → laketower-0.4.1}/tests/conftest.py +8 -1
- {laketower-0.3.0 → laketower-0.4.1}/tests/test_cli.py +163 -3
- {laketower-0.3.0 → laketower-0.4.1}/tests/test_web.py +127 -13
- {laketower-0.3.0 → laketower-0.4.1}/uv.lock +50 -26
- laketower-0.3.0/demo/laketower.yml +0 -7
- laketower-0.3.0/laketower/__about__.py +0 -1
- {laketower-0.3.0 → laketower-0.4.1}/.github/workflows/ci-cd.yml +0 -0
- {laketower-0.3.0 → laketower-0.4.1}/.gitignore +0 -0
- {laketower-0.3.0 → laketower-0.4.1}/.python-version +0 -0
- {laketower-0.3.0 → laketower-0.4.1}/LICENSE.md +0 -0
- {laketower-0.3.0 → laketower-0.4.1}/demo/generate.py +0 -0
- {laketower-0.3.0 → laketower-0.4.1}/demo/sample_table/_delta_log/00000000000000000000.json +0 -0
- {laketower-0.3.0 → laketower-0.4.1}/demo/sample_table/_delta_log/00000000000000000001.json +0 -0
- {laketower-0.3.0 → laketower-0.4.1}/demo/sample_table/_delta_log/00000000000000000002.json +0 -0
- {laketower-0.3.0 → laketower-0.4.1}/demo/sample_table/_delta_log/00000000000000000003.json +0 -0
- {laketower-0.3.0 → laketower-0.4.1}/demo/sample_table/part-00001-1a31a393-6db6-4d1a-bf4e-81ea061ff8cd-c000.snappy.parquet +0 -0
- {laketower-0.3.0 → laketower-0.4.1}/demo/sample_table/part-00001-5af77102-9207-4c89-aaf6-37e1f815ec26-c000.snappy.parquet +0 -0
- {laketower-0.3.0 → laketower-0.4.1}/demo/sample_table/part-00001-b11bab55-43d0-4d05-ae88-5b9481ae57db-c000.snappy.parquet +0 -0
- {laketower-0.3.0 → laketower-0.4.1}/demo/weather/_delta_log/00000000000000000000.json +0 -0
- {laketower-0.3.0 → laketower-0.4.1}/demo/weather/_delta_log/00000000000000000001.json +0 -0
- {laketower-0.3.0 → laketower-0.4.1}/demo/weather/_delta_log/00000000000000000002.json +0 -0
- {laketower-0.3.0 → laketower-0.4.1}/demo/weather/part-00001-2323b963-be56-44e0-8c10-e237e7e6d4b9-c000.snappy.parquet +0 -0
- {laketower-0.3.0 → laketower-0.4.1}/demo/weather/part-00001-6360cbf8-f8a9-475f-8729-6f20b4ca64a9-c000.snappy.parquet +0 -0
- {laketower-0.3.0 → laketower-0.4.1}/laketower/__init__.py +0 -0
- {laketower-0.3.0 → laketower-0.4.1}/laketower/__main__.py +0 -0
- {laketower-0.3.0 → laketower-0.4.1}/laketower/static/.gitkeep +0 -0
- {laketower-0.3.0 → laketower-0.4.1}/laketower/templates/index.html +0 -0
- {laketower-0.3.0 → laketower-0.4.1}/laketower/templates/tables/history.html +0 -0
- {laketower-0.3.0 → laketower-0.4.1}/laketower/templates/tables/index.html +0 -0
- {laketower-0.3.0 → laketower-0.4.1}/renovate.json +0 -0
- {laketower-0.3.0 → laketower-0.4.1}/tasks.py +0 -0
- {laketower-0.3.0 → laketower-0.4.1}/tests/__init__.py +0 -0
|
@@ -7,6 +7,28 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
|
|
|
7
7
|
|
|
8
8
|
## [Unreleased]
|
|
9
9
|
|
|
10
|
+
## [0.4.1] - 2025-03-02
|
|
11
|
+
Minor release with fixes.
|
|
12
|
+
|
|
13
|
+
### Added
|
|
14
|
+
- web: allow editing queries
|
|
15
|
+
|
|
16
|
+
### Fixed
|
|
17
|
+
- web: missing tables query page title
|
|
18
|
+
- web: urlencode table view sql query link
|
|
19
|
+
|
|
20
|
+
## [0.4.0] - 2025-03-01
|
|
21
|
+
Introducing new features:
|
|
22
|
+
- Display tables statistics
|
|
23
|
+
- List and execute pre-defined queries
|
|
24
|
+
|
|
25
|
+
### Added
|
|
26
|
+
- web: add queries view page
|
|
27
|
+
- web: add table statistics page with version control
|
|
28
|
+
- cli: add queries view command
|
|
29
|
+
- cli: add queries list command
|
|
30
|
+
- cli: add table statistics command
|
|
31
|
+
|
|
10
32
|
## [0.3.0] - 2025-02-27
|
|
11
33
|
Minor release with fixes and dropped Python 3.9 support.
|
|
12
34
|
|
|
@@ -47,7 +69,9 @@ Initial release of `laketower`.
|
|
|
47
69
|
- View a given table with simple query builder
|
|
48
70
|
- Query all registered tables with DuckDB SQL dialect
|
|
49
71
|
|
|
50
|
-
[Unreleased]: https://github.com/datalpia/laketower/compare/0.
|
|
72
|
+
[Unreleased]: https://github.com/datalpia/laketower/compare/0.4.1...HEAD
|
|
73
|
+
[0.4.1]: https://github.com/datalpia/laketower/compare/0.4.0...0.4.1
|
|
74
|
+
[0.4.0]: https://github.com/datalpia/laketower/compare/0.3.0...0.4.0
|
|
51
75
|
[0.3.0]: https://github.com/datalpia/laketower/compare/0.2.0...0.3.0
|
|
52
76
|
[0.2.0]: https://github.com/datalpia/laketower/compare/0.1.0...0.2.0
|
|
53
77
|
[0.1.0]: https://github.com/datalpia/laketower/releases/tag/0.1.0
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: laketower
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.4.1
|
|
4
4
|
Summary: Oversee your lakehouse
|
|
5
5
|
Project-URL: Repository, https://github.com/datalpia/laketower
|
|
6
6
|
Project-URL: Issues, https://github.com/datalpia/laketower/issues
|
|
@@ -53,8 +53,10 @@ Utility application to explore and manage tables in your data lakehouse, especia
|
|
|
53
53
|
- Inspect table metadata
|
|
54
54
|
- Inspect table schema
|
|
55
55
|
- Inspect table history
|
|
56
|
+
- Get table statistics
|
|
56
57
|
- View table content with a simple query builder
|
|
57
58
|
- Query all registered tables with DuckDB SQL dialect
|
|
59
|
+
- Execute saved queries
|
|
58
60
|
- Static and versionable YAML configuration
|
|
59
61
|
- Web application
|
|
60
62
|
- CLI application
|
|
@@ -105,6 +107,30 @@ tables:
|
|
|
105
107
|
- name: weather
|
|
106
108
|
uri: demo/weather
|
|
107
109
|
format: delta
|
|
110
|
+
|
|
111
|
+
queries:
|
|
112
|
+
- name: all_data
|
|
113
|
+
title: All data
|
|
114
|
+
sql: |
|
|
115
|
+
select
|
|
116
|
+
sample_table.*,
|
|
117
|
+
weather.*
|
|
118
|
+
from
|
|
119
|
+
sample_table,
|
|
120
|
+
weather
|
|
121
|
+
limit 10
|
|
122
|
+
- name: daily_avg_temperature
|
|
123
|
+
title: Daily average temperature
|
|
124
|
+
sql: |
|
|
125
|
+
select
|
|
126
|
+
date_trunc('day', time) as day,
|
|
127
|
+
round(avg(temperature_2m)) as avg_temperature
|
|
128
|
+
from
|
|
129
|
+
weather
|
|
130
|
+
group by
|
|
131
|
+
day
|
|
132
|
+
order by
|
|
133
|
+
day asc
|
|
108
134
|
```
|
|
109
135
|
|
|
110
136
|
### Web Application
|
|
@@ -121,18 +147,20 @@ Laketower provides a CLI interface:
|
|
|
121
147
|
|
|
122
148
|
```bash
|
|
123
149
|
$ laketower --help
|
|
124
|
-
|
|
150
|
+
|
|
151
|
+
usage: laketower [-h] [--version] [--config CONFIG] {web,config,tables,queries} ...
|
|
125
152
|
|
|
126
153
|
options:
|
|
127
|
-
-h, --help
|
|
128
|
-
--version
|
|
129
|
-
--config, -c CONFIG
|
|
154
|
+
-h, --help show this help message and exit
|
|
155
|
+
--version show program's version number and exit
|
|
156
|
+
--config, -c CONFIG Path to the Laketower YAML configuration file (default: laketower.yml)
|
|
130
157
|
|
|
131
158
|
commands:
|
|
132
|
-
{web,config,tables}
|
|
133
|
-
web
|
|
134
|
-
config
|
|
135
|
-
tables
|
|
159
|
+
{web,config,tables,queries}
|
|
160
|
+
web Launch the web application
|
|
161
|
+
config Work with configuration
|
|
162
|
+
tables Work with tables
|
|
163
|
+
queries Work with queries
|
|
136
164
|
```
|
|
137
165
|
|
|
138
166
|
By default, a YAML configuration file named `laketower.yml` will be looked for.
|
|
@@ -245,6 +273,40 @@ weather
|
|
|
245
273
|
└── operation metrics
|
|
246
274
|
```
|
|
247
275
|
|
|
276
|
+
#### Get statistics of a given table
|
|
277
|
+
|
|
278
|
+
Get basic statistics on all columns of a given table:
|
|
279
|
+
|
|
280
|
+
```bash
|
|
281
|
+
$ laketower -c demo/laketower.yml tables statistics weather
|
|
282
|
+
|
|
283
|
+
┏━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━┳━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━┓
|
|
284
|
+
┃ column_name ┃ count ┃ avg ┃ std ┃ min ┃ max ┃
|
|
285
|
+
┡━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━╇━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━┩
|
|
286
|
+
│ time │ 576 │ None │ None │ 2025-01-26 01:00:00+01 │ 2025-02-12 00:00:00+01 │
|
|
287
|
+
│ city │ 576 │ None │ None │ Grenoble │ Grenoble │
|
|
288
|
+
│ temperature_2m │ 576 │ 5.2623263956047595 │ 3.326529069892729 │ 0.0 │ 15.1 │
|
|
289
|
+
│ relative_humidity_2m │ 576 │ 78.76909722222223 │ 15.701802163559918 │ 29.0 │ 100.0 │
|
|
290
|
+
│ wind_speed_10m │ 576 │ 7.535763886032833 │ 10.00898058743763 │ 0.0 │ 42.4 │
|
|
291
|
+
└──────────────────────┴───────┴────────────────────┴────────────────────┴────────────────────────┴────────────────────────┘
|
|
292
|
+
```
|
|
293
|
+
|
|
294
|
+
Specifying a table version yields according results:
|
|
295
|
+
|
|
296
|
+
```bash
|
|
297
|
+
$ laketower -c demo/laketower.yml tables statistics --version 0 weather
|
|
298
|
+
|
|
299
|
+
┏━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━┳━━━━━━┳━━━━━━┳━━━━━━┳━━━━━━┓
|
|
300
|
+
┃ column_name ┃ count ┃ avg ┃ std ┃ min ┃ max ┃
|
|
301
|
+
┡━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━╇━━━━━━╇━━━━━━╇━━━━━━╇━━━━━━┩
|
|
302
|
+
│ time │ 0 │ None │ None │ None │ None │
|
|
303
|
+
│ city │ 0 │ None │ None │ None │ None │
|
|
304
|
+
│ temperature_2m │ 0 │ None │ None │ None │ None │
|
|
305
|
+
│ relative_humidity_2m │ 0 │ None │ None │ None │ None │
|
|
306
|
+
│ wind_speed_10m │ 0 │ None │ None │ None │ None │
|
|
307
|
+
└──────────────────────┴───────┴──────┴──────┴──────┴──────┘
|
|
308
|
+
```
|
|
309
|
+
|
|
248
310
|
#### View a given table
|
|
249
311
|
|
|
250
312
|
Using a simple query builder, the content of a table can be displayed.
|
|
@@ -308,7 +370,6 @@ $ laketower -c demo/laketower.yml tables view weather --version 1
|
|
|
308
370
|
└───────────────────────────┴──────────┴───────────────────┴──────────────────────┴────────────────────┘
|
|
309
371
|
```
|
|
310
372
|
|
|
311
|
-
|
|
312
373
|
#### Query all registered tables
|
|
313
374
|
|
|
314
375
|
Query any registered tables using DuckDB SQL dialect!
|
|
@@ -325,6 +386,45 @@ $ laketower -c demo/laketower.yml tables query "select date_trunc('day', time) a
|
|
|
325
386
|
└───────────────────────────┴────────────────────┘
|
|
326
387
|
```
|
|
327
388
|
|
|
389
|
+
#### List saved queries
|
|
390
|
+
|
|
391
|
+
```bash
|
|
392
|
+
$ laketower -c demo/laketower.yml queries list
|
|
393
|
+
|
|
394
|
+
queries
|
|
395
|
+
├── all_data
|
|
396
|
+
└── daily_avg_temperature
|
|
397
|
+
```
|
|
398
|
+
|
|
399
|
+
#### Execute saved queries
|
|
400
|
+
|
|
401
|
+
```bash
|
|
402
|
+
$ laketower -c demo/laketower.yml queries view daily_avg_temperature
|
|
403
|
+
|
|
404
|
+
┏━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━┓
|
|
405
|
+
┃ day ┃ avg_temperature ┃
|
|
406
|
+
┡━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━┩
|
|
407
|
+
│ 2025-01-26 00:00:00+01:00 │ 8.0 │
|
|
408
|
+
│ 2025-01-27 00:00:00+01:00 │ 13.0 │
|
|
409
|
+
│ 2025-01-28 00:00:00+01:00 │ 7.0 │
|
|
410
|
+
│ 2025-01-29 00:00:00+01:00 │ 8.0 │
|
|
411
|
+
│ 2025-01-30 00:00:00+01:00 │ 9.0 │
|
|
412
|
+
│ 2025-01-31 00:00:00+01:00 │ 6.0 │
|
|
413
|
+
│ 2025-02-01 00:00:00+01:00 │ 4.0 │
|
|
414
|
+
│ 2025-02-02 00:00:00+01:00 │ 4.0 │
|
|
415
|
+
│ 2025-02-03 00:00:00+01:00 │ 4.0 │
|
|
416
|
+
│ 2025-02-04 00:00:00+01:00 │ 3.0 │
|
|
417
|
+
│ 2025-02-05 00:00:00+01:00 │ 3.0 │
|
|
418
|
+
│ 2025-02-06 00:00:00+01:00 │ 2.0 │
|
|
419
|
+
│ 2025-02-07 00:00:00+01:00 │ 6.0 │
|
|
420
|
+
│ 2025-02-08 00:00:00+01:00 │ 7.0 │
|
|
421
|
+
│ 2025-02-09 00:00:00+01:00 │ 5.0 │
|
|
422
|
+
│ 2025-02-10 00:00:00+01:00 │ 2.0 │
|
|
423
|
+
│ 2025-02-11 00:00:00+01:00 │ 5.0 │
|
|
424
|
+
│ 2025-02-12 00:00:00+01:00 │ 5.0 │
|
|
425
|
+
└───────────────────────────┴─────────────────┘
|
|
426
|
+
```
|
|
427
|
+
|
|
328
428
|
## License
|
|
329
429
|
|
|
330
430
|
Licensed under [GNU Affero General Public License v3.0 (AGPLv3)](LICENSE.md)
|
|
@@ -15,8 +15,10 @@ Utility application to explore and manage tables in your data lakehouse, especia
|
|
|
15
15
|
- Inspect table metadata
|
|
16
16
|
- Inspect table schema
|
|
17
17
|
- Inspect table history
|
|
18
|
+
- Get table statistics
|
|
18
19
|
- View table content with a simple query builder
|
|
19
20
|
- Query all registered tables with DuckDB SQL dialect
|
|
21
|
+
- Execute saved queries
|
|
20
22
|
- Static and versionable YAML configuration
|
|
21
23
|
- Web application
|
|
22
24
|
- CLI application
|
|
@@ -67,6 +69,30 @@ tables:
|
|
|
67
69
|
- name: weather
|
|
68
70
|
uri: demo/weather
|
|
69
71
|
format: delta
|
|
72
|
+
|
|
73
|
+
queries:
|
|
74
|
+
- name: all_data
|
|
75
|
+
title: All data
|
|
76
|
+
sql: |
|
|
77
|
+
select
|
|
78
|
+
sample_table.*,
|
|
79
|
+
weather.*
|
|
80
|
+
from
|
|
81
|
+
sample_table,
|
|
82
|
+
weather
|
|
83
|
+
limit 10
|
|
84
|
+
- name: daily_avg_temperature
|
|
85
|
+
title: Daily average temperature
|
|
86
|
+
sql: |
|
|
87
|
+
select
|
|
88
|
+
date_trunc('day', time) as day,
|
|
89
|
+
round(avg(temperature_2m)) as avg_temperature
|
|
90
|
+
from
|
|
91
|
+
weather
|
|
92
|
+
group by
|
|
93
|
+
day
|
|
94
|
+
order by
|
|
95
|
+
day asc
|
|
70
96
|
```
|
|
71
97
|
|
|
72
98
|
### Web Application
|
|
@@ -83,18 +109,20 @@ Laketower provides a CLI interface:
|
|
|
83
109
|
|
|
84
110
|
```bash
|
|
85
111
|
$ laketower --help
|
|
86
|
-
|
|
112
|
+
|
|
113
|
+
usage: laketower [-h] [--version] [--config CONFIG] {web,config,tables,queries} ...
|
|
87
114
|
|
|
88
115
|
options:
|
|
89
|
-
-h, --help
|
|
90
|
-
--version
|
|
91
|
-
--config, -c CONFIG
|
|
116
|
+
-h, --help show this help message and exit
|
|
117
|
+
--version show program's version number and exit
|
|
118
|
+
--config, -c CONFIG Path to the Laketower YAML configuration file (default: laketower.yml)
|
|
92
119
|
|
|
93
120
|
commands:
|
|
94
|
-
{web,config,tables}
|
|
95
|
-
web
|
|
96
|
-
config
|
|
97
|
-
tables
|
|
121
|
+
{web,config,tables,queries}
|
|
122
|
+
web Launch the web application
|
|
123
|
+
config Work with configuration
|
|
124
|
+
tables Work with tables
|
|
125
|
+
queries Work with queries
|
|
98
126
|
```
|
|
99
127
|
|
|
100
128
|
By default, a YAML configuration file named `laketower.yml` will be looked for.
|
|
@@ -207,6 +235,40 @@ weather
|
|
|
207
235
|
└── operation metrics
|
|
208
236
|
```
|
|
209
237
|
|
|
238
|
+
#### Get statistics of a given table
|
|
239
|
+
|
|
240
|
+
Get basic statistics on all columns of a given table:
|
|
241
|
+
|
|
242
|
+
```bash
|
|
243
|
+
$ laketower -c demo/laketower.yml tables statistics weather
|
|
244
|
+
|
|
245
|
+
┏━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━┳━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━┓
|
|
246
|
+
┃ column_name ┃ count ┃ avg ┃ std ┃ min ┃ max ┃
|
|
247
|
+
┡━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━╇━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━┩
|
|
248
|
+
│ time │ 576 │ None │ None │ 2025-01-26 01:00:00+01 │ 2025-02-12 00:00:00+01 │
|
|
249
|
+
│ city │ 576 │ None │ None │ Grenoble │ Grenoble │
|
|
250
|
+
│ temperature_2m │ 576 │ 5.2623263956047595 │ 3.326529069892729 │ 0.0 │ 15.1 │
|
|
251
|
+
│ relative_humidity_2m │ 576 │ 78.76909722222223 │ 15.701802163559918 │ 29.0 │ 100.0 │
|
|
252
|
+
│ wind_speed_10m │ 576 │ 7.535763886032833 │ 10.00898058743763 │ 0.0 │ 42.4 │
|
|
253
|
+
└──────────────────────┴───────┴────────────────────┴────────────────────┴────────────────────────┴────────────────────────┘
|
|
254
|
+
```
|
|
255
|
+
|
|
256
|
+
Specifying a table version yields according results:
|
|
257
|
+
|
|
258
|
+
```bash
|
|
259
|
+
$ laketower -c demo/laketower.yml tables statistics --version 0 weather
|
|
260
|
+
|
|
261
|
+
┏━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━┳━━━━━━┳━━━━━━┳━━━━━━┳━━━━━━┓
|
|
262
|
+
┃ column_name ┃ count ┃ avg ┃ std ┃ min ┃ max ┃
|
|
263
|
+
┡━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━╇━━━━━━╇━━━━━━╇━━━━━━╇━━━━━━┩
|
|
264
|
+
│ time │ 0 │ None │ None │ None │ None │
|
|
265
|
+
│ city │ 0 │ None │ None │ None │ None │
|
|
266
|
+
│ temperature_2m │ 0 │ None │ None │ None │ None │
|
|
267
|
+
│ relative_humidity_2m │ 0 │ None │ None │ None │ None │
|
|
268
|
+
│ wind_speed_10m │ 0 │ None │ None │ None │ None │
|
|
269
|
+
└──────────────────────┴───────┴──────┴──────┴──────┴──────┘
|
|
270
|
+
```
|
|
271
|
+
|
|
210
272
|
#### View a given table
|
|
211
273
|
|
|
212
274
|
Using a simple query builder, the content of a table can be displayed.
|
|
@@ -270,7 +332,6 @@ $ laketower -c demo/laketower.yml tables view weather --version 1
|
|
|
270
332
|
└───────────────────────────┴──────────┴───────────────────┴──────────────────────┴────────────────────┘
|
|
271
333
|
```
|
|
272
334
|
|
|
273
|
-
|
|
274
335
|
#### Query all registered tables
|
|
275
336
|
|
|
276
337
|
Query any registered tables using DuckDB SQL dialect!
|
|
@@ -287,6 +348,45 @@ $ laketower -c demo/laketower.yml tables query "select date_trunc('day', time) a
|
|
|
287
348
|
└───────────────────────────┴────────────────────┘
|
|
288
349
|
```
|
|
289
350
|
|
|
351
|
+
#### List saved queries
|
|
352
|
+
|
|
353
|
+
```bash
|
|
354
|
+
$ laketower -c demo/laketower.yml queries list
|
|
355
|
+
|
|
356
|
+
queries
|
|
357
|
+
├── all_data
|
|
358
|
+
└── daily_avg_temperature
|
|
359
|
+
```
|
|
360
|
+
|
|
361
|
+
#### Execute saved queries
|
|
362
|
+
|
|
363
|
+
```bash
|
|
364
|
+
$ laketower -c demo/laketower.yml queries view daily_avg_temperature
|
|
365
|
+
|
|
366
|
+
┏━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━┓
|
|
367
|
+
┃ day ┃ avg_temperature ┃
|
|
368
|
+
┡━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━┩
|
|
369
|
+
│ 2025-01-26 00:00:00+01:00 │ 8.0 │
|
|
370
|
+
│ 2025-01-27 00:00:00+01:00 │ 13.0 │
|
|
371
|
+
│ 2025-01-28 00:00:00+01:00 │ 7.0 │
|
|
372
|
+
│ 2025-01-29 00:00:00+01:00 │ 8.0 │
|
|
373
|
+
│ 2025-01-30 00:00:00+01:00 │ 9.0 │
|
|
374
|
+
│ 2025-01-31 00:00:00+01:00 │ 6.0 │
|
|
375
|
+
│ 2025-02-01 00:00:00+01:00 │ 4.0 │
|
|
376
|
+
│ 2025-02-02 00:00:00+01:00 │ 4.0 │
|
|
377
|
+
│ 2025-02-03 00:00:00+01:00 │ 4.0 │
|
|
378
|
+
│ 2025-02-04 00:00:00+01:00 │ 3.0 │
|
|
379
|
+
│ 2025-02-05 00:00:00+01:00 │ 3.0 │
|
|
380
|
+
│ 2025-02-06 00:00:00+01:00 │ 2.0 │
|
|
381
|
+
│ 2025-02-07 00:00:00+01:00 │ 6.0 │
|
|
382
|
+
│ 2025-02-08 00:00:00+01:00 │ 7.0 │
|
|
383
|
+
│ 2025-02-09 00:00:00+01:00 │ 5.0 │
|
|
384
|
+
│ 2025-02-10 00:00:00+01:00 │ 2.0 │
|
|
385
|
+
│ 2025-02-11 00:00:00+01:00 │ 5.0 │
|
|
386
|
+
│ 2025-02-12 00:00:00+01:00 │ 5.0 │
|
|
387
|
+
└───────────────────────────┴─────────────────┘
|
|
388
|
+
```
|
|
389
|
+
|
|
290
390
|
## License
|
|
291
391
|
|
|
292
392
|
Licensed under [GNU Affero General Public License v3.0 (AGPLv3)](LICENSE.md)
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
tables:
|
|
2
|
+
- name: sample_table
|
|
3
|
+
uri: demo/sample_table
|
|
4
|
+
format: delta
|
|
5
|
+
- name: weather
|
|
6
|
+
uri: demo/weather
|
|
7
|
+
format: delta
|
|
8
|
+
|
|
9
|
+
queries:
|
|
10
|
+
- name: all_data
|
|
11
|
+
title: All data
|
|
12
|
+
sql: |
|
|
13
|
+
select
|
|
14
|
+
sample_table.*,
|
|
15
|
+
weather.*
|
|
16
|
+
from
|
|
17
|
+
sample_table,
|
|
18
|
+
weather
|
|
19
|
+
limit 10
|
|
20
|
+
- name: daily_avg_temperature
|
|
21
|
+
title: Daily average temperature
|
|
22
|
+
sql: |
|
|
23
|
+
select
|
|
24
|
+
date_trunc('day', time) as day,
|
|
25
|
+
round(avg(temperature_2m)) as avg_temperature
|
|
26
|
+
from
|
|
27
|
+
weather
|
|
28
|
+
group by
|
|
29
|
+
day
|
|
30
|
+
order by
|
|
31
|
+
day asc
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
__version__ = "0.4.1"
|
|
@@ -11,7 +11,12 @@ import uvicorn
|
|
|
11
11
|
|
|
12
12
|
from laketower.__about__ import __version__
|
|
13
13
|
from laketower.config import load_yaml_config
|
|
14
|
-
from laketower.tables import
|
|
14
|
+
from laketower.tables import (
|
|
15
|
+
execute_query,
|
|
16
|
+
generate_table_query,
|
|
17
|
+
generate_table_statistics_query,
|
|
18
|
+
load_table,
|
|
19
|
+
)
|
|
15
20
|
|
|
16
21
|
|
|
17
22
|
def run_web(config_path: Path, reload: bool) -> None: # pragma: no cover
|
|
@@ -97,6 +102,27 @@ def table_history(config_path: Path, table_name: str) -> None:
|
|
|
97
102
|
console.print(tree, markup=False)
|
|
98
103
|
|
|
99
104
|
|
|
105
|
+
def table_statistics(
|
|
106
|
+
config_path: Path, table_name: str, version: int | None = None
|
|
107
|
+
) -> None:
|
|
108
|
+
config = load_yaml_config(config_path)
|
|
109
|
+
table_config = next(filter(lambda x: x.name == table_name, config.tables))
|
|
110
|
+
table = load_table(table_config)
|
|
111
|
+
table_dataset = table.dataset(version=version)
|
|
112
|
+
sql_query = generate_table_statistics_query(table_name)
|
|
113
|
+
results = execute_query({table_name: table_dataset}, sql_query)
|
|
114
|
+
|
|
115
|
+
out = rich.table.Table()
|
|
116
|
+
for column in results.columns:
|
|
117
|
+
out.add_column(column)
|
|
118
|
+
for value_list in results.to_numpy().tolist():
|
|
119
|
+
row = [str(x) for x in value_list]
|
|
120
|
+
out.add_row(*row)
|
|
121
|
+
|
|
122
|
+
console = rich.get_console()
|
|
123
|
+
console.print(out, markup=False) # disable markup to allow bracket characters
|
|
124
|
+
|
|
125
|
+
|
|
100
126
|
def view_table(
|
|
101
127
|
config_path: Path,
|
|
102
128
|
table_name: str,
|
|
@@ -149,6 +175,40 @@ def query_table(config_path: Path, sql_query: str) -> None:
|
|
|
149
175
|
console.print(out)
|
|
150
176
|
|
|
151
177
|
|
|
178
|
+
def list_queries(config_path: Path) -> None:
|
|
179
|
+
config = load_yaml_config(config_path)
|
|
180
|
+
tree = rich.tree.Tree("queries")
|
|
181
|
+
for query in config.queries:
|
|
182
|
+
tree.add(query.name)
|
|
183
|
+
console = rich.get_console()
|
|
184
|
+
console.print(tree)
|
|
185
|
+
|
|
186
|
+
|
|
187
|
+
def view_query(config_path: Path, query_name: str) -> None:
|
|
188
|
+
config = load_yaml_config(config_path)
|
|
189
|
+
query_config = next(filter(lambda x: x.name == query_name, config.queries))
|
|
190
|
+
sql_query = query_config.sql
|
|
191
|
+
tables_dataset = {
|
|
192
|
+
table_config.name: load_table(table_config).dataset()
|
|
193
|
+
for table_config in config.tables
|
|
194
|
+
}
|
|
195
|
+
|
|
196
|
+
out: rich.jupyter.JupyterMixin
|
|
197
|
+
try:
|
|
198
|
+
results = execute_query(tables_dataset, sql_query)
|
|
199
|
+
out = rich.table.Table()
|
|
200
|
+
for column in results.columns:
|
|
201
|
+
out.add_column(column)
|
|
202
|
+
for value_list in results.values.tolist():
|
|
203
|
+
row = [str(x) for x in value_list]
|
|
204
|
+
out.add_row(*row)
|
|
205
|
+
except ValueError as e:
|
|
206
|
+
out = rich.panel.Panel.fit(f"[red]{e}")
|
|
207
|
+
|
|
208
|
+
console = rich.get_console()
|
|
209
|
+
console.print(out)
|
|
210
|
+
|
|
211
|
+
|
|
152
212
|
def cli() -> None:
|
|
153
213
|
parser = argparse.ArgumentParser(
|
|
154
214
|
"laketower", formatter_class=argparse.ArgumentDefaultsHelpFormatter
|
|
@@ -212,6 +272,17 @@ def cli() -> None:
|
|
|
212
272
|
parser_tables_history.add_argument("table", help="Name of the table")
|
|
213
273
|
parser_tables_history.set_defaults(func=lambda x: table_history(x.config, x.table))
|
|
214
274
|
|
|
275
|
+
parser_tables_statistics = subsparsers_tables.add_parser(
|
|
276
|
+
"statistics", help="Display summary statistics of a given table schema"
|
|
277
|
+
)
|
|
278
|
+
parser_tables_statistics.add_argument("table", help="Name of the table")
|
|
279
|
+
parser_tables_statistics.add_argument(
|
|
280
|
+
"--version", type=int, help="Time-travel to table revision number"
|
|
281
|
+
)
|
|
282
|
+
parser_tables_statistics.set_defaults(
|
|
283
|
+
func=lambda x: table_statistics(x.config, x.table, x.version)
|
|
284
|
+
)
|
|
285
|
+
|
|
215
286
|
parser_tables_view = subsparsers_tables.add_parser(
|
|
216
287
|
"view", help="View a given table"
|
|
217
288
|
)
|
|
@@ -242,5 +313,19 @@ def cli() -> None:
|
|
|
242
313
|
parser_tables_query.add_argument("sql", help="SQL query to execute")
|
|
243
314
|
parser_tables_query.set_defaults(func=lambda x: query_table(x.config, x.sql))
|
|
244
315
|
|
|
316
|
+
parser_queries = subparsers.add_parser("queries", help="Work with queries")
|
|
317
|
+
subsparsers_queries = parser_queries.add_subparsers(required=True)
|
|
318
|
+
|
|
319
|
+
parser_queries_list = subsparsers_queries.add_parser(
|
|
320
|
+
"list", help="List all registered queries"
|
|
321
|
+
)
|
|
322
|
+
parser_queries_list.set_defaults(func=lambda x: list_queries(x.config))
|
|
323
|
+
|
|
324
|
+
parser_queries_view = subsparsers_queries.add_parser(
|
|
325
|
+
"view", help="View a given query"
|
|
326
|
+
)
|
|
327
|
+
parser_queries_view.add_argument("query", help="Name of the query")
|
|
328
|
+
parser_queries_view.set_defaults(func=lambda x: view_query(x.config, x.query))
|
|
329
|
+
|
|
245
330
|
args = parser.parse_args()
|
|
246
331
|
args.func(args)
|
|
@@ -29,6 +29,7 @@ class ConfigTable(pydantic.BaseModel):
|
|
|
29
29
|
|
|
30
30
|
class ConfigQuery(pydantic.BaseModel):
|
|
31
31
|
name: str
|
|
32
|
+
title: str
|
|
32
33
|
sql: str
|
|
33
34
|
|
|
34
35
|
|
|
@@ -38,6 +39,7 @@ class ConfigDashboard(pydantic.BaseModel):
|
|
|
38
39
|
|
|
39
40
|
class Config(pydantic.BaseModel):
|
|
40
41
|
tables: list[ConfigTable] = []
|
|
42
|
+
queries: list[ConfigQuery] = []
|
|
41
43
|
|
|
42
44
|
|
|
43
45
|
def load_yaml_config(config_path: Path) -> Config:
|
|
@@ -9,6 +9,7 @@ import pyarrow.dataset as padataset
|
|
|
9
9
|
import pydantic
|
|
10
10
|
import sqlglot
|
|
11
11
|
import sqlglot.dialects.duckdb
|
|
12
|
+
import sqlglot.expressions
|
|
12
13
|
|
|
13
14
|
from laketower.config import ConfigTable, TableFormats
|
|
14
15
|
|
|
@@ -120,6 +121,12 @@ def generate_table_query(
|
|
|
120
121
|
)
|
|
121
122
|
|
|
122
123
|
|
|
124
|
+
def generate_table_statistics_query(table_name: str) -> str:
|
|
125
|
+
return (
|
|
126
|
+
f"SELECT column_name, count, avg, std, min, max FROM (SUMMARIZE {table_name})" # nosec B608
|
|
127
|
+
)
|
|
128
|
+
|
|
129
|
+
|
|
123
130
|
def execute_query(
|
|
124
131
|
tables_datasets: dict[str, padataset.Dataset], sql_query: str
|
|
125
132
|
) -> pd.DataFrame:
|
|
@@ -19,7 +19,14 @@
|
|
|
19
19
|
<a class="sidebar-brand" href="/">
|
|
20
20
|
Laketower
|
|
21
21
|
</a>
|
|
22
|
-
<button
|
|
22
|
+
<button
|
|
23
|
+
type="button"
|
|
24
|
+
class="btn-close d-md-none"
|
|
25
|
+
data-bs-dismiss="offcanvas"
|
|
26
|
+
aria-label="Close"
|
|
27
|
+
data-bs-target="#sidebar"
|
|
28
|
+
>
|
|
29
|
+
</button>
|
|
23
30
|
</div>
|
|
24
31
|
<div class="offcanvas-body">
|
|
25
32
|
<ul class="sidebar-nav">
|
|
@@ -30,12 +37,36 @@
|
|
|
30
37
|
{% for table in tables %}
|
|
31
38
|
{% set table_url = '/tables/' + table.name %}
|
|
32
39
|
<li class="nav-item">
|
|
33
|
-
<a
|
|
40
|
+
<a
|
|
41
|
+
class="text-truncate nav-link{% if request.url.path.startswith(table_url) %} active{% endif %}"
|
|
42
|
+
href="{{ table_url }}"
|
|
43
|
+
aria-current="page"
|
|
44
|
+
>
|
|
34
45
|
<i class="bi-table" aria-hidden="true"></i>
|
|
35
46
|
{{ table.name }}
|
|
36
47
|
</a>
|
|
37
48
|
</li>
|
|
38
49
|
{% endfor %}
|
|
50
|
+
|
|
51
|
+
<li><hr class="sidebar-divider"></li>
|
|
52
|
+
|
|
53
|
+
<li>
|
|
54
|
+
<h6 class="sidebar-header">Queries</h6>
|
|
55
|
+
</li>
|
|
56
|
+
<li><hr class="sidebar-divider"></li>
|
|
57
|
+
{% for query in queries %}
|
|
58
|
+
{% set query_url = '/queries/' + query.name + '/view' %}
|
|
59
|
+
<li class="nav-item">
|
|
60
|
+
<a
|
|
61
|
+
class="text-truncate nav-link{% if request.url.path.startswith(query_url) %} active{% endif %}"
|
|
62
|
+
href="{{ query_url }}"
|
|
63
|
+
aria-current="page"
|
|
64
|
+
>
|
|
65
|
+
<i class="bi-code-square" aria-hidden="true"></i>
|
|
66
|
+
{{ query.title }}
|
|
67
|
+
</a>
|
|
68
|
+
</li>
|
|
69
|
+
{% endfor %}
|
|
39
70
|
</ul>
|
|
40
71
|
</div>
|
|
41
72
|
</nav>
|
|
@@ -0,0 +1,60 @@
|
|
|
1
|
+
{% extends "_base.html" %}
|
|
2
|
+
|
|
3
|
+
{% block body %}
|
|
4
|
+
<div class="row">
|
|
5
|
+
<div class="col">
|
|
6
|
+
<h2 class="mb-3">{{ query.title }}</h2>
|
|
7
|
+
|
|
8
|
+
<form action="{{ request.url.path }}" method="get">
|
|
9
|
+
<div class="mb-3">
|
|
10
|
+
<textarea disabled name="sql" rows="5" class="form-control">{{ query.sql }}</textarea>
|
|
11
|
+
</div>
|
|
12
|
+
|
|
13
|
+
<div class="mb-3">
|
|
14
|
+
<div class="d-flex justify-content-end">
|
|
15
|
+
<div class="row">
|
|
16
|
+
<div class="col">
|
|
17
|
+
<a href="/tables/query?sql={{ query.sql | urlencode }}" class="btn btn-secondary" type="button" >
|
|
18
|
+
<i class="bi-code" aria-hidden="true"></i> Edit SQL
|
|
19
|
+
</a>
|
|
20
|
+
</div>
|
|
21
|
+
|
|
22
|
+
<div class="col-auto">
|
|
23
|
+
<button type="submit" class="btn btn-primary">
|
|
24
|
+
<i class="bi-lightning" aria-hidden="true"></i> Execute
|
|
25
|
+
</button>
|
|
26
|
+
</div>
|
|
27
|
+
</div>
|
|
28
|
+
</div>
|
|
29
|
+
</div>
|
|
30
|
+
</form>
|
|
31
|
+
|
|
32
|
+
{% if error %}
|
|
33
|
+
<div class="alert alert-danger" role="alert">
|
|
34
|
+
{{ error.message }}
|
|
35
|
+
</div>
|
|
36
|
+
{% else %}
|
|
37
|
+
<div class="table-responsive">
|
|
38
|
+
<table class="table table-sm table-bordered table-striped table-hover">
|
|
39
|
+
<thead>
|
|
40
|
+
<tr>
|
|
41
|
+
{% for column in query_results.columns %}
|
|
42
|
+
<th>{{ column }}</th>
|
|
43
|
+
{% endfor %}
|
|
44
|
+
</tr>
|
|
45
|
+
</thead>
|
|
46
|
+
<tbody class="table-group-divider">
|
|
47
|
+
{% for row in query_results.to_numpy().tolist() %}
|
|
48
|
+
<tr>
|
|
49
|
+
{% for col in row %}
|
|
50
|
+
<td>{{ col }}</td>
|
|
51
|
+
{% endfor %}
|
|
52
|
+
</tr>
|
|
53
|
+
{% endfor %}
|
|
54
|
+
</tbody>
|
|
55
|
+
</table>
|
|
56
|
+
</div>
|
|
57
|
+
{% endif %}
|
|
58
|
+
</div>
|
|
59
|
+
</div>
|
|
60
|
+
{% endblock %}
|