laketower 0.2.0__tar.gz → 0.4.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of laketower might be problematic. Click here for more details.

Files changed (48) hide show
  1. {laketower-0.2.0 → laketower-0.4.0}/.github/workflows/ci-cd.yml +1 -1
  2. {laketower-0.2.0 → laketower-0.4.0}/CHANGELOG.md +25 -1
  3. {laketower-0.2.0 → laketower-0.4.0}/PKG-INFO +111 -12
  4. {laketower-0.2.0 → laketower-0.4.0}/README.md +109 -9
  5. laketower-0.4.0/demo/laketower.yml +31 -0
  6. laketower-0.4.0/laketower/__about__.py +1 -0
  7. {laketower-0.2.0 → laketower-0.4.0}/laketower/cli.py +86 -3
  8. {laketower-0.2.0 → laketower-0.4.0}/laketower/config.py +2 -0
  9. {laketower-0.2.0 → laketower-0.4.0}/laketower/tables.py +11 -6
  10. {laketower-0.2.0 → laketower-0.4.0}/laketower/templates/_base.html +33 -2
  11. laketower-0.4.0/laketower/templates/queries/view.html +48 -0
  12. {laketower-0.2.0 → laketower-0.4.0}/laketower/templates/tables/_macros.html +3 -0
  13. {laketower-0.2.0 → laketower-0.4.0}/laketower/templates/tables/query.html +7 -1
  14. laketower-0.4.0/laketower/templates/tables/statistics.html +56 -0
  15. {laketower-0.2.0 → laketower-0.4.0}/laketower/web.py +84 -8
  16. {laketower-0.2.0 → laketower-0.4.0}/pyproject.toml +14 -14
  17. laketower-0.4.0/renovate.json +11 -0
  18. {laketower-0.2.0 → laketower-0.4.0}/tests/conftest.py +8 -1
  19. {laketower-0.2.0 → laketower-0.4.0}/tests/test_cli.py +163 -3
  20. {laketower-0.2.0 → laketower-0.4.0}/tests/test_web.py +122 -5
  21. {laketower-0.2.0 → laketower-0.4.0}/uv.lock +110 -292
  22. laketower-0.2.0/demo/laketower.yml +0 -7
  23. laketower-0.2.0/laketower/__about__.py +0 -1
  24. {laketower-0.2.0 → laketower-0.4.0}/.gitignore +0 -0
  25. {laketower-0.2.0 → laketower-0.4.0}/.python-version +0 -0
  26. {laketower-0.2.0 → laketower-0.4.0}/LICENSE.md +0 -0
  27. {laketower-0.2.0 → laketower-0.4.0}/demo/generate.py +0 -0
  28. {laketower-0.2.0 → laketower-0.4.0}/demo/sample_table/_delta_log/00000000000000000000.json +0 -0
  29. {laketower-0.2.0 → laketower-0.4.0}/demo/sample_table/_delta_log/00000000000000000001.json +0 -0
  30. {laketower-0.2.0 → laketower-0.4.0}/demo/sample_table/_delta_log/00000000000000000002.json +0 -0
  31. {laketower-0.2.0 → laketower-0.4.0}/demo/sample_table/_delta_log/00000000000000000003.json +0 -0
  32. {laketower-0.2.0 → laketower-0.4.0}/demo/sample_table/part-00001-1a31a393-6db6-4d1a-bf4e-81ea061ff8cd-c000.snappy.parquet +0 -0
  33. {laketower-0.2.0 → laketower-0.4.0}/demo/sample_table/part-00001-5af77102-9207-4c89-aaf6-37e1f815ec26-c000.snappy.parquet +0 -0
  34. {laketower-0.2.0 → laketower-0.4.0}/demo/sample_table/part-00001-b11bab55-43d0-4d05-ae88-5b9481ae57db-c000.snappy.parquet +0 -0
  35. {laketower-0.2.0 → laketower-0.4.0}/demo/weather/_delta_log/00000000000000000000.json +0 -0
  36. {laketower-0.2.0 → laketower-0.4.0}/demo/weather/_delta_log/00000000000000000001.json +0 -0
  37. {laketower-0.2.0 → laketower-0.4.0}/demo/weather/_delta_log/00000000000000000002.json +0 -0
  38. {laketower-0.2.0 → laketower-0.4.0}/demo/weather/part-00001-2323b963-be56-44e0-8c10-e237e7e6d4b9-c000.snappy.parquet +0 -0
  39. {laketower-0.2.0 → laketower-0.4.0}/demo/weather/part-00001-6360cbf8-f8a9-475f-8729-6f20b4ca64a9-c000.snappy.parquet +0 -0
  40. {laketower-0.2.0 → laketower-0.4.0}/laketower/__init__.py +0 -0
  41. {laketower-0.2.0 → laketower-0.4.0}/laketower/__main__.py +0 -0
  42. {laketower-0.2.0 → laketower-0.4.0}/laketower/static/.gitkeep +0 -0
  43. {laketower-0.2.0 → laketower-0.4.0}/laketower/templates/index.html +0 -0
  44. {laketower-0.2.0 → laketower-0.4.0}/laketower/templates/tables/history.html +0 -0
  45. {laketower-0.2.0 → laketower-0.4.0}/laketower/templates/tables/index.html +0 -0
  46. {laketower-0.2.0 → laketower-0.4.0}/laketower/templates/tables/view.html +0 -0
  47. {laketower-0.2.0 → laketower-0.4.0}/tasks.py +0 -0
  48. {laketower-0.2.0 → laketower-0.4.0}/tests/__init__.py +0 -0
@@ -14,7 +14,7 @@ jobs:
14
14
  runs-on: ubuntu-latest
15
15
  strategy:
16
16
  matrix:
17
- python-version: ["3.9", "3.10", "3.11", "3.12", "3.13"]
17
+ python-version: ["3.10", "3.11", "3.12", "3.13"]
18
18
  steps:
19
19
  - uses: actions/checkout@v4
20
20
  - name: Install uv
@@ -7,6 +7,28 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
7
7
 
8
8
  ## [Unreleased]
9
9
 
10
+ ## [0.4.0] - 2025-03-01
11
+ Introducing new features:
12
+ - Display tables statistics
13
+ - List and execute pre-defined queries
14
+
15
+ ### Added
16
+ - web: add queries view page
17
+ - web: add table statistics page with version control
18
+ - cli: add queries view command
19
+ - cli: add queries list command
20
+ - cli: add table statistics command
21
+
22
+ ## [0.3.0] - 2025-02-27
23
+ Minor release with fixes and dropped Python 3.9 support.
24
+
25
+ ### BREAKING CHANGES
26
+ - deps: drop support for python 3.9
27
+
28
+ ### Fixed
29
+ - web: handle invalid tables sql query
30
+ - web: truncate long table names in sidebar
31
+
10
32
  ## [0.2.0] - 2025-02-25
11
33
  Introducing the Laketower web application!
12
34
 
@@ -37,6 +59,8 @@ Initial release of `laketower`.
37
59
  - View a given table with simple query builder
38
60
  - Query all registered tables with DuckDB SQL dialect
39
61
 
40
- [Unreleased]: https://github.com/datalpia/laketower/compare/0.2.0...HEAD
62
+ [Unreleased]: https://github.com/datalpia/laketower/compare/0.4.0...HEAD
63
+ [0.4.0]: https://github.com/datalpia/laketower/compare/0.3.0...0.4.0
64
+ [0.3.0]: https://github.com/datalpia/laketower/compare/0.2.0...0.3.0
41
65
  [0.2.0]: https://github.com/datalpia/laketower/compare/0.1.0...0.2.0
42
66
  [0.1.0]: https://github.com/datalpia/laketower/releases/tag/0.1.0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: laketower
3
- Version: 0.2.0
3
+ Version: 0.4.0
4
4
  Summary: Oversee your lakehouse
5
5
  Project-URL: Repository, https://github.com/datalpia/laketower
6
6
  Project-URL: Issues, https://github.com/datalpia/laketower/issues
@@ -14,7 +14,6 @@ Classifier: Intended Audience :: Developers
14
14
  Classifier: Intended Audience :: End Users/Desktop
15
15
  Classifier: Intended Audience :: Information Technology
16
16
  Classifier: Intended Audience :: Other Audience
17
- Classifier: Programming Language :: Python :: 3.9
18
17
  Classifier: Programming Language :: Python :: 3.10
19
18
  Classifier: Programming Language :: Python :: 3.11
20
19
  Classifier: Programming Language :: Python :: 3.12
@@ -22,7 +21,7 @@ Classifier: Programming Language :: Python :: 3.13
22
21
  Classifier: Topic :: Database
23
22
  Classifier: Topic :: Software Development
24
23
  Classifier: Topic :: Utilities
25
- Requires-Python: <3.14,>=3.9
24
+ Requires-Python: <3.14,>=3.10
26
25
  Requires-Dist: deltalake
27
26
  Requires-Dist: duckdb
28
27
  Requires-Dist: fastapi
@@ -54,8 +53,10 @@ Utility application to explore and manage tables in your data lakehouse, especia
54
53
  - Inspect table metadata
55
54
  - Inspect table schema
56
55
  - Inspect table history
56
+ - Get table statistics
57
57
  - View table content with a simple query builder
58
58
  - Query all registered tables with DuckDB SQL dialect
59
+ - Execute saved queries
59
60
  - Static and versionable YAML configuration
60
61
  - Web application
61
62
  - CLI application
@@ -106,6 +107,30 @@ tables:
106
107
  - name: weather
107
108
  uri: demo/weather
108
109
  format: delta
110
+
111
+ queries:
112
+ - name: all_data
113
+ title: All data
114
+ sql: |
115
+ select
116
+ sample_table.*,
117
+ weather.*
118
+ from
119
+ sample_table,
120
+ weather
121
+ limit 10
122
+ - name: daily_avg_temperature
123
+ title: Daily average temperature
124
+ sql: |
125
+ select
126
+ date_trunc('day', time) as day,
127
+ round(avg(temperature_2m)) as avg_temperature
128
+ from
129
+ weather
130
+ group by
131
+ day
132
+ order by
133
+ day asc
109
134
  ```
110
135
 
111
136
  ### Web Application
@@ -122,18 +147,20 @@ Laketower provides a CLI interface:
122
147
 
123
148
  ```bash
124
149
  $ laketower --help
125
- usage: laketower [-h] [--version] [--config CONFIG] {web,config,tables} ...
150
+
151
+ usage: laketower [-h] [--version] [--config CONFIG] {web,config,tables,queries} ...
126
152
 
127
153
  options:
128
- -h, --help show this help message and exit
129
- --version show program's version number and exit
130
- --config, -c CONFIG Path to the Laketower YAML configuration file (default: laketower.yml)
154
+ -h, --help show this help message and exit
155
+ --version show program's version number and exit
156
+ --config, -c CONFIG Path to the Laketower YAML configuration file (default: laketower.yml)
131
157
 
132
158
  commands:
133
- {web,config,tables}
134
- web Launch the web application
135
- config Work with configuration
136
- tables Work with tables
159
+ {web,config,tables,queries}
160
+ web Launch the web application
161
+ config Work with configuration
162
+ tables Work with tables
163
+ queries Work with queries
137
164
  ```
138
165
 
139
166
  By default, a YAML configuration file named `laketower.yml` will be looked for.
@@ -246,6 +273,40 @@ weather
246
273
  └── operation metrics
247
274
  ```
248
275
 
276
+ #### Get statistics of a given table
277
+
278
+ Get basic statistics on all columns of a given table:
279
+
280
+ ```bash
281
+ $ laketower -c demo/laketower.yml tables statistics weather
282
+
283
+ ┏━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━┳━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━┓
284
+ ┃ column_name ┃ count ┃ avg ┃ std ┃ min ┃ max ┃
285
+ ┡━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━╇━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━┩
286
+ │ time │ 576 │ None │ None │ 2025-01-26 01:00:00+01 │ 2025-02-12 00:00:00+01 │
287
+ │ city │ 576 │ None │ None │ Grenoble │ Grenoble │
288
+ │ temperature_2m │ 576 │ 5.2623263956047595 │ 3.326529069892729 │ 0.0 │ 15.1 │
289
+ │ relative_humidity_2m │ 576 │ 78.76909722222223 │ 15.701802163559918 │ 29.0 │ 100.0 │
290
+ │ wind_speed_10m │ 576 │ 7.535763886032833 │ 10.00898058743763 │ 0.0 │ 42.4 │
291
+ └──────────────────────┴───────┴────────────────────┴────────────────────┴────────────────────────┴────────────────────────┘
292
+ ```
293
+
294
+ Specifying a table version yields according results:
295
+
296
+ ```bash
297
+ $ laketower -c demo/laketower.yml tables statistics --version 0 weather
298
+
299
+ ┏━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━┳━━━━━━┳━━━━━━┳━━━━━━┳━━━━━━┓
300
+ ┃ column_name ┃ count ┃ avg ┃ std ┃ min ┃ max ┃
301
+ ┡━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━╇━━━━━━╇━━━━━━╇━━━━━━╇━━━━━━┩
302
+ │ time │ 0 │ None │ None │ None │ None │
303
+ │ city │ 0 │ None │ None │ None │ None │
304
+ │ temperature_2m │ 0 │ None │ None │ None │ None │
305
+ │ relative_humidity_2m │ 0 │ None │ None │ None │ None │
306
+ │ wind_speed_10m │ 0 │ None │ None │ None │ None │
307
+ └──────────────────────┴───────┴──────┴──────┴──────┴──────┘
308
+ ```
309
+
249
310
  #### View a given table
250
311
 
251
312
  Using a simple query builder, the content of a table can be displayed.
@@ -309,7 +370,6 @@ $ laketower -c demo/laketower.yml tables view weather --version 1
309
370
  └───────────────────────────┴──────────┴───────────────────┴──────────────────────┴────────────────────┘
310
371
  ```
311
372
 
312
-
313
373
  #### Query all registered tables
314
374
 
315
375
  Query any registered tables using DuckDB SQL dialect!
@@ -326,6 +386,45 @@ $ laketower -c demo/laketower.yml tables query "select date_trunc('day', time) a
326
386
  └───────────────────────────┴────────────────────┘
327
387
  ```
328
388
 
389
+ #### List saved queries
390
+
391
+ ```bash
392
+ $ laketower -c demo/laketower.yml queries list
393
+
394
+ queries
395
+ ├── all_data
396
+ └── daily_avg_temperature
397
+ ```
398
+
399
+ #### Execute saved queries
400
+
401
+ ```bash
402
+ $ laketower -c demo/laketower.yml queries view daily_avg_temperature
403
+
404
+ ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━┓
405
+ ┃ day ┃ avg_temperature ┃
406
+ ┡━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━┩
407
+ │ 2025-01-26 00:00:00+01:00 │ 8.0 │
408
+ │ 2025-01-27 00:00:00+01:00 │ 13.0 │
409
+ │ 2025-01-28 00:00:00+01:00 │ 7.0 │
410
+ │ 2025-01-29 00:00:00+01:00 │ 8.0 │
411
+ │ 2025-01-30 00:00:00+01:00 │ 9.0 │
412
+ │ 2025-01-31 00:00:00+01:00 │ 6.0 │
413
+ │ 2025-02-01 00:00:00+01:00 │ 4.0 │
414
+ │ 2025-02-02 00:00:00+01:00 │ 4.0 │
415
+ │ 2025-02-03 00:00:00+01:00 │ 4.0 │
416
+ │ 2025-02-04 00:00:00+01:00 │ 3.0 │
417
+ │ 2025-02-05 00:00:00+01:00 │ 3.0 │
418
+ │ 2025-02-06 00:00:00+01:00 │ 2.0 │
419
+ │ 2025-02-07 00:00:00+01:00 │ 6.0 │
420
+ │ 2025-02-08 00:00:00+01:00 │ 7.0 │
421
+ │ 2025-02-09 00:00:00+01:00 │ 5.0 │
422
+ │ 2025-02-10 00:00:00+01:00 │ 2.0 │
423
+ │ 2025-02-11 00:00:00+01:00 │ 5.0 │
424
+ │ 2025-02-12 00:00:00+01:00 │ 5.0 │
425
+ └───────────────────────────┴─────────────────┘
426
+ ```
427
+
329
428
  ## License
330
429
 
331
430
  Licensed under [GNU Affero General Public License v3.0 (AGPLv3)](LICENSE.md)
@@ -15,8 +15,10 @@ Utility application to explore and manage tables in your data lakehouse, especia
15
15
  - Inspect table metadata
16
16
  - Inspect table schema
17
17
  - Inspect table history
18
+ - Get table statistics
18
19
  - View table content with a simple query builder
19
20
  - Query all registered tables with DuckDB SQL dialect
21
+ - Execute saved queries
20
22
  - Static and versionable YAML configuration
21
23
  - Web application
22
24
  - CLI application
@@ -67,6 +69,30 @@ tables:
67
69
  - name: weather
68
70
  uri: demo/weather
69
71
  format: delta
72
+
73
+ queries:
74
+ - name: all_data
75
+ title: All data
76
+ sql: |
77
+ select
78
+ sample_table.*,
79
+ weather.*
80
+ from
81
+ sample_table,
82
+ weather
83
+ limit 10
84
+ - name: daily_avg_temperature
85
+ title: Daily average temperature
86
+ sql: |
87
+ select
88
+ date_trunc('day', time) as day,
89
+ round(avg(temperature_2m)) as avg_temperature
90
+ from
91
+ weather
92
+ group by
93
+ day
94
+ order by
95
+ day asc
70
96
  ```
71
97
 
72
98
  ### Web Application
@@ -83,18 +109,20 @@ Laketower provides a CLI interface:
83
109
 
84
110
  ```bash
85
111
  $ laketower --help
86
- usage: laketower [-h] [--version] [--config CONFIG] {web,config,tables} ...
112
+
113
+ usage: laketower [-h] [--version] [--config CONFIG] {web,config,tables,queries} ...
87
114
 
88
115
  options:
89
- -h, --help show this help message and exit
90
- --version show program's version number and exit
91
- --config, -c CONFIG Path to the Laketower YAML configuration file (default: laketower.yml)
116
+ -h, --help show this help message and exit
117
+ --version show program's version number and exit
118
+ --config, -c CONFIG Path to the Laketower YAML configuration file (default: laketower.yml)
92
119
 
93
120
  commands:
94
- {web,config,tables}
95
- web Launch the web application
96
- config Work with configuration
97
- tables Work with tables
121
+ {web,config,tables,queries}
122
+ web Launch the web application
123
+ config Work with configuration
124
+ tables Work with tables
125
+ queries Work with queries
98
126
  ```
99
127
 
100
128
  By default, a YAML configuration file named `laketower.yml` will be looked for.
@@ -207,6 +235,40 @@ weather
207
235
  └── operation metrics
208
236
  ```
209
237
 
238
+ #### Get statistics of a given table
239
+
240
+ Get basic statistics on all columns of a given table:
241
+
242
+ ```bash
243
+ $ laketower -c demo/laketower.yml tables statistics weather
244
+
245
+ ┏━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━┳━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━┓
246
+ ┃ column_name ┃ count ┃ avg ┃ std ┃ min ┃ max ┃
247
+ ┡━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━╇━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━┩
248
+ │ time │ 576 │ None │ None │ 2025-01-26 01:00:00+01 │ 2025-02-12 00:00:00+01 │
249
+ │ city │ 576 │ None │ None │ Grenoble │ Grenoble │
250
+ │ temperature_2m │ 576 │ 5.2623263956047595 │ 3.326529069892729 │ 0.0 │ 15.1 │
251
+ │ relative_humidity_2m │ 576 │ 78.76909722222223 │ 15.701802163559918 │ 29.0 │ 100.0 │
252
+ │ wind_speed_10m │ 576 │ 7.535763886032833 │ 10.00898058743763 │ 0.0 │ 42.4 │
253
+ └──────────────────────┴───────┴────────────────────┴────────────────────┴────────────────────────┴────────────────────────┘
254
+ ```
255
+
256
+ Specifying a table version yields according results:
257
+
258
+ ```bash
259
+ $ laketower -c demo/laketower.yml tables statistics --version 0 weather
260
+
261
+ ┏━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━┳━━━━━━┳━━━━━━┳━━━━━━┳━━━━━━┓
262
+ ┃ column_name ┃ count ┃ avg ┃ std ┃ min ┃ max ┃
263
+ ┡━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━╇━━━━━━╇━━━━━━╇━━━━━━╇━━━━━━┩
264
+ │ time │ 0 │ None │ None │ None │ None │
265
+ │ city │ 0 │ None │ None │ None │ None │
266
+ │ temperature_2m │ 0 │ None │ None │ None │ None │
267
+ │ relative_humidity_2m │ 0 │ None │ None │ None │ None │
268
+ │ wind_speed_10m │ 0 │ None │ None │ None │ None │
269
+ └──────────────────────┴───────┴──────┴──────┴──────┴──────┘
270
+ ```
271
+
210
272
  #### View a given table
211
273
 
212
274
  Using a simple query builder, the content of a table can be displayed.
@@ -270,7 +332,6 @@ $ laketower -c demo/laketower.yml tables view weather --version 1
270
332
  └───────────────────────────┴──────────┴───────────────────┴──────────────────────┴────────────────────┘
271
333
  ```
272
334
 
273
-
274
335
  #### Query all registered tables
275
336
 
276
337
  Query any registered tables using DuckDB SQL dialect!
@@ -287,6 +348,45 @@ $ laketower -c demo/laketower.yml tables query "select date_trunc('day', time) a
287
348
  └───────────────────────────┴────────────────────┘
288
349
  ```
289
350
 
351
+ #### List saved queries
352
+
353
+ ```bash
354
+ $ laketower -c demo/laketower.yml queries list
355
+
356
+ queries
357
+ ├── all_data
358
+ └── daily_avg_temperature
359
+ ```
360
+
361
+ #### Execute saved queries
362
+
363
+ ```bash
364
+ $ laketower -c demo/laketower.yml queries view daily_avg_temperature
365
+
366
+ ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━┓
367
+ ┃ day ┃ avg_temperature ┃
368
+ ┡━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━┩
369
+ │ 2025-01-26 00:00:00+01:00 │ 8.0 │
370
+ │ 2025-01-27 00:00:00+01:00 │ 13.0 │
371
+ │ 2025-01-28 00:00:00+01:00 │ 7.0 │
372
+ │ 2025-01-29 00:00:00+01:00 │ 8.0 │
373
+ │ 2025-01-30 00:00:00+01:00 │ 9.0 │
374
+ │ 2025-01-31 00:00:00+01:00 │ 6.0 │
375
+ │ 2025-02-01 00:00:00+01:00 │ 4.0 │
376
+ │ 2025-02-02 00:00:00+01:00 │ 4.0 │
377
+ │ 2025-02-03 00:00:00+01:00 │ 4.0 │
378
+ │ 2025-02-04 00:00:00+01:00 │ 3.0 │
379
+ │ 2025-02-05 00:00:00+01:00 │ 3.0 │
380
+ │ 2025-02-06 00:00:00+01:00 │ 2.0 │
381
+ │ 2025-02-07 00:00:00+01:00 │ 6.0 │
382
+ │ 2025-02-08 00:00:00+01:00 │ 7.0 │
383
+ │ 2025-02-09 00:00:00+01:00 │ 5.0 │
384
+ │ 2025-02-10 00:00:00+01:00 │ 2.0 │
385
+ │ 2025-02-11 00:00:00+01:00 │ 5.0 │
386
+ │ 2025-02-12 00:00:00+01:00 │ 5.0 │
387
+ └───────────────────────────┴─────────────────┘
388
+ ```
389
+
290
390
  ## License
291
391
 
292
392
  Licensed under [GNU Affero General Public License v3.0 (AGPLv3)](LICENSE.md)
@@ -0,0 +1,31 @@
1
+ tables:
2
+ - name: sample_table
3
+ uri: demo/sample_table
4
+ format: delta
5
+ - name: weather
6
+ uri: demo/weather
7
+ format: delta
8
+
9
+ queries:
10
+ - name: all_data
11
+ title: All data
12
+ sql: |
13
+ select
14
+ sample_table.*,
15
+ weather.*
16
+ from
17
+ sample_table,
18
+ weather
19
+ limit 10
20
+ - name: daily_avg_temperature
21
+ title: Daily average temperature
22
+ sql: |
23
+ select
24
+ date_trunc('day', time) as day,
25
+ round(avg(temperature_2m)) as avg_temperature
26
+ from
27
+ weather
28
+ group by
29
+ day
30
+ order by
31
+ day asc
@@ -0,0 +1 @@
1
+ __version__ = "0.4.0"
@@ -1,5 +1,3 @@
1
- from __future__ import annotations
2
-
3
1
  import argparse
4
2
  import os
5
3
  from pathlib import Path
@@ -13,7 +11,12 @@ import uvicorn
13
11
 
14
12
  from laketower.__about__ import __version__
15
13
  from laketower.config import load_yaml_config
16
- from laketower.tables import execute_query, generate_table_query, load_table
14
+ from laketower.tables import (
15
+ execute_query,
16
+ generate_table_query,
17
+ generate_table_statistics_query,
18
+ load_table,
19
+ )
17
20
 
18
21
 
19
22
  def run_web(config_path: Path, reload: bool) -> None: # pragma: no cover
@@ -99,6 +102,27 @@ def table_history(config_path: Path, table_name: str) -> None:
99
102
  console.print(tree, markup=False)
100
103
 
101
104
 
105
+ def table_statistics(
106
+ config_path: Path, table_name: str, version: int | None = None
107
+ ) -> None:
108
+ config = load_yaml_config(config_path)
109
+ table_config = next(filter(lambda x: x.name == table_name, config.tables))
110
+ table = load_table(table_config)
111
+ table_dataset = table.dataset(version=version)
112
+ sql_query = generate_table_statistics_query(table_name)
113
+ results = execute_query({table_name: table_dataset}, sql_query)
114
+
115
+ out = rich.table.Table()
116
+ for column in results.columns:
117
+ out.add_column(column)
118
+ for value_list in results.to_numpy().tolist():
119
+ row = [str(x) for x in value_list]
120
+ out.add_row(*row)
121
+
122
+ console = rich.get_console()
123
+ console.print(out, markup=False) # disable markup to allow bracket characters
124
+
125
+
102
126
  def view_table(
103
127
  config_path: Path,
104
128
  table_name: str,
@@ -151,6 +175,40 @@ def query_table(config_path: Path, sql_query: str) -> None:
151
175
  console.print(out)
152
176
 
153
177
 
178
+ def list_queries(config_path: Path) -> None:
179
+ config = load_yaml_config(config_path)
180
+ tree = rich.tree.Tree("queries")
181
+ for query in config.queries:
182
+ tree.add(query.name)
183
+ console = rich.get_console()
184
+ console.print(tree)
185
+
186
+
187
+ def view_query(config_path: Path, query_name: str) -> None:
188
+ config = load_yaml_config(config_path)
189
+ query_config = next(filter(lambda x: x.name == query_name, config.queries))
190
+ sql_query = query_config.sql
191
+ tables_dataset = {
192
+ table_config.name: load_table(table_config).dataset()
193
+ for table_config in config.tables
194
+ }
195
+
196
+ out: rich.jupyter.JupyterMixin
197
+ try:
198
+ results = execute_query(tables_dataset, sql_query)
199
+ out = rich.table.Table()
200
+ for column in results.columns:
201
+ out.add_column(column)
202
+ for value_list in results.values.tolist():
203
+ row = [str(x) for x in value_list]
204
+ out.add_row(*row)
205
+ except ValueError as e:
206
+ out = rich.panel.Panel.fit(f"[red]{e}")
207
+
208
+ console = rich.get_console()
209
+ console.print(out)
210
+
211
+
154
212
  def cli() -> None:
155
213
  parser = argparse.ArgumentParser(
156
214
  "laketower", formatter_class=argparse.ArgumentDefaultsHelpFormatter
@@ -214,6 +272,17 @@ def cli() -> None:
214
272
  parser_tables_history.add_argument("table", help="Name of the table")
215
273
  parser_tables_history.set_defaults(func=lambda x: table_history(x.config, x.table))
216
274
 
275
+ parser_tables_statistics = subsparsers_tables.add_parser(
276
+ "statistics", help="Display summary statistics of a given table schema"
277
+ )
278
+ parser_tables_statistics.add_argument("table", help="Name of the table")
279
+ parser_tables_statistics.add_argument(
280
+ "--version", type=int, help="Time-travel to table revision number"
281
+ )
282
+ parser_tables_statistics.set_defaults(
283
+ func=lambda x: table_statistics(x.config, x.table, x.version)
284
+ )
285
+
217
286
  parser_tables_view = subsparsers_tables.add_parser(
218
287
  "view", help="View a given table"
219
288
  )
@@ -244,5 +313,19 @@ def cli() -> None:
244
313
  parser_tables_query.add_argument("sql", help="SQL query to execute")
245
314
  parser_tables_query.set_defaults(func=lambda x: query_table(x.config, x.sql))
246
315
 
316
+ parser_queries = subparsers.add_parser("queries", help="Work with queries")
317
+ subsparsers_queries = parser_queries.add_subparsers(required=True)
318
+
319
+ parser_queries_list = subsparsers_queries.add_parser(
320
+ "list", help="List all registered queries"
321
+ )
322
+ parser_queries_list.set_defaults(func=lambda x: list_queries(x.config))
323
+
324
+ parser_queries_view = subsparsers_queries.add_parser(
325
+ "view", help="View a given query"
326
+ )
327
+ parser_queries_view.add_argument("query", help="Name of the query")
328
+ parser_queries_view.set_defaults(func=lambda x: view_query(x.config, x.query))
329
+
247
330
  args = parser.parse_args()
248
331
  args.func(args)
@@ -29,6 +29,7 @@ class ConfigTable(pydantic.BaseModel):
29
29
 
30
30
  class ConfigQuery(pydantic.BaseModel):
31
31
  name: str
32
+ title: str
32
33
  sql: str
33
34
 
34
35
 
@@ -38,6 +39,7 @@ class ConfigDashboard(pydantic.BaseModel):
38
39
 
39
40
  class Config(pydantic.BaseModel):
40
41
  tables: list[ConfigTable] = []
42
+ queries: list[ConfigQuery] = []
41
43
 
42
44
 
43
45
  def load_yaml_config(config_path: Path) -> Config:
@@ -1,7 +1,5 @@
1
- from __future__ import annotations
2
-
3
1
  from datetime import datetime, timezone
4
- from typing import Any, Optional, Protocol
2
+ from typing import Any, Protocol
5
3
 
6
4
  import deltalake
7
5
  import duckdb
@@ -11,6 +9,7 @@ import pyarrow.dataset as padataset
11
9
  import pydantic
12
10
  import sqlglot
13
11
  import sqlglot.dialects.duckdb
12
+ import sqlglot.expressions
14
13
 
15
14
  from laketower.config import ConfigTable, TableFormats
16
15
 
@@ -20,8 +19,8 @@ DEFAULT_LIMIT = 10
20
19
 
21
20
  class TableMetadata(pydantic.BaseModel):
22
21
  table_format: TableFormats
23
- name: Optional[str] = None
24
- description: Optional[str] = None
22
+ name: str | None = None
23
+ description: str | None = None
25
24
  uri: str
26
25
  id: str
27
26
  version: int
@@ -33,7 +32,7 @@ class TableMetadata(pydantic.BaseModel):
33
32
  class TableRevision(pydantic.BaseModel):
34
33
  version: int
35
34
  timestamp: datetime
36
- client_version: Optional[str] = None
35
+ client_version: str | None = None
37
36
  operation: str
38
37
  operation_parameters: dict[str, Any]
39
38
  operation_metrics: dict[str, Any]
@@ -122,6 +121,12 @@ def generate_table_query(
122
121
  )
123
122
 
124
123
 
124
+ def generate_table_statistics_query(table_name: str) -> str:
125
+ return (
126
+ f"SELECT column_name, count, avg, std, min, max FROM (SUMMARIZE {table_name})" # nosec B608
127
+ )
128
+
129
+
125
130
  def execute_query(
126
131
  tables_datasets: dict[str, padataset.Dataset], sql_query: str
127
132
  ) -> pd.DataFrame:
@@ -19,7 +19,14 @@
19
19
  <a class="sidebar-brand" href="/">
20
20
  Laketower
21
21
  </a>
22
- <button type="button" class="btn-close d-md-none" data-bs-dismiss="offcanvas" aria-label="Close" data-bs-target="#sidebar"></button>
22
+ <button
23
+ type="button"
24
+ class="btn-close d-md-none"
25
+ data-bs-dismiss="offcanvas"
26
+ aria-label="Close"
27
+ data-bs-target="#sidebar"
28
+ >
29
+ </button>
23
30
  </div>
24
31
  <div class="offcanvas-body">
25
32
  <ul class="sidebar-nav">
@@ -30,12 +37,36 @@
30
37
  {% for table in tables %}
31
38
  {% set table_url = '/tables/' + table.name %}
32
39
  <li class="nav-item">
33
- <a class="nav-link{% if request.url.path.startswith(table_url) %} active{% endif %}" href="{{ table_url }}" aria-current="page">
40
+ <a
41
+ class="text-truncate nav-link{% if request.url.path.startswith(table_url) %} active{% endif %}"
42
+ href="{{ table_url }}"
43
+ aria-current="page"
44
+ >
34
45
  <i class="bi-table" aria-hidden="true"></i>
35
46
  {{ table.name }}
36
47
  </a>
37
48
  </li>
38
49
  {% endfor %}
50
+
51
+ <li><hr class="sidebar-divider"></li>
52
+
53
+ <li>
54
+ <h6 class="sidebar-header">Queries</h6>
55
+ </li>
56
+ <li><hr class="sidebar-divider"></li>
57
+ {% for query in queries %}
58
+ {% set query_url = '/queries/' + query.name + '/view' %}
59
+ <li class="nav-item">
60
+ <a
61
+ class="text-truncate nav-link{% if request.url.path.startswith(query_url) %} active{% endif %}"
62
+ href="{{ query_url }}"
63
+ aria-current="page"
64
+ >
65
+ <i class="bi-code-square" aria-hidden="true"></i>
66
+ {{ query.title }}
67
+ </a>
68
+ </li>
69
+ {% endfor %}
39
70
  </ul>
40
71
  </div>
41
72
  </nav>