laketower 0.3.0__tar.gz → 0.4.1__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of laketower might be problematic. Click here for more details.

Files changed (48) hide show
  1. {laketower-0.3.0 → laketower-0.4.1}/CHANGELOG.md +25 -1
  2. {laketower-0.3.0 → laketower-0.4.1}/PKG-INFO +110 -10
  3. {laketower-0.3.0 → laketower-0.4.1}/README.md +109 -9
  4. laketower-0.4.1/demo/laketower.yml +31 -0
  5. laketower-0.4.1/laketower/__about__.py +1 -0
  6. {laketower-0.3.0 → laketower-0.4.1}/laketower/cli.py +86 -1
  7. {laketower-0.3.0 → laketower-0.4.1}/laketower/config.py +2 -0
  8. {laketower-0.3.0 → laketower-0.4.1}/laketower/tables.py +7 -0
  9. {laketower-0.3.0 → laketower-0.4.1}/laketower/templates/_base.html +33 -2
  10. laketower-0.4.1/laketower/templates/queries/view.html +60 -0
  11. {laketower-0.3.0 → laketower-0.4.1}/laketower/templates/tables/_macros.html +3 -0
  12. {laketower-0.3.0 → laketower-0.4.1}/laketower/templates/tables/query.html +6 -2
  13. laketower-0.4.1/laketower/templates/tables/statistics.html +56 -0
  14. {laketower-0.3.0 → laketower-0.4.1}/laketower/templates/tables/view.html +1 -1
  15. {laketower-0.3.0 → laketower-0.4.1}/laketower/web.py +70 -1
  16. {laketower-0.3.0 → laketower-0.4.1}/pyproject.toml +3 -2
  17. {laketower-0.3.0 → laketower-0.4.1}/tests/conftest.py +8 -1
  18. {laketower-0.3.0 → laketower-0.4.1}/tests/test_cli.py +163 -3
  19. {laketower-0.3.0 → laketower-0.4.1}/tests/test_web.py +127 -13
  20. {laketower-0.3.0 → laketower-0.4.1}/uv.lock +50 -26
  21. laketower-0.3.0/demo/laketower.yml +0 -7
  22. laketower-0.3.0/laketower/__about__.py +0 -1
  23. {laketower-0.3.0 → laketower-0.4.1}/.github/workflows/ci-cd.yml +0 -0
  24. {laketower-0.3.0 → laketower-0.4.1}/.gitignore +0 -0
  25. {laketower-0.3.0 → laketower-0.4.1}/.python-version +0 -0
  26. {laketower-0.3.0 → laketower-0.4.1}/LICENSE.md +0 -0
  27. {laketower-0.3.0 → laketower-0.4.1}/demo/generate.py +0 -0
  28. {laketower-0.3.0 → laketower-0.4.1}/demo/sample_table/_delta_log/00000000000000000000.json +0 -0
  29. {laketower-0.3.0 → laketower-0.4.1}/demo/sample_table/_delta_log/00000000000000000001.json +0 -0
  30. {laketower-0.3.0 → laketower-0.4.1}/demo/sample_table/_delta_log/00000000000000000002.json +0 -0
  31. {laketower-0.3.0 → laketower-0.4.1}/demo/sample_table/_delta_log/00000000000000000003.json +0 -0
  32. {laketower-0.3.0 → laketower-0.4.1}/demo/sample_table/part-00001-1a31a393-6db6-4d1a-bf4e-81ea061ff8cd-c000.snappy.parquet +0 -0
  33. {laketower-0.3.0 → laketower-0.4.1}/demo/sample_table/part-00001-5af77102-9207-4c89-aaf6-37e1f815ec26-c000.snappy.parquet +0 -0
  34. {laketower-0.3.0 → laketower-0.4.1}/demo/sample_table/part-00001-b11bab55-43d0-4d05-ae88-5b9481ae57db-c000.snappy.parquet +0 -0
  35. {laketower-0.3.0 → laketower-0.4.1}/demo/weather/_delta_log/00000000000000000000.json +0 -0
  36. {laketower-0.3.0 → laketower-0.4.1}/demo/weather/_delta_log/00000000000000000001.json +0 -0
  37. {laketower-0.3.0 → laketower-0.4.1}/demo/weather/_delta_log/00000000000000000002.json +0 -0
  38. {laketower-0.3.0 → laketower-0.4.1}/demo/weather/part-00001-2323b963-be56-44e0-8c10-e237e7e6d4b9-c000.snappy.parquet +0 -0
  39. {laketower-0.3.0 → laketower-0.4.1}/demo/weather/part-00001-6360cbf8-f8a9-475f-8729-6f20b4ca64a9-c000.snappy.parquet +0 -0
  40. {laketower-0.3.0 → laketower-0.4.1}/laketower/__init__.py +0 -0
  41. {laketower-0.3.0 → laketower-0.4.1}/laketower/__main__.py +0 -0
  42. {laketower-0.3.0 → laketower-0.4.1}/laketower/static/.gitkeep +0 -0
  43. {laketower-0.3.0 → laketower-0.4.1}/laketower/templates/index.html +0 -0
  44. {laketower-0.3.0 → laketower-0.4.1}/laketower/templates/tables/history.html +0 -0
  45. {laketower-0.3.0 → laketower-0.4.1}/laketower/templates/tables/index.html +0 -0
  46. {laketower-0.3.0 → laketower-0.4.1}/renovate.json +0 -0
  47. {laketower-0.3.0 → laketower-0.4.1}/tasks.py +0 -0
  48. {laketower-0.3.0 → laketower-0.4.1}/tests/__init__.py +0 -0
@@ -7,6 +7,28 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
7
7
 
8
8
  ## [Unreleased]
9
9
 
10
+ ## [0.4.1] - 2025-03-02
11
+ Minor release with fixes.
12
+
13
+ ### Added
14
+ - web: allow editing queries
15
+
16
+ ### Fixed
17
+ - web: missing tables query page title
18
+ - web: urlencode table view sql query link
19
+
20
+ ## [0.4.0] - 2025-03-01
21
+ Introducing new features:
22
+ - Display tables statistics
23
+ - List and execute pre-defined queries
24
+
25
+ ### Added
26
+ - web: add queries view page
27
+ - web: add table statistics page with version control
28
+ - cli: add queries view command
29
+ - cli: add queries list command
30
+ - cli: add table statistics command
31
+
10
32
  ## [0.3.0] - 2025-02-27
11
33
  Minor release with fixes and dropped Python 3.9 support.
12
34
 
@@ -47,7 +69,9 @@ Initial release of `laketower`.
47
69
  - View a given table with simple query builder
48
70
  - Query all registered tables with DuckDB SQL dialect
49
71
 
50
- [Unreleased]: https://github.com/datalpia/laketower/compare/0.3.0...HEAD
72
+ [Unreleased]: https://github.com/datalpia/laketower/compare/0.4.1...HEAD
73
+ [0.4.1]: https://github.com/datalpia/laketower/compare/0.4.0...0.4.1
74
+ [0.4.0]: https://github.com/datalpia/laketower/compare/0.3.0...0.4.0
51
75
  [0.3.0]: https://github.com/datalpia/laketower/compare/0.2.0...0.3.0
52
76
  [0.2.0]: https://github.com/datalpia/laketower/compare/0.1.0...0.2.0
53
77
  [0.1.0]: https://github.com/datalpia/laketower/releases/tag/0.1.0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: laketower
3
- Version: 0.3.0
3
+ Version: 0.4.1
4
4
  Summary: Oversee your lakehouse
5
5
  Project-URL: Repository, https://github.com/datalpia/laketower
6
6
  Project-URL: Issues, https://github.com/datalpia/laketower/issues
@@ -53,8 +53,10 @@ Utility application to explore and manage tables in your data lakehouse, especia
53
53
  - Inspect table metadata
54
54
  - Inspect table schema
55
55
  - Inspect table history
56
+ - Get table statistics
56
57
  - View table content with a simple query builder
57
58
  - Query all registered tables with DuckDB SQL dialect
59
+ - Execute saved queries
58
60
  - Static and versionable YAML configuration
59
61
  - Web application
60
62
  - CLI application
@@ -105,6 +107,30 @@ tables:
105
107
  - name: weather
106
108
  uri: demo/weather
107
109
  format: delta
110
+
111
+ queries:
112
+ - name: all_data
113
+ title: All data
114
+ sql: |
115
+ select
116
+ sample_table.*,
117
+ weather.*
118
+ from
119
+ sample_table,
120
+ weather
121
+ limit 10
122
+ - name: daily_avg_temperature
123
+ title: Daily average temperature
124
+ sql: |
125
+ select
126
+ date_trunc('day', time) as day,
127
+ round(avg(temperature_2m)) as avg_temperature
128
+ from
129
+ weather
130
+ group by
131
+ day
132
+ order by
133
+ day asc
108
134
  ```
109
135
 
110
136
  ### Web Application
@@ -121,18 +147,20 @@ Laketower provides a CLI interface:
121
147
 
122
148
  ```bash
123
149
  $ laketower --help
124
- usage: laketower [-h] [--version] [--config CONFIG] {web,config,tables} ...
150
+
151
+ usage: laketower [-h] [--version] [--config CONFIG] {web,config,tables,queries} ...
125
152
 
126
153
  options:
127
- -h, --help show this help message and exit
128
- --version show program's version number and exit
129
- --config, -c CONFIG Path to the Laketower YAML configuration file (default: laketower.yml)
154
+ -h, --help show this help message and exit
155
+ --version show program's version number and exit
156
+ --config, -c CONFIG Path to the Laketower YAML configuration file (default: laketower.yml)
130
157
 
131
158
  commands:
132
- {web,config,tables}
133
- web Launch the web application
134
- config Work with configuration
135
- tables Work with tables
159
+ {web,config,tables,queries}
160
+ web Launch the web application
161
+ config Work with configuration
162
+ tables Work with tables
163
+ queries Work with queries
136
164
  ```
137
165
 
138
166
  By default, a YAML configuration file named `laketower.yml` will be looked for.
@@ -245,6 +273,40 @@ weather
245
273
  └── operation metrics
246
274
  ```
247
275
 
276
+ #### Get statistics of a given table
277
+
278
+ Get basic statistics on all columns of a given table:
279
+
280
+ ```bash
281
+ $ laketower -c demo/laketower.yml tables statistics weather
282
+
283
+ ┏━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━┳━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━┓
284
+ ┃ column_name ┃ count ┃ avg ┃ std ┃ min ┃ max ┃
285
+ ┡━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━╇━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━┩
286
+ │ time │ 576 │ None │ None │ 2025-01-26 01:00:00+01 │ 2025-02-12 00:00:00+01 │
287
+ │ city │ 576 │ None │ None │ Grenoble │ Grenoble │
288
+ │ temperature_2m │ 576 │ 5.2623263956047595 │ 3.326529069892729 │ 0.0 │ 15.1 │
289
+ │ relative_humidity_2m │ 576 │ 78.76909722222223 │ 15.701802163559918 │ 29.0 │ 100.0 │
290
+ │ wind_speed_10m │ 576 │ 7.535763886032833 │ 10.00898058743763 │ 0.0 │ 42.4 │
291
+ └──────────────────────┴───────┴────────────────────┴────────────────────┴────────────────────────┴────────────────────────┘
292
+ ```
293
+
294
+ Specifying a table version yields according results:
295
+
296
+ ```bash
297
+ $ laketower -c demo/laketower.yml tables statistics --version 0 weather
298
+
299
+ ┏━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━┳━━━━━━┳━━━━━━┳━━━━━━┳━━━━━━┓
300
+ ┃ column_name ┃ count ┃ avg ┃ std ┃ min ┃ max ┃
301
+ ┡━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━╇━━━━━━╇━━━━━━╇━━━━━━╇━━━━━━┩
302
+ │ time │ 0 │ None │ None │ None │ None │
303
+ │ city │ 0 │ None │ None │ None │ None │
304
+ │ temperature_2m │ 0 │ None │ None │ None │ None │
305
+ │ relative_humidity_2m │ 0 │ None │ None │ None │ None │
306
+ │ wind_speed_10m │ 0 │ None │ None │ None │ None │
307
+ └──────────────────────┴───────┴──────┴──────┴──────┴──────┘
308
+ ```
309
+
248
310
  #### View a given table
249
311
 
250
312
  Using a simple query builder, the content of a table can be displayed.
@@ -308,7 +370,6 @@ $ laketower -c demo/laketower.yml tables view weather --version 1
308
370
  └───────────────────────────┴──────────┴───────────────────┴──────────────────────┴────────────────────┘
309
371
  ```
310
372
 
311
-
312
373
  #### Query all registered tables
313
374
 
314
375
  Query any registered tables using DuckDB SQL dialect!
@@ -325,6 +386,45 @@ $ laketower -c demo/laketower.yml tables query "select date_trunc('day', time) a
325
386
  └───────────────────────────┴────────────────────┘
326
387
  ```
327
388
 
389
+ #### List saved queries
390
+
391
+ ```bash
392
+ $ laketower -c demo/laketower.yml queries list
393
+
394
+ queries
395
+ ├── all_data
396
+ └── daily_avg_temperature
397
+ ```
398
+
399
+ #### Execute saved queries
400
+
401
+ ```bash
402
+ $ laketower -c demo/laketower.yml queries view daily_avg_temperature
403
+
404
+ ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━┓
405
+ ┃ day ┃ avg_temperature ┃
406
+ ┡━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━┩
407
+ │ 2025-01-26 00:00:00+01:00 │ 8.0 │
408
+ │ 2025-01-27 00:00:00+01:00 │ 13.0 │
409
+ │ 2025-01-28 00:00:00+01:00 │ 7.0 │
410
+ │ 2025-01-29 00:00:00+01:00 │ 8.0 │
411
+ │ 2025-01-30 00:00:00+01:00 │ 9.0 │
412
+ │ 2025-01-31 00:00:00+01:00 │ 6.0 │
413
+ │ 2025-02-01 00:00:00+01:00 │ 4.0 │
414
+ │ 2025-02-02 00:00:00+01:00 │ 4.0 │
415
+ │ 2025-02-03 00:00:00+01:00 │ 4.0 │
416
+ │ 2025-02-04 00:00:00+01:00 │ 3.0 │
417
+ │ 2025-02-05 00:00:00+01:00 │ 3.0 │
418
+ │ 2025-02-06 00:00:00+01:00 │ 2.0 │
419
+ │ 2025-02-07 00:00:00+01:00 │ 6.0 │
420
+ │ 2025-02-08 00:00:00+01:00 │ 7.0 │
421
+ │ 2025-02-09 00:00:00+01:00 │ 5.0 │
422
+ │ 2025-02-10 00:00:00+01:00 │ 2.0 │
423
+ │ 2025-02-11 00:00:00+01:00 │ 5.0 │
424
+ │ 2025-02-12 00:00:00+01:00 │ 5.0 │
425
+ └───────────────────────────┴─────────────────┘
426
+ ```
427
+
328
428
  ## License
329
429
 
330
430
  Licensed under [GNU Affero General Public License v3.0 (AGPLv3)](LICENSE.md)
@@ -15,8 +15,10 @@ Utility application to explore and manage tables in your data lakehouse, especia
15
15
  - Inspect table metadata
16
16
  - Inspect table schema
17
17
  - Inspect table history
18
+ - Get table statistics
18
19
  - View table content with a simple query builder
19
20
  - Query all registered tables with DuckDB SQL dialect
21
+ - Execute saved queries
20
22
  - Static and versionable YAML configuration
21
23
  - Web application
22
24
  - CLI application
@@ -67,6 +69,30 @@ tables:
67
69
  - name: weather
68
70
  uri: demo/weather
69
71
  format: delta
72
+
73
+ queries:
74
+ - name: all_data
75
+ title: All data
76
+ sql: |
77
+ select
78
+ sample_table.*,
79
+ weather.*
80
+ from
81
+ sample_table,
82
+ weather
83
+ limit 10
84
+ - name: daily_avg_temperature
85
+ title: Daily average temperature
86
+ sql: |
87
+ select
88
+ date_trunc('day', time) as day,
89
+ round(avg(temperature_2m)) as avg_temperature
90
+ from
91
+ weather
92
+ group by
93
+ day
94
+ order by
95
+ day asc
70
96
  ```
71
97
 
72
98
  ### Web Application
@@ -83,18 +109,20 @@ Laketower provides a CLI interface:
83
109
 
84
110
  ```bash
85
111
  $ laketower --help
86
- usage: laketower [-h] [--version] [--config CONFIG] {web,config,tables} ...
112
+
113
+ usage: laketower [-h] [--version] [--config CONFIG] {web,config,tables,queries} ...
87
114
 
88
115
  options:
89
- -h, --help show this help message and exit
90
- --version show program's version number and exit
91
- --config, -c CONFIG Path to the Laketower YAML configuration file (default: laketower.yml)
116
+ -h, --help show this help message and exit
117
+ --version show program's version number and exit
118
+ --config, -c CONFIG Path to the Laketower YAML configuration file (default: laketower.yml)
92
119
 
93
120
  commands:
94
- {web,config,tables}
95
- web Launch the web application
96
- config Work with configuration
97
- tables Work with tables
121
+ {web,config,tables,queries}
122
+ web Launch the web application
123
+ config Work with configuration
124
+ tables Work with tables
125
+ queries Work with queries
98
126
  ```
99
127
 
100
128
  By default, a YAML configuration file named `laketower.yml` will be looked for.
@@ -207,6 +235,40 @@ weather
207
235
  └── operation metrics
208
236
  ```
209
237
 
238
+ #### Get statistics of a given table
239
+
240
+ Get basic statistics on all columns of a given table:
241
+
242
+ ```bash
243
+ $ laketower -c demo/laketower.yml tables statistics weather
244
+
245
+ ┏━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━┳━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━┓
246
+ ┃ column_name ┃ count ┃ avg ┃ std ┃ min ┃ max ┃
247
+ ┡━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━╇━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━┩
248
+ │ time │ 576 │ None │ None │ 2025-01-26 01:00:00+01 │ 2025-02-12 00:00:00+01 │
249
+ │ city │ 576 │ None │ None │ Grenoble │ Grenoble │
250
+ │ temperature_2m │ 576 │ 5.2623263956047595 │ 3.326529069892729 │ 0.0 │ 15.1 │
251
+ │ relative_humidity_2m │ 576 │ 78.76909722222223 │ 15.701802163559918 │ 29.0 │ 100.0 │
252
+ │ wind_speed_10m │ 576 │ 7.535763886032833 │ 10.00898058743763 │ 0.0 │ 42.4 │
253
+ └──────────────────────┴───────┴────────────────────┴────────────────────┴────────────────────────┴────────────────────────┘
254
+ ```
255
+
256
+ Specifying a table version yields according results:
257
+
258
+ ```bash
259
+ $ laketower -c demo/laketower.yml tables statistics --version 0 weather
260
+
261
+ ┏━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━┳━━━━━━┳━━━━━━┳━━━━━━┳━━━━━━┓
262
+ ┃ column_name ┃ count ┃ avg ┃ std ┃ min ┃ max ┃
263
+ ┡━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━╇━━━━━━╇━━━━━━╇━━━━━━╇━━━━━━┩
264
+ │ time │ 0 │ None │ None │ None │ None │
265
+ │ city │ 0 │ None │ None │ None │ None │
266
+ │ temperature_2m │ 0 │ None │ None │ None │ None │
267
+ │ relative_humidity_2m │ 0 │ None │ None │ None │ None │
268
+ │ wind_speed_10m │ 0 │ None │ None │ None │ None │
269
+ └──────────────────────┴───────┴──────┴──────┴──────┴──────┘
270
+ ```
271
+
210
272
  #### View a given table
211
273
 
212
274
  Using a simple query builder, the content of a table can be displayed.
@@ -270,7 +332,6 @@ $ laketower -c demo/laketower.yml tables view weather --version 1
270
332
  └───────────────────────────┴──────────┴───────────────────┴──────────────────────┴────────────────────┘
271
333
  ```
272
334
 
273
-
274
335
  #### Query all registered tables
275
336
 
276
337
  Query any registered tables using DuckDB SQL dialect!
@@ -287,6 +348,45 @@ $ laketower -c demo/laketower.yml tables query "select date_trunc('day', time) a
287
348
  └───────────────────────────┴────────────────────┘
288
349
  ```
289
350
 
351
+ #### List saved queries
352
+
353
+ ```bash
354
+ $ laketower -c demo/laketower.yml queries list
355
+
356
+ queries
357
+ ├── all_data
358
+ └── daily_avg_temperature
359
+ ```
360
+
361
+ #### Execute saved queries
362
+
363
+ ```bash
364
+ $ laketower -c demo/laketower.yml queries view daily_avg_temperature
365
+
366
+ ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━┓
367
+ ┃ day ┃ avg_temperature ┃
368
+ ┡━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━┩
369
+ │ 2025-01-26 00:00:00+01:00 │ 8.0 │
370
+ │ 2025-01-27 00:00:00+01:00 │ 13.0 │
371
+ │ 2025-01-28 00:00:00+01:00 │ 7.0 │
372
+ │ 2025-01-29 00:00:00+01:00 │ 8.0 │
373
+ │ 2025-01-30 00:00:00+01:00 │ 9.0 │
374
+ │ 2025-01-31 00:00:00+01:00 │ 6.0 │
375
+ │ 2025-02-01 00:00:00+01:00 │ 4.0 │
376
+ │ 2025-02-02 00:00:00+01:00 │ 4.0 │
377
+ │ 2025-02-03 00:00:00+01:00 │ 4.0 │
378
+ │ 2025-02-04 00:00:00+01:00 │ 3.0 │
379
+ │ 2025-02-05 00:00:00+01:00 │ 3.0 │
380
+ │ 2025-02-06 00:00:00+01:00 │ 2.0 │
381
+ │ 2025-02-07 00:00:00+01:00 │ 6.0 │
382
+ │ 2025-02-08 00:00:00+01:00 │ 7.0 │
383
+ │ 2025-02-09 00:00:00+01:00 │ 5.0 │
384
+ │ 2025-02-10 00:00:00+01:00 │ 2.0 │
385
+ │ 2025-02-11 00:00:00+01:00 │ 5.0 │
386
+ │ 2025-02-12 00:00:00+01:00 │ 5.0 │
387
+ └───────────────────────────┴─────────────────┘
388
+ ```
389
+
290
390
  ## License
291
391
 
292
392
  Licensed under [GNU Affero General Public License v3.0 (AGPLv3)](LICENSE.md)
@@ -0,0 +1,31 @@
1
+ tables:
2
+ - name: sample_table
3
+ uri: demo/sample_table
4
+ format: delta
5
+ - name: weather
6
+ uri: demo/weather
7
+ format: delta
8
+
9
+ queries:
10
+ - name: all_data
11
+ title: All data
12
+ sql: |
13
+ select
14
+ sample_table.*,
15
+ weather.*
16
+ from
17
+ sample_table,
18
+ weather
19
+ limit 10
20
+ - name: daily_avg_temperature
21
+ title: Daily average temperature
22
+ sql: |
23
+ select
24
+ date_trunc('day', time) as day,
25
+ round(avg(temperature_2m)) as avg_temperature
26
+ from
27
+ weather
28
+ group by
29
+ day
30
+ order by
31
+ day asc
@@ -0,0 +1 @@
1
+ __version__ = "0.4.1"
@@ -11,7 +11,12 @@ import uvicorn
11
11
 
12
12
  from laketower.__about__ import __version__
13
13
  from laketower.config import load_yaml_config
14
- from laketower.tables import execute_query, generate_table_query, load_table
14
+ from laketower.tables import (
15
+ execute_query,
16
+ generate_table_query,
17
+ generate_table_statistics_query,
18
+ load_table,
19
+ )
15
20
 
16
21
 
17
22
  def run_web(config_path: Path, reload: bool) -> None: # pragma: no cover
@@ -97,6 +102,27 @@ def table_history(config_path: Path, table_name: str) -> None:
97
102
  console.print(tree, markup=False)
98
103
 
99
104
 
105
+ def table_statistics(
106
+ config_path: Path, table_name: str, version: int | None = None
107
+ ) -> None:
108
+ config = load_yaml_config(config_path)
109
+ table_config = next(filter(lambda x: x.name == table_name, config.tables))
110
+ table = load_table(table_config)
111
+ table_dataset = table.dataset(version=version)
112
+ sql_query = generate_table_statistics_query(table_name)
113
+ results = execute_query({table_name: table_dataset}, sql_query)
114
+
115
+ out = rich.table.Table()
116
+ for column in results.columns:
117
+ out.add_column(column)
118
+ for value_list in results.to_numpy().tolist():
119
+ row = [str(x) for x in value_list]
120
+ out.add_row(*row)
121
+
122
+ console = rich.get_console()
123
+ console.print(out, markup=False) # disable markup to allow bracket characters
124
+
125
+
100
126
  def view_table(
101
127
  config_path: Path,
102
128
  table_name: str,
@@ -149,6 +175,40 @@ def query_table(config_path: Path, sql_query: str) -> None:
149
175
  console.print(out)
150
176
 
151
177
 
178
+ def list_queries(config_path: Path) -> None:
179
+ config = load_yaml_config(config_path)
180
+ tree = rich.tree.Tree("queries")
181
+ for query in config.queries:
182
+ tree.add(query.name)
183
+ console = rich.get_console()
184
+ console.print(tree)
185
+
186
+
187
+ def view_query(config_path: Path, query_name: str) -> None:
188
+ config = load_yaml_config(config_path)
189
+ query_config = next(filter(lambda x: x.name == query_name, config.queries))
190
+ sql_query = query_config.sql
191
+ tables_dataset = {
192
+ table_config.name: load_table(table_config).dataset()
193
+ for table_config in config.tables
194
+ }
195
+
196
+ out: rich.jupyter.JupyterMixin
197
+ try:
198
+ results = execute_query(tables_dataset, sql_query)
199
+ out = rich.table.Table()
200
+ for column in results.columns:
201
+ out.add_column(column)
202
+ for value_list in results.values.tolist():
203
+ row = [str(x) for x in value_list]
204
+ out.add_row(*row)
205
+ except ValueError as e:
206
+ out = rich.panel.Panel.fit(f"[red]{e}")
207
+
208
+ console = rich.get_console()
209
+ console.print(out)
210
+
211
+
152
212
  def cli() -> None:
153
213
  parser = argparse.ArgumentParser(
154
214
  "laketower", formatter_class=argparse.ArgumentDefaultsHelpFormatter
@@ -212,6 +272,17 @@ def cli() -> None:
212
272
  parser_tables_history.add_argument("table", help="Name of the table")
213
273
  parser_tables_history.set_defaults(func=lambda x: table_history(x.config, x.table))
214
274
 
275
+ parser_tables_statistics = subsparsers_tables.add_parser(
276
+ "statistics", help="Display summary statistics of a given table schema"
277
+ )
278
+ parser_tables_statistics.add_argument("table", help="Name of the table")
279
+ parser_tables_statistics.add_argument(
280
+ "--version", type=int, help="Time-travel to table revision number"
281
+ )
282
+ parser_tables_statistics.set_defaults(
283
+ func=lambda x: table_statistics(x.config, x.table, x.version)
284
+ )
285
+
215
286
  parser_tables_view = subsparsers_tables.add_parser(
216
287
  "view", help="View a given table"
217
288
  )
@@ -242,5 +313,19 @@ def cli() -> None:
242
313
  parser_tables_query.add_argument("sql", help="SQL query to execute")
243
314
  parser_tables_query.set_defaults(func=lambda x: query_table(x.config, x.sql))
244
315
 
316
+ parser_queries = subparsers.add_parser("queries", help="Work with queries")
317
+ subsparsers_queries = parser_queries.add_subparsers(required=True)
318
+
319
+ parser_queries_list = subsparsers_queries.add_parser(
320
+ "list", help="List all registered queries"
321
+ )
322
+ parser_queries_list.set_defaults(func=lambda x: list_queries(x.config))
323
+
324
+ parser_queries_view = subsparsers_queries.add_parser(
325
+ "view", help="View a given query"
326
+ )
327
+ parser_queries_view.add_argument("query", help="Name of the query")
328
+ parser_queries_view.set_defaults(func=lambda x: view_query(x.config, x.query))
329
+
245
330
  args = parser.parse_args()
246
331
  args.func(args)
@@ -29,6 +29,7 @@ class ConfigTable(pydantic.BaseModel):
29
29
 
30
30
  class ConfigQuery(pydantic.BaseModel):
31
31
  name: str
32
+ title: str
32
33
  sql: str
33
34
 
34
35
 
@@ -38,6 +39,7 @@ class ConfigDashboard(pydantic.BaseModel):
38
39
 
39
40
  class Config(pydantic.BaseModel):
40
41
  tables: list[ConfigTable] = []
42
+ queries: list[ConfigQuery] = []
41
43
 
42
44
 
43
45
  def load_yaml_config(config_path: Path) -> Config:
@@ -9,6 +9,7 @@ import pyarrow.dataset as padataset
9
9
  import pydantic
10
10
  import sqlglot
11
11
  import sqlglot.dialects.duckdb
12
+ import sqlglot.expressions
12
13
 
13
14
  from laketower.config import ConfigTable, TableFormats
14
15
 
@@ -120,6 +121,12 @@ def generate_table_query(
120
121
  )
121
122
 
122
123
 
124
+ def generate_table_statistics_query(table_name: str) -> str:
125
+ return (
126
+ f"SELECT column_name, count, avg, std, min, max FROM (SUMMARIZE {table_name})" # nosec B608
127
+ )
128
+
129
+
123
130
  def execute_query(
124
131
  tables_datasets: dict[str, padataset.Dataset], sql_query: str
125
132
  ) -> pd.DataFrame:
@@ -19,7 +19,14 @@
19
19
  <a class="sidebar-brand" href="/">
20
20
  Laketower
21
21
  </a>
22
- <button type="button" class="btn-close d-md-none" data-bs-dismiss="offcanvas" aria-label="Close" data-bs-target="#sidebar"></button>
22
+ <button
23
+ type="button"
24
+ class="btn-close d-md-none"
25
+ data-bs-dismiss="offcanvas"
26
+ aria-label="Close"
27
+ data-bs-target="#sidebar"
28
+ >
29
+ </button>
23
30
  </div>
24
31
  <div class="offcanvas-body">
25
32
  <ul class="sidebar-nav">
@@ -30,12 +37,36 @@
30
37
  {% for table in tables %}
31
38
  {% set table_url = '/tables/' + table.name %}
32
39
  <li class="nav-item">
33
- <a class="text-truncate nav-link{% if request.url.path.startswith(table_url) %} active{% endif %}" href="{{ table_url }}" aria-current="page">
40
+ <a
41
+ class="text-truncate nav-link{% if request.url.path.startswith(table_url) %} active{% endif %}"
42
+ href="{{ table_url }}"
43
+ aria-current="page"
44
+ >
34
45
  <i class="bi-table" aria-hidden="true"></i>
35
46
  {{ table.name }}
36
47
  </a>
37
48
  </li>
38
49
  {% endfor %}
50
+
51
+ <li><hr class="sidebar-divider"></li>
52
+
53
+ <li>
54
+ <h6 class="sidebar-header">Queries</h6>
55
+ </li>
56
+ <li><hr class="sidebar-divider"></li>
57
+ {% for query in queries %}
58
+ {% set query_url = '/queries/' + query.name + '/view' %}
59
+ <li class="nav-item">
60
+ <a
61
+ class="text-truncate nav-link{% if request.url.path.startswith(query_url) %} active{% endif %}"
62
+ href="{{ query_url }}"
63
+ aria-current="page"
64
+ >
65
+ <i class="bi-code-square" aria-hidden="true"></i>
66
+ {{ query.title }}
67
+ </a>
68
+ </li>
69
+ {% endfor %}
39
70
  </ul>
40
71
  </div>
41
72
  </nav>
@@ -0,0 +1,60 @@
1
+ {% extends "_base.html" %}
2
+
3
+ {% block body %}
4
+ <div class="row">
5
+ <div class="col">
6
+ <h2 class="mb-3">{{ query.title }}</h2>
7
+
8
+ <form action="{{ request.url.path }}" method="get">
9
+ <div class="mb-3">
10
+ <textarea disabled name="sql" rows="5" class="form-control">{{ query.sql }}</textarea>
11
+ </div>
12
+
13
+ <div class="mb-3">
14
+ <div class="d-flex justify-content-end">
15
+ <div class="row">
16
+ <div class="col">
17
+ <a href="/tables/query?sql={{ query.sql | urlencode }}" class="btn btn-secondary" type="button" >
18
+ <i class="bi-code" aria-hidden="true"></i> Edit SQL
19
+ </a>
20
+ </div>
21
+
22
+ <div class="col-auto">
23
+ <button type="submit" class="btn btn-primary">
24
+ <i class="bi-lightning" aria-hidden="true"></i> Execute
25
+ </button>
26
+ </div>
27
+ </div>
28
+ </div>
29
+ </div>
30
+ </form>
31
+
32
+ {% if error %}
33
+ <div class="alert alert-danger" role="alert">
34
+ {{ error.message }}
35
+ </div>
36
+ {% else %}
37
+ <div class="table-responsive">
38
+ <table class="table table-sm table-bordered table-striped table-hover">
39
+ <thead>
40
+ <tr>
41
+ {% for column in query_results.columns %}
42
+ <th>{{ column }}</th>
43
+ {% endfor %}
44
+ </tr>
45
+ </thead>
46
+ <tbody class="table-group-divider">
47
+ {% for row in query_results.to_numpy().tolist() %}
48
+ <tr>
49
+ {% for col in row %}
50
+ <td>{{ col }}</td>
51
+ {% endfor %}
52
+ </tr>
53
+ {% endfor %}
54
+ </tbody>
55
+ </table>
56
+ </div>
57
+ {% endif %}
58
+ </div>
59
+ </div>
60
+ {% endblock %}