laketower 0.1.0__tar.gz → 0.2.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of laketower might be problematic. Click here for more details.
- {laketower-0.1.0 → laketower-0.2.0}/.github/workflows/ci-cd.yml +42 -4
- laketower-0.2.0/CHANGELOG.md +42 -0
- laketower-0.1.0/README.md → laketower-0.2.0/PKG-INFO +75 -4
- laketower-0.1.0/PKG-INFO → laketower-0.2.0/README.md +36 -27
- laketower-0.2.0/laketower/__about__.py +1 -0
- {laketower-0.1.0 → laketower-0.2.0}/laketower/cli.py +45 -177
- laketower-0.2.0/laketower/config.py +45 -0
- laketower-0.2.0/laketower/tables.py +136 -0
- laketower-0.2.0/laketower/templates/_base.html +72 -0
- laketower-0.2.0/laketower/templates/index.html +4 -0
- laketower-0.2.0/laketower/templates/tables/_macros.html +13 -0
- laketower-0.2.0/laketower/templates/tables/history.html +42 -0
- laketower-0.2.0/laketower/templates/tables/index.html +84 -0
- laketower-0.2.0/laketower/templates/tables/query.html +41 -0
- laketower-0.2.0/laketower/templates/tables/view.html +96 -0
- laketower-0.2.0/laketower/web.py +160 -0
- {laketower-0.1.0 → laketower-0.2.0}/pyproject.toml +21 -2
- laketower-0.2.0/tests/__init__.py +0 -0
- laketower-0.2.0/tests/conftest.py +53 -0
- {laketower-0.1.0 → laketower-0.2.0}/tests/test_cli.py +46 -71
- laketower-0.2.0/tests/test_web.py +290 -0
- {laketower-0.1.0 → laketower-0.2.0}/uv.lock +228 -2
- laketower-0.1.0/CHANGELOG.md +0 -24
- laketower-0.1.0/laketower/__about__.py +0 -1
- {laketower-0.1.0 → laketower-0.2.0}/.gitignore +0 -0
- {laketower-0.1.0 → laketower-0.2.0}/.python-version +0 -0
- {laketower-0.1.0 → laketower-0.2.0}/LICENSE.md +0 -0
- {laketower-0.1.0 → laketower-0.2.0}/demo/generate.py +0 -0
- {laketower-0.1.0 → laketower-0.2.0}/demo/laketower.yml +0 -0
- {laketower-0.1.0 → laketower-0.2.0}/demo/sample_table/_delta_log/00000000000000000000.json +0 -0
- {laketower-0.1.0 → laketower-0.2.0}/demo/sample_table/_delta_log/00000000000000000001.json +0 -0
- {laketower-0.1.0 → laketower-0.2.0}/demo/sample_table/_delta_log/00000000000000000002.json +0 -0
- {laketower-0.1.0 → laketower-0.2.0}/demo/sample_table/_delta_log/00000000000000000003.json +0 -0
- {laketower-0.1.0 → laketower-0.2.0}/demo/sample_table/part-00001-1a31a393-6db6-4d1a-bf4e-81ea061ff8cd-c000.snappy.parquet +0 -0
- {laketower-0.1.0 → laketower-0.2.0}/demo/sample_table/part-00001-5af77102-9207-4c89-aaf6-37e1f815ec26-c000.snappy.parquet +0 -0
- {laketower-0.1.0 → laketower-0.2.0}/demo/sample_table/part-00001-b11bab55-43d0-4d05-ae88-5b9481ae57db-c000.snappy.parquet +0 -0
- {laketower-0.1.0 → laketower-0.2.0}/demo/weather/_delta_log/00000000000000000000.json +0 -0
- {laketower-0.1.0 → laketower-0.2.0}/demo/weather/_delta_log/00000000000000000001.json +0 -0
- {laketower-0.1.0 → laketower-0.2.0}/demo/weather/_delta_log/00000000000000000002.json +0 -0
- {laketower-0.1.0 → laketower-0.2.0}/demo/weather/part-00001-2323b963-be56-44e0-8c10-e237e7e6d4b9-c000.snappy.parquet +0 -0
- {laketower-0.1.0 → laketower-0.2.0}/demo/weather/part-00001-6360cbf8-f8a9-475f-8729-6f20b4ca64a9-c000.snappy.parquet +0 -0
- {laketower-0.1.0 → laketower-0.2.0}/laketower/__init__.py +0 -0
- {laketower-0.1.0 → laketower-0.2.0}/laketower/__main__.py +0 -0
- /laketower-0.1.0/tests/__init__.py → /laketower-0.2.0/laketower/static/.gitkeep +0 -0
- {laketower-0.1.0 → laketower-0.2.0}/tasks.py +0 -0
|
@@ -151,10 +151,10 @@ jobs:
|
|
|
151
151
|
with:
|
|
152
152
|
repository-url: https://test.pypi.org/legacy/
|
|
153
153
|
skip-existing: true
|
|
154
|
-
- name: Install uv
|
|
155
|
-
|
|
156
|
-
- name: Validate package is available with uvx
|
|
157
|
-
|
|
154
|
+
# - name: Install uv
|
|
155
|
+
# uses: astral-sh/setup-uv@v5
|
|
156
|
+
# - name: Validate package is available with uvx
|
|
157
|
+
# run: uvx --index https://test.pypi.org/simple/ --index-strategy unsafe-best-match laketower --version
|
|
158
158
|
|
|
159
159
|
pypi-publish:
|
|
160
160
|
name: Upload release to PyPI
|
|
@@ -178,3 +178,41 @@ jobs:
|
|
|
178
178
|
uses: astral-sh/setup-uv@v5
|
|
179
179
|
- name: Validate package is available with uvx
|
|
180
180
|
run: uvx laketower --version
|
|
181
|
+
|
|
182
|
+
release-publish:
|
|
183
|
+
name: Publish release as GitHub Release
|
|
184
|
+
runs-on: ubuntu-latest
|
|
185
|
+
needs: build
|
|
186
|
+
if: ${{ contains(github.ref, 'tags') }}
|
|
187
|
+
env:
|
|
188
|
+
TAG_NAME: "${{ github.ref_name }}"
|
|
189
|
+
RELEASE_NOTES_MD_FILE: "release_notes.md"
|
|
190
|
+
permissions:
|
|
191
|
+
contents: write
|
|
192
|
+
steps:
|
|
193
|
+
- uses: actions/checkout@v4
|
|
194
|
+
- name: Download package build artifacts
|
|
195
|
+
uses: actions/download-artifact@v4
|
|
196
|
+
with:
|
|
197
|
+
name: build
|
|
198
|
+
path: dist/
|
|
199
|
+
- name: Install uv
|
|
200
|
+
uses: astral-sh/setup-uv@v5
|
|
201
|
+
with:
|
|
202
|
+
python-version: '3.13'
|
|
203
|
+
enable-cache: true
|
|
204
|
+
cache-dependency-glob: "uv.lock"
|
|
205
|
+
- name: Extract changelog notes
|
|
206
|
+
run: |
|
|
207
|
+
uvx keepachangelog show ${TAG_NAME} >> "${RELEASE_NOTES_MD_FILE}"
|
|
208
|
+
cat "${RELEASE_NOTES_MD_FILE}" >> "${GITHUB_STEP_SUMMARY}"
|
|
209
|
+
- name: Create GitHub Release from changelog
|
|
210
|
+
env:
|
|
211
|
+
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
|
212
|
+
run: |
|
|
213
|
+
gh release create "${TAG_NAME}" \
|
|
214
|
+
--repo "${GITHUB_REPOSITORY}" \
|
|
215
|
+
--title "${TAG_NAME}" \
|
|
216
|
+
--notes-file "${RELEASE_NOTES_MD_FILE}" \
|
|
217
|
+
--verify-tag \
|
|
218
|
+
dist/*
|
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
# Changelog
|
|
2
|
+
|
|
3
|
+
All notable changes to this project will be documented in this file.
|
|
4
|
+
|
|
5
|
+
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
|
|
6
|
+
and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
|
|
7
|
+
|
|
8
|
+
## [Unreleased]
|
|
9
|
+
|
|
10
|
+
## [0.2.0] - 2025-02-25
|
|
11
|
+
Introducing the Laketower web application!
|
|
12
|
+
|
|
13
|
+
### Added
|
|
14
|
+
- `web` module
|
|
15
|
+
- List all registered tables
|
|
16
|
+
- Display table overview (metadata and schema)
|
|
17
|
+
- Display table history
|
|
18
|
+
- View a given table with simple query builder
|
|
19
|
+
- Query all registered tables with DuckDB SQL dialect
|
|
20
|
+
- CLI: add `tables view --version` argument to time-travel table version
|
|
21
|
+
|
|
22
|
+
### Fixed
|
|
23
|
+
- Delta tables metadata compatibility when name and/or description is missing
|
|
24
|
+
- Delta tables history compatibility when created with Spark
|
|
25
|
+
- CLI: show default argument values in help
|
|
26
|
+
|
|
27
|
+
## [0.1.0] - 2025-02-15
|
|
28
|
+
Initial release of `laketower`.
|
|
29
|
+
|
|
30
|
+
### Added
|
|
31
|
+
- `cli` module
|
|
32
|
+
- Validate YAML configuration
|
|
33
|
+
- List all registered tables
|
|
34
|
+
- Display a given table metadata
|
|
35
|
+
- Display a given table schema
|
|
36
|
+
- Display a given table history
|
|
37
|
+
- View a given table with simple query builder
|
|
38
|
+
- Query all registered tables with DuckDB SQL dialect
|
|
39
|
+
|
|
40
|
+
[Unreleased]: https://github.com/datalpia/laketower/compare/0.2.0...HEAD
|
|
41
|
+
[0.2.0]: https://github.com/datalpia/laketower/compare/0.1.0...0.2.0
|
|
42
|
+
[0.1.0]: https://github.com/datalpia/laketower/releases/tag/0.1.0
|
|
@@ -1,10 +1,50 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: laketower
|
|
3
|
+
Version: 0.2.0
|
|
4
|
+
Summary: Oversee your lakehouse
|
|
5
|
+
Project-URL: Repository, https://github.com/datalpia/laketower
|
|
6
|
+
Project-URL: Issues, https://github.com/datalpia/laketower/issues
|
|
7
|
+
Project-URL: Changelog, https://github.com/datalpia/laketower/blob/master/CHANGELOG.md
|
|
8
|
+
Author-email: Romain Clement <git@romain-clement.net>
|
|
9
|
+
License: AGPL-3.0-or-later
|
|
10
|
+
License-File: LICENSE.md
|
|
11
|
+
Keywords: data,delta-lake,lakehouse,sql
|
|
12
|
+
Classifier: Development Status :: 2 - Pre-Alpha
|
|
13
|
+
Classifier: Intended Audience :: Developers
|
|
14
|
+
Classifier: Intended Audience :: End Users/Desktop
|
|
15
|
+
Classifier: Intended Audience :: Information Technology
|
|
16
|
+
Classifier: Intended Audience :: Other Audience
|
|
17
|
+
Classifier: Programming Language :: Python :: 3.9
|
|
18
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
19
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
20
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
21
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
22
|
+
Classifier: Topic :: Database
|
|
23
|
+
Classifier: Topic :: Software Development
|
|
24
|
+
Classifier: Topic :: Utilities
|
|
25
|
+
Requires-Python: <3.14,>=3.9
|
|
26
|
+
Requires-Dist: deltalake
|
|
27
|
+
Requires-Dist: duckdb
|
|
28
|
+
Requires-Dist: fastapi
|
|
29
|
+
Requires-Dist: jinja2>=3
|
|
30
|
+
Requires-Dist: pandas
|
|
31
|
+
Requires-Dist: pyarrow!=19.0.0
|
|
32
|
+
Requires-Dist: pydantic-settings>=2
|
|
33
|
+
Requires-Dist: pydantic>=2
|
|
34
|
+
Requires-Dist: pyyaml
|
|
35
|
+
Requires-Dist: rich
|
|
36
|
+
Requires-Dist: sqlglot
|
|
37
|
+
Requires-Dist: uvicorn
|
|
38
|
+
Description-Content-Type: text/markdown
|
|
39
|
+
|
|
1
40
|
# Laketower
|
|
2
41
|
|
|
3
42
|
> Oversee your lakehouse
|
|
4
43
|
|
|
5
44
|
[](https://pypi.org/project/laketower/)
|
|
45
|
+
[](https://pypi.org/project/laketower/)
|
|
6
46
|
[](https://github.com/datalpia/laketower/actions/workflows/ci-cd.yml)
|
|
7
|
-
[](https://github.com/datalpia/laketower/blob/main/LICENSE)
|
|
47
|
+
[](https://github.com/datalpia/laketower/blob/main/LICENSE.md)
|
|
8
48
|
|
|
9
49
|
Utility application to explore and manage tables in your data lakehouse, especially tailored for data pipelines local development.
|
|
10
50
|
|
|
@@ -17,6 +57,7 @@ Utility application to explore and manage tables in your data lakehouse, especia
|
|
|
17
57
|
- View table content with a simple query builder
|
|
18
58
|
- Query all registered tables with DuckDB SQL dialect
|
|
19
59
|
- Static and versionable YAML configuration
|
|
60
|
+
- Web application
|
|
20
61
|
- CLI application
|
|
21
62
|
|
|
22
63
|
## Installation
|
|
@@ -67,21 +108,30 @@ tables:
|
|
|
67
108
|
format: delta
|
|
68
109
|
```
|
|
69
110
|
|
|
111
|
+
### Web Application
|
|
112
|
+
|
|
113
|
+
The easiest way to get started is to launch the Laketower web application:
|
|
114
|
+
|
|
115
|
+
```bash
|
|
116
|
+
$ laketower -c demo/laketower.yml web
|
|
117
|
+
```
|
|
118
|
+
|
|
70
119
|
### CLI
|
|
71
120
|
|
|
72
121
|
Laketower provides a CLI interface:
|
|
73
122
|
|
|
74
123
|
```bash
|
|
75
124
|
$ laketower --help
|
|
76
|
-
usage: laketower [-h] [--version] [--config CONFIG] {config,tables} ...
|
|
125
|
+
usage: laketower [-h] [--version] [--config CONFIG] {web,config,tables} ...
|
|
77
126
|
|
|
78
127
|
options:
|
|
79
128
|
-h, --help show this help message and exit
|
|
80
129
|
--version show program's version number and exit
|
|
81
|
-
--config, -c CONFIG Path to the Laketower YAML configuration file
|
|
130
|
+
--config, -c CONFIG Path to the Laketower YAML configuration file (default: laketower.yml)
|
|
82
131
|
|
|
83
132
|
commands:
|
|
84
|
-
{config,tables}
|
|
133
|
+
{web,config,tables}
|
|
134
|
+
web Launch the web application
|
|
85
135
|
config Work with configuration
|
|
86
136
|
tables Work with tables
|
|
87
137
|
```
|
|
@@ -205,6 +255,7 @@ Optional arguments:
|
|
|
205
255
|
- `--sort-asc <col>`: sort by a column name in ascending order
|
|
206
256
|
- `--sort-desc <col>`: sort by a column name in descending order
|
|
207
257
|
- `--limit <num>` (default 10): limit the number of rows
|
|
258
|
+
- `--version`: time-travel to table revision number
|
|
208
259
|
|
|
209
260
|
```bash
|
|
210
261
|
$ laketower -c demo/laketower.yml tables view weather
|
|
@@ -239,6 +290,26 @@ $ laketower -c demo/laketower.yml tables view weather --cols time city temperatu
|
|
|
239
290
|
└───────────────────────────┴──────────┴───────────────────┘
|
|
240
291
|
```
|
|
241
292
|
|
|
293
|
+
```bash
|
|
294
|
+
$ laketower -c demo/laketower.yml tables view weather --version 1
|
|
295
|
+
|
|
296
|
+
┏━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━┓
|
|
297
|
+
┃ time ┃ city ┃ temperature_2m ┃ relative_humidity_2m ┃ wind_speed_10m ┃
|
|
298
|
+
┡━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━┩
|
|
299
|
+
│ 2025-01-26 01:00:00+01:00 │ Grenoble │ 7.0 │ 87.0 │ 8.899999618530273 │
|
|
300
|
+
│ 2025-01-26 02:00:00+01:00 │ Grenoble │ 6.099999904632568 │ 87.0 │ 6.199999809265137 │
|
|
301
|
+
│ 2025-01-26 03:00:00+01:00 │ Grenoble │ 6.0 │ 86.0 │ 2.700000047683716 │
|
|
302
|
+
│ 2025-01-26 04:00:00+01:00 │ Grenoble │ 6.099999904632568 │ 82.0 │ 3.0999999046325684 │
|
|
303
|
+
│ 2025-01-26 05:00:00+01:00 │ Grenoble │ 5.5 │ 87.0 │ 3.299999952316284 │
|
|
304
|
+
│ 2025-01-26 06:00:00+01:00 │ Grenoble │ 5.199999809265137 │ 91.0 │ 2.200000047683716 │
|
|
305
|
+
│ 2025-01-26 07:00:00+01:00 │ Grenoble │ 4.800000190734863 │ 86.0 │ 3.0 │
|
|
306
|
+
│ 2025-01-26 08:00:00+01:00 │ Grenoble │ 4.900000095367432 │ 83.0 │ 1.100000023841858 │
|
|
307
|
+
│ 2025-01-26 09:00:00+01:00 │ Grenoble │ 4.0 │ 92.0 │ 3.0999999046325684 │
|
|
308
|
+
│ 2025-01-26 10:00:00+01:00 │ Grenoble │ 5.0 │ 86.0 │ 6.400000095367432 │
|
|
309
|
+
└───────────────────────────┴──────────┴───────────────────┴──────────────────────┴────────────────────┘
|
|
310
|
+
```
|
|
311
|
+
|
|
312
|
+
|
|
242
313
|
#### Query all registered tables
|
|
243
314
|
|
|
244
315
|
Query any registered tables using DuckDB SQL dialect!
|
|
@@ -1,33 +1,11 @@
|
|
|
1
|
-
Metadata-Version: 2.4
|
|
2
|
-
Name: laketower
|
|
3
|
-
Version: 0.1.0
|
|
4
|
-
Summary: Oversee your lakehouse
|
|
5
|
-
Author-email: Romain Clement <git@romain-clement.net>
|
|
6
|
-
License: AGPL-3.0-or-later
|
|
7
|
-
License-File: LICENSE.md
|
|
8
|
-
Classifier: Development Status :: 2 - Pre-Alpha
|
|
9
|
-
Classifier: Intended Audience :: Developers
|
|
10
|
-
Classifier: Intended Audience :: Information Technology
|
|
11
|
-
Classifier: Topic :: Software Development
|
|
12
|
-
Classifier: Topic :: Utilities
|
|
13
|
-
Requires-Python: <3.14,>=3.9
|
|
14
|
-
Requires-Dist: deltalake
|
|
15
|
-
Requires-Dist: duckdb
|
|
16
|
-
Requires-Dist: pandas
|
|
17
|
-
Requires-Dist: pyarrow<19
|
|
18
|
-
Requires-Dist: pydantic
|
|
19
|
-
Requires-Dist: pyyaml
|
|
20
|
-
Requires-Dist: rich
|
|
21
|
-
Requires-Dist: sqlglot
|
|
22
|
-
Description-Content-Type: text/markdown
|
|
23
|
-
|
|
24
1
|
# Laketower
|
|
25
2
|
|
|
26
3
|
> Oversee your lakehouse
|
|
27
4
|
|
|
28
5
|
[](https://pypi.org/project/laketower/)
|
|
6
|
+
[](https://pypi.org/project/laketower/)
|
|
29
7
|
[](https://github.com/datalpia/laketower/actions/workflows/ci-cd.yml)
|
|
30
|
-
[](https://github.com/datalpia/laketower/blob/main/LICENSE)
|
|
8
|
+
[](https://github.com/datalpia/laketower/blob/main/LICENSE.md)
|
|
31
9
|
|
|
32
10
|
Utility application to explore and manage tables in your data lakehouse, especially tailored for data pipelines local development.
|
|
33
11
|
|
|
@@ -40,6 +18,7 @@ Utility application to explore and manage tables in your data lakehouse, especia
|
|
|
40
18
|
- View table content with a simple query builder
|
|
41
19
|
- Query all registered tables with DuckDB SQL dialect
|
|
42
20
|
- Static and versionable YAML configuration
|
|
21
|
+
- Web application
|
|
43
22
|
- CLI application
|
|
44
23
|
|
|
45
24
|
## Installation
|
|
@@ -90,21 +69,30 @@ tables:
|
|
|
90
69
|
format: delta
|
|
91
70
|
```
|
|
92
71
|
|
|
72
|
+
### Web Application
|
|
73
|
+
|
|
74
|
+
The easiest way to get started is to launch the Laketower web application:
|
|
75
|
+
|
|
76
|
+
```bash
|
|
77
|
+
$ laketower -c demo/laketower.yml web
|
|
78
|
+
```
|
|
79
|
+
|
|
93
80
|
### CLI
|
|
94
81
|
|
|
95
82
|
Laketower provides a CLI interface:
|
|
96
83
|
|
|
97
84
|
```bash
|
|
98
85
|
$ laketower --help
|
|
99
|
-
usage: laketower [-h] [--version] [--config CONFIG] {config,tables} ...
|
|
86
|
+
usage: laketower [-h] [--version] [--config CONFIG] {web,config,tables} ...
|
|
100
87
|
|
|
101
88
|
options:
|
|
102
89
|
-h, --help show this help message and exit
|
|
103
90
|
--version show program's version number and exit
|
|
104
|
-
--config, -c CONFIG Path to the Laketower YAML configuration file
|
|
91
|
+
--config, -c CONFIG Path to the Laketower YAML configuration file (default: laketower.yml)
|
|
105
92
|
|
|
106
93
|
commands:
|
|
107
|
-
{config,tables}
|
|
94
|
+
{web,config,tables}
|
|
95
|
+
web Launch the web application
|
|
108
96
|
config Work with configuration
|
|
109
97
|
tables Work with tables
|
|
110
98
|
```
|
|
@@ -228,6 +216,7 @@ Optional arguments:
|
|
|
228
216
|
- `--sort-asc <col>`: sort by a column name in ascending order
|
|
229
217
|
- `--sort-desc <col>`: sort by a column name in descending order
|
|
230
218
|
- `--limit <num>` (default 10): limit the number of rows
|
|
219
|
+
- `--version`: time-travel to table revision number
|
|
231
220
|
|
|
232
221
|
```bash
|
|
233
222
|
$ laketower -c demo/laketower.yml tables view weather
|
|
@@ -262,6 +251,26 @@ $ laketower -c demo/laketower.yml tables view weather --cols time city temperatu
|
|
|
262
251
|
└───────────────────────────┴──────────┴───────────────────┘
|
|
263
252
|
```
|
|
264
253
|
|
|
254
|
+
```bash
|
|
255
|
+
$ laketower -c demo/laketower.yml tables view weather --version 1
|
|
256
|
+
|
|
257
|
+
┏━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━┓
|
|
258
|
+
┃ time ┃ city ┃ temperature_2m ┃ relative_humidity_2m ┃ wind_speed_10m ┃
|
|
259
|
+
┡━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━┩
|
|
260
|
+
│ 2025-01-26 01:00:00+01:00 │ Grenoble │ 7.0 │ 87.0 │ 8.899999618530273 │
|
|
261
|
+
│ 2025-01-26 02:00:00+01:00 │ Grenoble │ 6.099999904632568 │ 87.0 │ 6.199999809265137 │
|
|
262
|
+
│ 2025-01-26 03:00:00+01:00 │ Grenoble │ 6.0 │ 86.0 │ 2.700000047683716 │
|
|
263
|
+
│ 2025-01-26 04:00:00+01:00 │ Grenoble │ 6.099999904632568 │ 82.0 │ 3.0999999046325684 │
|
|
264
|
+
│ 2025-01-26 05:00:00+01:00 │ Grenoble │ 5.5 │ 87.0 │ 3.299999952316284 │
|
|
265
|
+
│ 2025-01-26 06:00:00+01:00 │ Grenoble │ 5.199999809265137 │ 91.0 │ 2.200000047683716 │
|
|
266
|
+
│ 2025-01-26 07:00:00+01:00 │ Grenoble │ 4.800000190734863 │ 86.0 │ 3.0 │
|
|
267
|
+
│ 2025-01-26 08:00:00+01:00 │ Grenoble │ 4.900000095367432 │ 83.0 │ 1.100000023841858 │
|
|
268
|
+
│ 2025-01-26 09:00:00+01:00 │ Grenoble │ 4.0 │ 92.0 │ 3.0999999046325684 │
|
|
269
|
+
│ 2025-01-26 10:00:00+01:00 │ Grenoble │ 5.0 │ 86.0 │ 6.400000095367432 │
|
|
270
|
+
└───────────────────────────┴──────────┴───────────────────┴──────────────────────┴────────────────────┘
|
|
271
|
+
```
|
|
272
|
+
|
|
273
|
+
|
|
265
274
|
#### Query all registered tables
|
|
266
275
|
|
|
267
276
|
Query any registered tables using DuckDB SQL dialect!
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
__version__ = "0.2.0"
|
|
@@ -1,178 +1,24 @@
|
|
|
1
1
|
from __future__ import annotations
|
|
2
2
|
|
|
3
3
|
import argparse
|
|
4
|
-
import
|
|
5
|
-
from datetime import datetime, timezone
|
|
4
|
+
import os
|
|
6
5
|
from pathlib import Path
|
|
7
|
-
from typing import Any
|
|
8
6
|
|
|
9
|
-
import
|
|
10
|
-
import duckdb
|
|
11
|
-
import pandas as pd
|
|
12
|
-
import pyarrow as pa
|
|
13
|
-
import pydantic
|
|
7
|
+
import rich.jupyter
|
|
14
8
|
import rich.panel
|
|
15
9
|
import rich.table
|
|
16
10
|
import rich.text
|
|
17
11
|
import rich.tree
|
|
18
|
-
import
|
|
19
|
-
import sqlglot.dialects
|
|
20
|
-
import sqlglot.dialects.duckdb
|
|
21
|
-
import sqlglot.generator
|
|
22
|
-
import yaml
|
|
12
|
+
import uvicorn
|
|
23
13
|
|
|
24
14
|
from laketower.__about__ import __version__
|
|
15
|
+
from laketower.config import load_yaml_config
|
|
16
|
+
from laketower.tables import execute_query, generate_table_query, load_table
|
|
25
17
|
|
|
26
18
|
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
class ConfigTable(pydantic.BaseModel):
|
|
32
|
-
name: str
|
|
33
|
-
uri: str
|
|
34
|
-
table_format: TableFormats = pydantic.Field(alias="format")
|
|
35
|
-
|
|
36
|
-
@pydantic.model_validator(mode="after")
|
|
37
|
-
def check_table(self) -> "ConfigTable":
|
|
38
|
-
def check_delta_table(table_uri: str) -> None:
|
|
39
|
-
if not deltalake.DeltaTable.is_deltatable(table_uri):
|
|
40
|
-
raise ValueError(f"{table_uri} is not a valid Delta table")
|
|
41
|
-
|
|
42
|
-
format_check = {TableFormats.delta: check_delta_table}
|
|
43
|
-
format_check[self.table_format](self.uri)
|
|
44
|
-
|
|
45
|
-
return self
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
class ConfigQuery(pydantic.BaseModel):
|
|
49
|
-
name: str
|
|
50
|
-
sql: str
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
class ConfigDashboard(pydantic.BaseModel):
|
|
54
|
-
name: str
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
class Config(pydantic.BaseModel):
|
|
58
|
-
tables: list[ConfigTable] = []
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
def load_yaml_config(config_path: Path) -> Config:
|
|
62
|
-
config_dict = yaml.safe_load(config_path.read_text())
|
|
63
|
-
return Config.model_validate(config_dict)
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
class TableMetadata(pydantic.BaseModel):
|
|
67
|
-
table_format: TableFormats
|
|
68
|
-
name: str
|
|
69
|
-
description: str
|
|
70
|
-
uri: str
|
|
71
|
-
id: str
|
|
72
|
-
version: int
|
|
73
|
-
created_at: datetime
|
|
74
|
-
partitions: list[str]
|
|
75
|
-
configuration: dict[str, str]
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
class TableRevision(pydantic.BaseModel):
|
|
79
|
-
version: int
|
|
80
|
-
timestamp: datetime
|
|
81
|
-
client_version: str
|
|
82
|
-
operation: str
|
|
83
|
-
operation_parameters: dict[str, Any]
|
|
84
|
-
operation_metrics: dict[str, Any]
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
class TableHistory(pydantic.BaseModel):
|
|
88
|
-
revisions: list[TableRevision]
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
def load_table_metadata(table_config: ConfigTable) -> TableMetadata:
|
|
92
|
-
def load_delta_table_metadata(table_config: ConfigTable) -> TableMetadata:
|
|
93
|
-
delta_table = deltalake.DeltaTable(table_config.uri)
|
|
94
|
-
metadata = delta_table.metadata()
|
|
95
|
-
return TableMetadata(
|
|
96
|
-
table_format=table_config.table_format,
|
|
97
|
-
name=metadata.name,
|
|
98
|
-
description=metadata.description,
|
|
99
|
-
uri=delta_table.table_uri,
|
|
100
|
-
id=str(metadata.id),
|
|
101
|
-
version=delta_table.version(),
|
|
102
|
-
created_at=datetime.fromtimestamp(
|
|
103
|
-
metadata.created_time / 1000, tz=timezone.utc
|
|
104
|
-
),
|
|
105
|
-
partitions=metadata.partition_columns,
|
|
106
|
-
configuration=metadata.configuration,
|
|
107
|
-
)
|
|
108
|
-
|
|
109
|
-
format_handler = {TableFormats.delta: load_delta_table_metadata}
|
|
110
|
-
return format_handler[table_config.table_format](table_config)
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
def load_table_schema(table_config: ConfigTable) -> pa.Schema:
|
|
114
|
-
def load_delta_table_schema(table_config: ConfigTable) -> pa.Schema:
|
|
115
|
-
delta_table = deltalake.DeltaTable(table_config.uri)
|
|
116
|
-
return delta_table.schema().to_pyarrow()
|
|
117
|
-
|
|
118
|
-
format_handler = {TableFormats.delta: load_delta_table_schema}
|
|
119
|
-
return format_handler[table_config.table_format](table_config)
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
def load_table_history(table_config: ConfigTable) -> TableHistory:
|
|
123
|
-
def load_delta_table_history(table_config: ConfigTable) -> TableHistory:
|
|
124
|
-
delta_table = deltalake.DeltaTable(table_config.uri)
|
|
125
|
-
delta_history = delta_table.history()
|
|
126
|
-
revisions = [
|
|
127
|
-
TableRevision(
|
|
128
|
-
version=event["version"],
|
|
129
|
-
timestamp=datetime.fromtimestamp(
|
|
130
|
-
event["timestamp"] / 1000, tz=timezone.utc
|
|
131
|
-
),
|
|
132
|
-
client_version=event["clientVersion"],
|
|
133
|
-
operation=event["operation"],
|
|
134
|
-
operation_parameters=event["operationParameters"],
|
|
135
|
-
operation_metrics=event.get("operationMetrics") or {},
|
|
136
|
-
)
|
|
137
|
-
for event in delta_history
|
|
138
|
-
]
|
|
139
|
-
return TableHistory(revisions=revisions)
|
|
140
|
-
|
|
141
|
-
format_handler = {TableFormats.delta: load_delta_table_history}
|
|
142
|
-
return format_handler[table_config.table_format](table_config)
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
def load_table_dataset(table_config: ConfigTable) -> pa.dataset.Dataset:
|
|
146
|
-
def load_delta_table_metadata(table_config: ConfigTable) -> pa.dataset.Dataset:
|
|
147
|
-
delta_table = deltalake.DeltaTable(table_config.uri)
|
|
148
|
-
return delta_table.to_pyarrow_dataset()
|
|
149
|
-
|
|
150
|
-
format_handler = {TableFormats.delta: load_delta_table_metadata}
|
|
151
|
-
return format_handler[table_config.table_format](table_config)
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
def execute_query_table(table_config: ConfigTable, sql_query: str) -> pd.DataFrame:
|
|
155
|
-
table_dataset = load_table_dataset(table_config)
|
|
156
|
-
table_name = table_config.name
|
|
157
|
-
view_name = f"{table_name}_view"
|
|
158
|
-
conn = duckdb.connect()
|
|
159
|
-
conn.register(view_name, table_dataset)
|
|
160
|
-
conn.execute(f"create table {table_name} as select * from {view_name}") # nosec B608
|
|
161
|
-
return conn.execute(sql_query).df()
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
def execute_query(tables_config: list[ConfigTable], sql_query: str) -> pd.DataFrame:
|
|
165
|
-
try:
|
|
166
|
-
conn = duckdb.connect()
|
|
167
|
-
for table_config in tables_config:
|
|
168
|
-
table_dataset = load_table_dataset(table_config)
|
|
169
|
-
table_name = table_config.name
|
|
170
|
-
view_name = f"{table_name}_view"
|
|
171
|
-
conn.register(view_name, table_dataset)
|
|
172
|
-
conn.execute(f"create table {table_name} as select * from {view_name}") # nosec B608
|
|
173
|
-
return conn.execute(sql_query).df()
|
|
174
|
-
except duckdb.Error as e:
|
|
175
|
-
raise ValueError(str(e)) from e
|
|
19
|
+
def run_web(config_path: Path, reload: bool) -> None: # pragma: no cover
|
|
20
|
+
os.environ["LAKETOWER_CONFIG_PATH"] = str(config_path.absolute())
|
|
21
|
+
uvicorn.run("laketower.web:create_app", factory=True, reload=reload)
|
|
176
22
|
|
|
177
23
|
|
|
178
24
|
def validate_config(config_path: Path) -> None:
|
|
@@ -200,7 +46,8 @@ def list_tables(config_path: Path) -> None:
|
|
|
200
46
|
def table_metadata(config_path: Path, table_name: str) -> None:
|
|
201
47
|
config = load_yaml_config(config_path)
|
|
202
48
|
table_config = next(filter(lambda x: x.name == table_name, config.tables))
|
|
203
|
-
|
|
49
|
+
table = load_table(table_config)
|
|
50
|
+
metadata = table.metadata()
|
|
204
51
|
|
|
205
52
|
tree = rich.tree.Tree(table_name)
|
|
206
53
|
tree.add(f"name: {metadata.name}")
|
|
@@ -219,7 +66,8 @@ def table_metadata(config_path: Path, table_name: str) -> None:
|
|
|
219
66
|
def table_schema(config_path: Path, table_name: str) -> None:
|
|
220
67
|
config = load_yaml_config(config_path)
|
|
221
68
|
table_config = next(filter(lambda x: x.name == table_name, config.tables))
|
|
222
|
-
|
|
69
|
+
table = load_table(table_config)
|
|
70
|
+
schema = table.schema()
|
|
223
71
|
|
|
224
72
|
tree = rich.tree.Tree(table_name)
|
|
225
73
|
for field in schema:
|
|
@@ -232,7 +80,8 @@ def table_schema(config_path: Path, table_name: str) -> None:
|
|
|
232
80
|
def table_history(config_path: Path, table_name: str) -> None:
|
|
233
81
|
config = load_yaml_config(config_path)
|
|
234
82
|
table_config = next(filter(lambda x: x.name == table_name, config.tables))
|
|
235
|
-
|
|
83
|
+
table = load_table(table_config)
|
|
84
|
+
history = table.history()
|
|
236
85
|
|
|
237
86
|
tree = rich.tree.Tree(table_name)
|
|
238
87
|
for rev in history.revisions:
|
|
@@ -257,22 +106,21 @@ def view_table(
|
|
|
257
106
|
cols: list[str] | None = None,
|
|
258
107
|
sort_asc: str | None = None,
|
|
259
108
|
sort_desc: str | None = None,
|
|
109
|
+
version: int | None = None,
|
|
260
110
|
) -> None:
|
|
261
111
|
config = load_yaml_config(config_path)
|
|
262
112
|
table_config = next(filter(lambda x: x.name == table_name, config.tables))
|
|
113
|
+
table = load_table(table_config)
|
|
114
|
+
table_dataset = table.dataset(version=version)
|
|
115
|
+
sql_query = generate_table_query(
|
|
116
|
+
table_name, limit=limit, cols=cols, sort_asc=sort_asc, sort_desc=sort_desc
|
|
117
|
+
)
|
|
118
|
+
results = execute_query({table_name: table_dataset}, sql_query)
|
|
263
119
|
|
|
264
|
-
query_expr = sqlglot.select(*(cols or ["*"])).from_(table_name).limit(limit or 10)
|
|
265
|
-
if sort_asc:
|
|
266
|
-
query_expr = query_expr.order_by(f"{sort_asc} asc")
|
|
267
|
-
elif sort_desc:
|
|
268
|
-
query_expr = query_expr.order_by(f"{sort_desc} desc")
|
|
269
|
-
sql_query = sqlglot.Generator(dialect=sqlglot.dialects.DuckDB).generate(query_expr)
|
|
270
|
-
|
|
271
|
-
results = execute_query_table(table_config, sql_query)
|
|
272
120
|
out = rich.table.Table()
|
|
273
121
|
for column in results.columns:
|
|
274
122
|
out.add_column(column)
|
|
275
|
-
for value_list in results.
|
|
123
|
+
for value_list in results.to_numpy().tolist():
|
|
276
124
|
row = [str(x) for x in value_list]
|
|
277
125
|
out.add_row(*row)
|
|
278
126
|
|
|
@@ -282,10 +130,14 @@ def view_table(
|
|
|
282
130
|
|
|
283
131
|
def query_table(config_path: Path, sql_query: str) -> None:
|
|
284
132
|
config = load_yaml_config(config_path)
|
|
133
|
+
tables_dataset = {
|
|
134
|
+
table_config.name: load_table(table_config).dataset()
|
|
135
|
+
for table_config in config.tables
|
|
136
|
+
}
|
|
285
137
|
|
|
286
138
|
out: rich.jupyter.JupyterMixin
|
|
287
139
|
try:
|
|
288
|
-
results = execute_query(
|
|
140
|
+
results = execute_query(tables_dataset, sql_query)
|
|
289
141
|
out = rich.table.Table()
|
|
290
142
|
for column in results.columns:
|
|
291
143
|
out.add_column(column)
|
|
@@ -300,7 +152,9 @@ def query_table(config_path: Path, sql_query: str) -> None:
|
|
|
300
152
|
|
|
301
153
|
|
|
302
154
|
def cli() -> None:
|
|
303
|
-
parser = argparse.ArgumentParser(
|
|
155
|
+
parser = argparse.ArgumentParser(
|
|
156
|
+
"laketower", formatter_class=argparse.ArgumentDefaultsHelpFormatter
|
|
157
|
+
)
|
|
304
158
|
parser.add_argument("--version", action="version", version=__version__)
|
|
305
159
|
parser.add_argument(
|
|
306
160
|
"--config",
|
|
@@ -311,6 +165,17 @@ def cli() -> None:
|
|
|
311
165
|
)
|
|
312
166
|
subparsers = parser.add_subparsers(title="commands", required=True)
|
|
313
167
|
|
|
168
|
+
parser_web = subparsers.add_parser(
|
|
169
|
+
"web", help="Launch the web application", add_help=True
|
|
170
|
+
)
|
|
171
|
+
parser_web.add_argument(
|
|
172
|
+
"--reload",
|
|
173
|
+
help="Reload the web server on changes",
|
|
174
|
+
action="store_true",
|
|
175
|
+
required=False,
|
|
176
|
+
)
|
|
177
|
+
parser_web.set_defaults(func=lambda x: run_web(x.config, x.reload))
|
|
178
|
+
|
|
314
179
|
parser_config = subparsers.add_parser(
|
|
315
180
|
"config", help="Work with configuration", add_help=True
|
|
316
181
|
)
|
|
@@ -364,9 +229,12 @@ def cli() -> None:
|
|
|
364
229
|
parser_tables_view_sort_group.add_argument(
|
|
365
230
|
"--sort-desc", help="Sort by given column in descending order"
|
|
366
231
|
)
|
|
232
|
+
parser_tables_view.add_argument(
|
|
233
|
+
"--version", type=int, help="Time-travel to table revision number"
|
|
234
|
+
)
|
|
367
235
|
parser_tables_view.set_defaults(
|
|
368
236
|
func=lambda x: view_table(
|
|
369
|
-
x.config, x.table, x.limit, x.cols, x.sort_asc, x.sort_desc
|
|
237
|
+
x.config, x.table, x.limit, x.cols, x.sort_asc, x.sort_desc, x.version
|
|
370
238
|
)
|
|
371
239
|
)
|
|
372
240
|
|