laketower 0.5.1__tar.gz → 0.6.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of laketower might be problematic. Click here for more details.
- {laketower-0.5.1 → laketower-0.6.1}/.github/workflows/ci-cd.yml +10 -10
- {laketower-0.5.1 → laketower-0.6.1}/.gitignore +3 -0
- {laketower-0.5.1 → laketower-0.6.1}/CHANGELOG.md +48 -1
- {laketower-0.5.1 → laketower-0.6.1}/PKG-INFO +173 -2
- {laketower-0.5.1 → laketower-0.6.1}/README.md +169 -1
- laketower-0.6.1/demo/laketower.yml +56 -0
- laketower-0.6.1/docs/static/queries_view.png +0 -0
- laketower-0.6.1/docs/static/tables_history.png +0 -0
- laketower-0.6.1/docs/static/tables_import.png +0 -0
- laketower-0.6.1/docs/static/tables_overview.png +0 -0
- laketower-0.6.1/docs/static/tables_query.png +0 -0
- laketower-0.6.1/docs/static/tables_statistics.png +0 -0
- laketower-0.6.1/docs/static/tables_view.png +0 -0
- laketower-0.6.1/laketower/__about__.py +1 -0
- laketower-0.6.1/laketower/cli.py +461 -0
- laketower-0.6.1/laketower/config.py +119 -0
- laketower-0.6.1/laketower/static/editor.bundle.js +27433 -0
- laketower-0.6.1/laketower/static/editor.js +74 -0
- laketower-0.6.1/laketower/static/vendor/bootstrap/bootstrap.bundle.min.js +7 -0
- laketower-0.6.1/laketower/static/vendor/bootstrap-icons/bootstrap-icons.min.css +5 -0
- laketower-0.6.1/laketower/static/vendor/bootstrap-icons/fonts/bootstrap-icons.woff +0 -0
- laketower-0.6.1/laketower/static/vendor/bootstrap-icons/fonts/bootstrap-icons.woff2 +0 -0
- laketower-0.6.1/laketower/static/vendor/halfmoon/halfmoon.min.css +22 -0
- laketower-0.6.1/laketower/static/vendor/halfmoon/halfmoon.modern.css +282 -0
- laketower-0.6.1/laketower/tables.py +311 -0
- {laketower-0.5.1 → laketower-0.6.1}/laketower/templates/_base.html +22 -19
- {laketower-0.5.1 → laketower-0.6.1}/laketower/templates/queries/view.html +24 -4
- {laketower-0.5.1 → laketower-0.6.1}/laketower/templates/tables/_macros.html +3 -0
- {laketower-0.5.1 → laketower-0.6.1}/laketower/templates/tables/history.html +6 -0
- laketower-0.6.1/laketower/templates/tables/import.html +71 -0
- {laketower-0.5.1 → laketower-0.6.1}/laketower/templates/tables/index.html +6 -0
- laketower-0.6.1/laketower/templates/tables/query.html +81 -0
- {laketower-0.5.1 → laketower-0.6.1}/laketower/templates/tables/statistics.html +6 -0
- {laketower-0.5.1 → laketower-0.6.1}/laketower/templates/tables/view.html +6 -0
- laketower-0.6.1/laketower/web.py +422 -0
- laketower-0.6.1/package-lock.json +1139 -0
- laketower-0.6.1/package.json +16 -0
- {laketower-0.5.1 → laketower-0.6.1}/pyproject.toml +15 -10
- {laketower-0.5.1 → laketower-0.6.1}/tasks.py +65 -7
- laketower-0.6.1/tests/conftest.py +119 -0
- {laketower-0.5.1 → laketower-0.6.1}/tests/test_cli.py +530 -28
- laketower-0.6.1/tests/test_config.py +229 -0
- laketower-0.6.1/tests/test_tables.py +111 -0
- laketower-0.6.1/tests/test_web.py +844 -0
- laketower-0.6.1/uv.lock +1812 -0
- laketower-0.5.1/demo/laketower.yml +0 -31
- laketower-0.5.1/docs/static/queries_view.png +0 -0
- laketower-0.5.1/docs/static/tables_history.png +0 -0
- laketower-0.5.1/docs/static/tables_overview.png +0 -0
- laketower-0.5.1/docs/static/tables_query.png +0 -0
- laketower-0.5.1/docs/static/tables_statistics.png +0 -0
- laketower-0.5.1/docs/static/tables_view.png +0 -0
- laketower-0.5.1/laketower/__about__.py +0 -1
- laketower-0.5.1/laketower/cli.py +0 -331
- laketower-0.5.1/laketower/config.py +0 -47
- laketower-0.5.1/laketower/tables.py +0 -141
- laketower-0.5.1/laketower/templates/tables/query.html +0 -51
- laketower-0.5.1/laketower/web.py +0 -236
- laketower-0.5.1/tests/conftest.py +0 -60
- laketower-0.5.1/tests/test_web.py +0 -425
- laketower-0.5.1/uv.lock +0 -1671
- {laketower-0.5.1 → laketower-0.6.1}/.python-version +0 -0
- {laketower-0.5.1 → laketower-0.6.1}/LICENSE +0 -0
- {laketower-0.5.1 → laketower-0.6.1}/demo/generate.py +0 -0
- {laketower-0.5.1 → laketower-0.6.1}/demo/sample_table/_delta_log/00000000000000000000.json +0 -0
- {laketower-0.5.1 → laketower-0.6.1}/demo/sample_table/_delta_log/00000000000000000001.json +0 -0
- {laketower-0.5.1 → laketower-0.6.1}/demo/sample_table/_delta_log/00000000000000000002.json +0 -0
- {laketower-0.5.1 → laketower-0.6.1}/demo/sample_table/_delta_log/00000000000000000003.json +0 -0
- {laketower-0.5.1 → laketower-0.6.1}/demo/sample_table/part-00001-1a31a393-6db6-4d1a-bf4e-81ea061ff8cd-c000.snappy.parquet +0 -0
- {laketower-0.5.1 → laketower-0.6.1}/demo/sample_table/part-00001-5af77102-9207-4c89-aaf6-37e1f815ec26-c000.snappy.parquet +0 -0
- {laketower-0.5.1 → laketower-0.6.1}/demo/sample_table/part-00001-b11bab55-43d0-4d05-ae88-5b9481ae57db-c000.snappy.parquet +0 -0
- {laketower-0.5.1 → laketower-0.6.1}/demo/weather/_delta_log/00000000000000000000.json +0 -0
- {laketower-0.5.1 → laketower-0.6.1}/demo/weather/_delta_log/00000000000000000001.json +0 -0
- {laketower-0.5.1 → laketower-0.6.1}/demo/weather/_delta_log/00000000000000000002.json +0 -0
- {laketower-0.5.1 → laketower-0.6.1}/demo/weather/part-00001-2323b963-be56-44e0-8c10-e237e7e6d4b9-c000.snappy.parquet +0 -0
- {laketower-0.5.1 → laketower-0.6.1}/demo/weather/part-00001-6360cbf8-f8a9-475f-8729-6f20b4ca64a9-c000.snappy.parquet +0 -0
- {laketower-0.5.1 → laketower-0.6.1}/laketower/__init__.py +0 -0
- {laketower-0.5.1 → laketower-0.6.1}/laketower/__main__.py +0 -0
- {laketower-0.5.1 → laketower-0.6.1}/laketower/static/.gitkeep +0 -0
- {laketower-0.5.1 → laketower-0.6.1}/laketower/templates/index.html +0 -0
- {laketower-0.5.1 → laketower-0.6.1}/renovate.json +0 -0
- {laketower-0.5.1 → laketower-0.6.1}/tests/__init__.py +0 -0
|
@@ -16,7 +16,7 @@ jobs:
|
|
|
16
16
|
matrix:
|
|
17
17
|
python-version: ["3.10", "3.11", "3.12", "3.13"]
|
|
18
18
|
steps:
|
|
19
|
-
- uses: actions/checkout@
|
|
19
|
+
- uses: actions/checkout@v5
|
|
20
20
|
- name: Install uv
|
|
21
21
|
uses: astral-sh/setup-uv@v6
|
|
22
22
|
with:
|
|
@@ -47,10 +47,10 @@ jobs:
|
|
|
47
47
|
CODECOV_MD_FILE: "coverage.md"
|
|
48
48
|
CODECOV_REPORT_TITLE: "Code coverage report"
|
|
49
49
|
steps:
|
|
50
|
-
- uses: actions/checkout@
|
|
50
|
+
- uses: actions/checkout@v5
|
|
51
51
|
with:
|
|
52
52
|
ref: main
|
|
53
|
-
- uses: actions/checkout@
|
|
53
|
+
- uses: actions/checkout@v5
|
|
54
54
|
- name: Install uv
|
|
55
55
|
uses: astral-sh/setup-uv@v6
|
|
56
56
|
with:
|
|
@@ -59,7 +59,7 @@ jobs:
|
|
|
59
59
|
cache-dependency-glob: "uv.lock"
|
|
60
60
|
- name: Install dependencies
|
|
61
61
|
run: uv sync --all-extras --dev
|
|
62
|
-
- uses: actions/download-artifact@
|
|
62
|
+
- uses: actions/download-artifact@v5
|
|
63
63
|
with:
|
|
64
64
|
name: coverage
|
|
65
65
|
- name: Generate Markdown code coverage report
|
|
@@ -78,7 +78,7 @@ jobs:
|
|
|
78
78
|
- name: Export code coverage report to job summary
|
|
79
79
|
run: cat "${CODECOV_MD_FILE}" >> "${GITHUB_STEP_SUMMARY}"
|
|
80
80
|
- name: Add or update code coverage comment to pull request
|
|
81
|
-
uses: actions/github-script@
|
|
81
|
+
uses: actions/github-script@v8
|
|
82
82
|
if: ${{ github.event_name == 'pull_request' }}
|
|
83
83
|
with:
|
|
84
84
|
script: |
|
|
@@ -124,7 +124,7 @@ jobs:
|
|
|
124
124
|
runs-on: ubuntu-latest
|
|
125
125
|
needs: ci
|
|
126
126
|
steps:
|
|
127
|
-
- uses: actions/checkout@
|
|
127
|
+
- uses: actions/checkout@v5
|
|
128
128
|
- name: Install uv
|
|
129
129
|
uses: astral-sh/setup-uv@v6
|
|
130
130
|
with:
|
|
@@ -154,7 +154,7 @@ jobs:
|
|
|
154
154
|
id-token: write
|
|
155
155
|
steps:
|
|
156
156
|
- name: Download package build artifacts
|
|
157
|
-
uses: actions/download-artifact@
|
|
157
|
+
uses: actions/download-artifact@v5
|
|
158
158
|
with:
|
|
159
159
|
name: build
|
|
160
160
|
path: dist/
|
|
@@ -180,7 +180,7 @@ jobs:
|
|
|
180
180
|
id-token: write
|
|
181
181
|
steps:
|
|
182
182
|
- name: Download package build artifacts
|
|
183
|
-
uses: actions/download-artifact@
|
|
183
|
+
uses: actions/download-artifact@v5
|
|
184
184
|
with:
|
|
185
185
|
name: build
|
|
186
186
|
path: dist/
|
|
@@ -202,9 +202,9 @@ jobs:
|
|
|
202
202
|
permissions:
|
|
203
203
|
contents: write
|
|
204
204
|
steps:
|
|
205
|
-
- uses: actions/checkout@
|
|
205
|
+
- uses: actions/checkout@v5
|
|
206
206
|
- name: Download package build artifacts
|
|
207
|
-
uses: actions/download-artifact@
|
|
207
|
+
uses: actions/download-artifact@v5
|
|
208
208
|
with:
|
|
209
209
|
name: build
|
|
210
210
|
path: dist/
|
|
@@ -7,6 +7,51 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
|
|
|
7
7
|
|
|
8
8
|
## [Unreleased]
|
|
9
9
|
|
|
10
|
+
## [0.6.1] - 2025-09-11
|
|
11
|
+
Patch release with minor enhancements (SQL syntax highlighting, query parameters,
|
|
12
|
+
predefined query Markdown description) and quality of life improvements
|
|
13
|
+
(hide read-only SQL editor in predefined queries, web app offline usage).
|
|
14
|
+
|
|
15
|
+
### Added
|
|
16
|
+
- web: display app version in sidebar
|
|
17
|
+
- web: use CodeMirror SQL query editor
|
|
18
|
+
- web: add support for tables query parameters
|
|
19
|
+
- web: add support for predefined queries parameters
|
|
20
|
+
- web: add optional markdown description for predefined queries
|
|
21
|
+
- cli: add support for tables query parameters
|
|
22
|
+
- cli: add support for predefined queries parameters
|
|
23
|
+
|
|
24
|
+
### Changed
|
|
25
|
+
- web: hide SQL editor for predefined queries
|
|
26
|
+
- demo: add parameters to daily avg temperature query
|
|
27
|
+
|
|
28
|
+
### Fixed
|
|
29
|
+
- handle empty SQL queries
|
|
30
|
+
|
|
31
|
+
### Misc
|
|
32
|
+
- vendor static assets for offline usage
|
|
33
|
+
|
|
34
|
+
## [0.6.0] - 2025-08-27
|
|
35
|
+
Minor release with new features (CSV import/export, S3/ADLS remote tables)
|
|
36
|
+
and quality of life improvements (tables lazy loading, quoted SQL identifiers).
|
|
37
|
+
|
|
38
|
+
### Added
|
|
39
|
+
- cli: add csv export option to tables query command
|
|
40
|
+
- cli: add tables import command
|
|
41
|
+
- web: add csv export to query views
|
|
42
|
+
- web: add table import form
|
|
43
|
+
- allow environment variable substitution in YAML configuration
|
|
44
|
+
- support for remote Delta tables (S3, ADLS)
|
|
45
|
+
|
|
46
|
+
### Changed
|
|
47
|
+
- cli: table uri lazy validation in app configuration
|
|
48
|
+
- web: table uri lazy validation in app configuration
|
|
49
|
+
- docs: update web application screenshots
|
|
50
|
+
|
|
51
|
+
### Fixed
|
|
52
|
+
- cli: laketower python entrypoint script
|
|
53
|
+
- always use quoted SQL identifiers in query builder
|
|
54
|
+
|
|
10
55
|
## [0.5.1] - 2025-05-30
|
|
11
56
|
Patch release with support for `deltalake` version 1.0.0.
|
|
12
57
|
|
|
@@ -85,7 +130,9 @@ Initial release of `laketower`.
|
|
|
85
130
|
- View a given table with simple query builder
|
|
86
131
|
- Query all registered tables with DuckDB SQL dialect
|
|
87
132
|
|
|
88
|
-
[Unreleased]: https://github.com/datalpia/laketower/compare/0.
|
|
133
|
+
[Unreleased]: https://github.com/datalpia/laketower/compare/0.6.1...HEAD
|
|
134
|
+
[0.6.1]: https://github.com/datalpia/laketower/compare/0.6.0...0.6.1
|
|
135
|
+
[0.6.0]: https://github.com/datalpia/laketower/compare/0.5.1...0.6.0
|
|
89
136
|
[0.5.1]: https://github.com/datalpia/laketower/compare/0.5.0...0.5.1
|
|
90
137
|
[0.5.0]: https://github.com/datalpia/laketower/compare/0.4.1...0.5.0
|
|
91
138
|
[0.4.1]: https://github.com/datalpia/laketower/compare/0.4.0...0.4.1
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: laketower
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.6.1
|
|
4
4
|
Summary: Oversee your lakehouse
|
|
5
5
|
Project-URL: Repository, https://github.com/datalpia/laketower
|
|
6
6
|
Project-URL: Issues, https://github.com/datalpia/laketower/issues
|
|
@@ -22,14 +22,17 @@ Classifier: Topic :: Database
|
|
|
22
22
|
Classifier: Topic :: Software Development
|
|
23
23
|
Classifier: Topic :: Utilities
|
|
24
24
|
Requires-Python: <3.14,>=3.10
|
|
25
|
+
Requires-Dist: bleach
|
|
25
26
|
Requires-Dist: deltalake<2,>=1
|
|
26
27
|
Requires-Dist: duckdb
|
|
27
28
|
Requires-Dist: fastapi
|
|
28
29
|
Requires-Dist: jinja2!=3.1.5,>=3
|
|
30
|
+
Requires-Dist: markdown
|
|
29
31
|
Requires-Dist: pandas
|
|
30
32
|
Requires-Dist: pyarrow!=19.0.0
|
|
31
33
|
Requires-Dist: pydantic-settings>=2
|
|
32
34
|
Requires-Dist: pydantic>=2
|
|
35
|
+
Requires-Dist: python-multipart
|
|
33
36
|
Requires-Dist: pyyaml
|
|
34
37
|
Requires-Dist: rich
|
|
35
38
|
Requires-Dist: sqlglot
|
|
@@ -50,13 +53,16 @@ Utility application to explore and manage tables in your data lakehouse, especia
|
|
|
50
53
|
## Features
|
|
51
54
|
|
|
52
55
|
- Delta Lake table format support
|
|
56
|
+
- Remote tables support (S3, ADLS)
|
|
53
57
|
- Inspect table metadata
|
|
54
58
|
- Inspect table schema
|
|
55
59
|
- Inspect table history
|
|
56
60
|
- Get table statistics
|
|
61
|
+
- Import data into a table from CSV files
|
|
57
62
|
- View table content with a simple query builder
|
|
58
63
|
- Query all registered tables with DuckDB SQL dialect
|
|
59
64
|
- Execute saved queries
|
|
65
|
+
- Export query results to CSV files
|
|
60
66
|
- Static and versionable YAML configuration
|
|
61
67
|
- Web application
|
|
62
68
|
- CLI application
|
|
@@ -94,12 +100,18 @@ tables:
|
|
|
94
100
|
queries:
|
|
95
101
|
- name: <query_name>
|
|
96
102
|
title: <Query name>
|
|
103
|
+
description: <Query description>
|
|
104
|
+
parameters:
|
|
105
|
+
<param_name_1>:
|
|
106
|
+
default: <default_value>
|
|
97
107
|
sql: <sql expression>
|
|
98
108
|
```
|
|
99
109
|
|
|
100
110
|
Current limitations:
|
|
101
111
|
|
|
102
|
-
- `tables.uri`:
|
|
112
|
+
- `tables.uri`:
|
|
113
|
+
- Local paths are supported (`./path/to/table`, `/abs/path/to/table`, `file:///abs/path/to/table`)
|
|
114
|
+
- Remote paths to S3 (`s3://<bucket>/<path>`) and ADLS (`abfss://<container>/<path>`)
|
|
103
115
|
- `tables.format`: only `delta` is allowed
|
|
104
116
|
|
|
105
117
|
Example from the provided demo:
|
|
@@ -138,6 +150,103 @@ queries:
|
|
|
138
150
|
day asc
|
|
139
151
|
```
|
|
140
152
|
|
|
153
|
+
Support for environment variables substitution is also supported within the YAML
|
|
154
|
+
configuration using a object containing a single key `env` with the name of the
|
|
155
|
+
environment variable to be injected. The value of the variable can contain JSON
|
|
156
|
+
and will be decoded in a best effort manner (default to string value). For instance:
|
|
157
|
+
|
|
158
|
+
```yaml
|
|
159
|
+
# export TABLE_URI=path/to/table
|
|
160
|
+
|
|
161
|
+
tables:
|
|
162
|
+
- name: sample_table
|
|
163
|
+
uri:
|
|
164
|
+
env: TABLE_URI
|
|
165
|
+
format: delta
|
|
166
|
+
```
|
|
167
|
+
|
|
168
|
+
#### Remote S3 Tables
|
|
169
|
+
|
|
170
|
+
Configuring S3 tables (AWS, MinIO, Cloudflare R2):
|
|
171
|
+
|
|
172
|
+
```yaml
|
|
173
|
+
tables:
|
|
174
|
+
- name: delta_table_s3
|
|
175
|
+
uri: s3://<bucket>/path/to/table
|
|
176
|
+
format: delta
|
|
177
|
+
connection:
|
|
178
|
+
s3:
|
|
179
|
+
s3_access_key_id: access-key-id
|
|
180
|
+
s3_secret_access_key: secret-access-key
|
|
181
|
+
s3_region: s3-region
|
|
182
|
+
s3_endpoint_url: http://s3.domain.com
|
|
183
|
+
s3_allow_http: false
|
|
184
|
+
```
|
|
185
|
+
|
|
186
|
+
Depending on your object storage location and configuration, one might have to
|
|
187
|
+
set part or all the available `connection.s3` parameters. The only required ones
|
|
188
|
+
are `s3_access_key_id` and `s3_secret_access_key`.
|
|
189
|
+
|
|
190
|
+
Also as a security best practice, it is best not to write secrets directly in
|
|
191
|
+
static configuration files, so one can use environment variables to all dynamic substitution,
|
|
192
|
+
e.g.
|
|
193
|
+
|
|
194
|
+
```yaml
|
|
195
|
+
tables:
|
|
196
|
+
- name: delta_table_s3
|
|
197
|
+
uri: s3://<bucket>/path/to/table
|
|
198
|
+
format: delta
|
|
199
|
+
connection:
|
|
200
|
+
s3:
|
|
201
|
+
s3_access_key_id: access-key-id
|
|
202
|
+
s3_secret_access_key:
|
|
203
|
+
env: S3_SECRET_ACCESS_KEY
|
|
204
|
+
s3_region: s3-region
|
|
205
|
+
s3_endpoint_url: http://s3.domain.com
|
|
206
|
+
s3_allow_http: false
|
|
207
|
+
```
|
|
208
|
+
|
|
209
|
+
#### Remote ADLS Tables
|
|
210
|
+
|
|
211
|
+
Configuring Azure ADLS tables:
|
|
212
|
+
|
|
213
|
+
```yaml
|
|
214
|
+
tables:
|
|
215
|
+
- name: delta_table_adls
|
|
216
|
+
uri: abfss://<container>/path/to/table
|
|
217
|
+
format: delta
|
|
218
|
+
connection:
|
|
219
|
+
adls:
|
|
220
|
+
adls_account_name: adls-account-name
|
|
221
|
+
adls_access_key: adls-access-key
|
|
222
|
+
adls_sas_key: adls-sas-key
|
|
223
|
+
adls_tenant_id: adls-tenant-id
|
|
224
|
+
adls_client_id: adls-client-id
|
|
225
|
+
adls_client_secret: adls-client-secret
|
|
226
|
+
azure_msi_endpoint: https://msi.azure.com
|
|
227
|
+
use_azure_cli: false
|
|
228
|
+
```
|
|
229
|
+
|
|
230
|
+
Depending on your object storage location and configuration, one might have to
|
|
231
|
+
set part or all the available `connection.adls` parameters. The only required one
|
|
232
|
+
is `adls_account_name`.
|
|
233
|
+
|
|
234
|
+
Also as a security best practice, it is best not to write secrets directly in
|
|
235
|
+
static configuration files, so one can use environment variables to all dynamic substitution,
|
|
236
|
+
e.g.
|
|
237
|
+
|
|
238
|
+
```yaml
|
|
239
|
+
tables:
|
|
240
|
+
- name: delta_table_adls
|
|
241
|
+
uri: abfss://<container>/path/to/table
|
|
242
|
+
format: delta
|
|
243
|
+
connection:
|
|
244
|
+
adls:
|
|
245
|
+
adls_account_name: adls-account-name
|
|
246
|
+
adls_access_key:
|
|
247
|
+
env: ADLS_ACCESS_KEY
|
|
248
|
+
```
|
|
249
|
+
|
|
141
250
|
### Web Application
|
|
142
251
|
|
|
143
252
|
The easiest way to get started is to launch the Laketower web application:
|
|
@@ -152,6 +261,7 @@ $ laketower -c demo/laketower.yml web
|
|
|
152
261
|

|
|
153
262
|

|
|
154
263
|

|
|
264
|
+

|
|
155
265
|

|
|
156
266
|

|
|
157
267
|
|
|
@@ -321,6 +431,29 @@ $ laketower -c demo/laketower.yml tables statistics --version 0 weather
|
|
|
321
431
|
└──────────────────────┴───────┴──────┴──────┴──────┴──────┘
|
|
322
432
|
```
|
|
323
433
|
|
|
434
|
+
#### Import data into a given table
|
|
435
|
+
|
|
436
|
+
Import a CSV dataset into a table in append mode:
|
|
437
|
+
|
|
438
|
+
```bash
|
|
439
|
+
$ laketower -c demo/laketower.yml tables import weather --file data.csv --mode append --format csv --delimiter ',' --encoding 'utf-8'
|
|
440
|
+
```
|
|
441
|
+
|
|
442
|
+
`--mode` argument can be one of:
|
|
443
|
+
- `append`: append rows to the table (default)
|
|
444
|
+
- `overwrite`: replace all rows with the ones from the input file
|
|
445
|
+
|
|
446
|
+
`--format` argument can be one of:
|
|
447
|
+
- `csv`: CSV file format (default)
|
|
448
|
+
|
|
449
|
+
`--delimiter` argument can be:
|
|
450
|
+
- Any single character (only valid for CSV file format)
|
|
451
|
+
- Default is _comma_ (`','`)
|
|
452
|
+
|
|
453
|
+
`--encoding` argument can be:
|
|
454
|
+
- Any [standard Python encoding](https://docs.python.org/3/library/codecs.html#standard-encodings),
|
|
455
|
+
- Default is `'utf-8'`
|
|
456
|
+
|
|
324
457
|
#### View a given table
|
|
325
458
|
|
|
326
459
|
Using a simple query builder, the content of a table can be displayed.
|
|
@@ -400,6 +533,28 @@ $ laketower -c demo/laketower.yml tables query "select date_trunc('day', time) a
|
|
|
400
533
|
└───────────────────────────┴────────────────────┘
|
|
401
534
|
```
|
|
402
535
|
|
|
536
|
+
Use named parameters within a giving query (note: escape `$` prefixes properly!):
|
|
537
|
+
|
|
538
|
+
```bash
|
|
539
|
+
$ laketower -c demo/laketower.yml tables query "select date_trunc('day', time) as day, avg(temperature_2m) as mean_temperature from weather where day between \$start_date and \$end_date group by day order by day desc" -p start_date 2025-01-29 -p end_date 2025-01-31
|
|
540
|
+
|
|
541
|
+
┏━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━┓
|
|
542
|
+
┃ day ┃ mean_temperature ┃
|
|
543
|
+
┡━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━┩
|
|
544
|
+
│ 2025-01-31 00:00:00+01:00 │ 5.683333257834117 │
|
|
545
|
+
│ 2025-01-30 00:00:00+01:00 │ 8.900000015894571 │
|
|
546
|
+
│ 2025-01-29 00:00:00+01:00 │ 7.770833313465118 │
|
|
547
|
+
└───────────────────────────┴────────────────────┘
|
|
548
|
+
```
|
|
549
|
+
|
|
550
|
+
Export query results to CSV:
|
|
551
|
+
|
|
552
|
+
```bash
|
|
553
|
+
$ laketower -c demo/laketower.yml tables query --output results.csv "select date_trunc('day', time) as day, avg(temperature_2m) as mean_temperature from weather group by day order by day desc limit 3"
|
|
554
|
+
|
|
555
|
+
Query results written to: results.csv
|
|
556
|
+
```
|
|
557
|
+
|
|
403
558
|
#### List saved queries
|
|
404
559
|
|
|
405
560
|
```bash
|
|
@@ -439,6 +594,22 @@ $ laketower -c demo/laketower.yml queries view daily_avg_temperature
|
|
|
439
594
|
└───────────────────────────┴─────────────────┘
|
|
440
595
|
```
|
|
441
596
|
|
|
597
|
+
Executing a predefined query with parameters (here `start_date` and `end_date`):
|
|
598
|
+
|
|
599
|
+
```bash
|
|
600
|
+
$ laketower -c demo/laketower.yml queries view daily_avg_temperature_params -p start_date 2025-02-01 -p end_date 2025-02-05
|
|
601
|
+
|
|
602
|
+
┏━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━┓
|
|
603
|
+
┃ day ┃ avg_temperature ┃
|
|
604
|
+
┡━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━┩
|
|
605
|
+
│ 2025-02-01 00:00:00+01:00 │ 4.0 │
|
|
606
|
+
│ 2025-02-02 00:00:00+01:00 │ 4.0 │
|
|
607
|
+
│ 2025-02-03 00:00:00+01:00 │ 4.0 │
|
|
608
|
+
│ 2025-02-04 00:00:00+01:00 │ 3.0 │
|
|
609
|
+
│ 2025-02-05 00:00:00+01:00 │ 3.0 │
|
|
610
|
+
└───────────────────────────┴─────────────────┘
|
|
611
|
+
```
|
|
612
|
+
|
|
442
613
|
## License
|
|
443
614
|
|
|
444
615
|
Licensed under [Apache License 2.0](LICENSE)
|
|
@@ -12,13 +12,16 @@ Utility application to explore and manage tables in your data lakehouse, especia
|
|
|
12
12
|
## Features
|
|
13
13
|
|
|
14
14
|
- Delta Lake table format support
|
|
15
|
+
- Remote tables support (S3, ADLS)
|
|
15
16
|
- Inspect table metadata
|
|
16
17
|
- Inspect table schema
|
|
17
18
|
- Inspect table history
|
|
18
19
|
- Get table statistics
|
|
20
|
+
- Import data into a table from CSV files
|
|
19
21
|
- View table content with a simple query builder
|
|
20
22
|
- Query all registered tables with DuckDB SQL dialect
|
|
21
23
|
- Execute saved queries
|
|
24
|
+
- Export query results to CSV files
|
|
22
25
|
- Static and versionable YAML configuration
|
|
23
26
|
- Web application
|
|
24
27
|
- CLI application
|
|
@@ -56,12 +59,18 @@ tables:
|
|
|
56
59
|
queries:
|
|
57
60
|
- name: <query_name>
|
|
58
61
|
title: <Query name>
|
|
62
|
+
description: <Query description>
|
|
63
|
+
parameters:
|
|
64
|
+
<param_name_1>:
|
|
65
|
+
default: <default_value>
|
|
59
66
|
sql: <sql expression>
|
|
60
67
|
```
|
|
61
68
|
|
|
62
69
|
Current limitations:
|
|
63
70
|
|
|
64
|
-
- `tables.uri`:
|
|
71
|
+
- `tables.uri`:
|
|
72
|
+
- Local paths are supported (`./path/to/table`, `/abs/path/to/table`, `file:///abs/path/to/table`)
|
|
73
|
+
- Remote paths to S3 (`s3://<bucket>/<path>`) and ADLS (`abfss://<container>/<path>`)
|
|
65
74
|
- `tables.format`: only `delta` is allowed
|
|
66
75
|
|
|
67
76
|
Example from the provided demo:
|
|
@@ -100,6 +109,103 @@ queries:
|
|
|
100
109
|
day asc
|
|
101
110
|
```
|
|
102
111
|
|
|
112
|
+
Support for environment variables substitution is also supported within the YAML
|
|
113
|
+
configuration using a object containing a single key `env` with the name of the
|
|
114
|
+
environment variable to be injected. The value of the variable can contain JSON
|
|
115
|
+
and will be decoded in a best effort manner (default to string value). For instance:
|
|
116
|
+
|
|
117
|
+
```yaml
|
|
118
|
+
# export TABLE_URI=path/to/table
|
|
119
|
+
|
|
120
|
+
tables:
|
|
121
|
+
- name: sample_table
|
|
122
|
+
uri:
|
|
123
|
+
env: TABLE_URI
|
|
124
|
+
format: delta
|
|
125
|
+
```
|
|
126
|
+
|
|
127
|
+
#### Remote S3 Tables
|
|
128
|
+
|
|
129
|
+
Configuring S3 tables (AWS, MinIO, Cloudflare R2):
|
|
130
|
+
|
|
131
|
+
```yaml
|
|
132
|
+
tables:
|
|
133
|
+
- name: delta_table_s3
|
|
134
|
+
uri: s3://<bucket>/path/to/table
|
|
135
|
+
format: delta
|
|
136
|
+
connection:
|
|
137
|
+
s3:
|
|
138
|
+
s3_access_key_id: access-key-id
|
|
139
|
+
s3_secret_access_key: secret-access-key
|
|
140
|
+
s3_region: s3-region
|
|
141
|
+
s3_endpoint_url: http://s3.domain.com
|
|
142
|
+
s3_allow_http: false
|
|
143
|
+
```
|
|
144
|
+
|
|
145
|
+
Depending on your object storage location and configuration, one might have to
|
|
146
|
+
set part or all the available `connection.s3` parameters. The only required ones
|
|
147
|
+
are `s3_access_key_id` and `s3_secret_access_key`.
|
|
148
|
+
|
|
149
|
+
Also as a security best practice, it is best not to write secrets directly in
|
|
150
|
+
static configuration files, so one can use environment variables to all dynamic substitution,
|
|
151
|
+
e.g.
|
|
152
|
+
|
|
153
|
+
```yaml
|
|
154
|
+
tables:
|
|
155
|
+
- name: delta_table_s3
|
|
156
|
+
uri: s3://<bucket>/path/to/table
|
|
157
|
+
format: delta
|
|
158
|
+
connection:
|
|
159
|
+
s3:
|
|
160
|
+
s3_access_key_id: access-key-id
|
|
161
|
+
s3_secret_access_key:
|
|
162
|
+
env: S3_SECRET_ACCESS_KEY
|
|
163
|
+
s3_region: s3-region
|
|
164
|
+
s3_endpoint_url: http://s3.domain.com
|
|
165
|
+
s3_allow_http: false
|
|
166
|
+
```
|
|
167
|
+
|
|
168
|
+
#### Remote ADLS Tables
|
|
169
|
+
|
|
170
|
+
Configuring Azure ADLS tables:
|
|
171
|
+
|
|
172
|
+
```yaml
|
|
173
|
+
tables:
|
|
174
|
+
- name: delta_table_adls
|
|
175
|
+
uri: abfss://<container>/path/to/table
|
|
176
|
+
format: delta
|
|
177
|
+
connection:
|
|
178
|
+
adls:
|
|
179
|
+
adls_account_name: adls-account-name
|
|
180
|
+
adls_access_key: adls-access-key
|
|
181
|
+
adls_sas_key: adls-sas-key
|
|
182
|
+
adls_tenant_id: adls-tenant-id
|
|
183
|
+
adls_client_id: adls-client-id
|
|
184
|
+
adls_client_secret: adls-client-secret
|
|
185
|
+
azure_msi_endpoint: https://msi.azure.com
|
|
186
|
+
use_azure_cli: false
|
|
187
|
+
```
|
|
188
|
+
|
|
189
|
+
Depending on your object storage location and configuration, one might have to
|
|
190
|
+
set part or all the available `connection.adls` parameters. The only required one
|
|
191
|
+
is `adls_account_name`.
|
|
192
|
+
|
|
193
|
+
Also as a security best practice, it is best not to write secrets directly in
|
|
194
|
+
static configuration files, so one can use environment variables to all dynamic substitution,
|
|
195
|
+
e.g.
|
|
196
|
+
|
|
197
|
+
```yaml
|
|
198
|
+
tables:
|
|
199
|
+
- name: delta_table_adls
|
|
200
|
+
uri: abfss://<container>/path/to/table
|
|
201
|
+
format: delta
|
|
202
|
+
connection:
|
|
203
|
+
adls:
|
|
204
|
+
adls_account_name: adls-account-name
|
|
205
|
+
adls_access_key:
|
|
206
|
+
env: ADLS_ACCESS_KEY
|
|
207
|
+
```
|
|
208
|
+
|
|
103
209
|
### Web Application
|
|
104
210
|
|
|
105
211
|
The easiest way to get started is to launch the Laketower web application:
|
|
@@ -114,6 +220,7 @@ $ laketower -c demo/laketower.yml web
|
|
|
114
220
|

|
|
115
221
|

|
|
116
222
|

|
|
223
|
+

|
|
117
224
|

|
|
118
225
|

|
|
119
226
|
|
|
@@ -283,6 +390,29 @@ $ laketower -c demo/laketower.yml tables statistics --version 0 weather
|
|
|
283
390
|
└──────────────────────┴───────┴──────┴──────┴──────┴──────┘
|
|
284
391
|
```
|
|
285
392
|
|
|
393
|
+
#### Import data into a given table
|
|
394
|
+
|
|
395
|
+
Import a CSV dataset into a table in append mode:
|
|
396
|
+
|
|
397
|
+
```bash
|
|
398
|
+
$ laketower -c demo/laketower.yml tables import weather --file data.csv --mode append --format csv --delimiter ',' --encoding 'utf-8'
|
|
399
|
+
```
|
|
400
|
+
|
|
401
|
+
`--mode` argument can be one of:
|
|
402
|
+
- `append`: append rows to the table (default)
|
|
403
|
+
- `overwrite`: replace all rows with the ones from the input file
|
|
404
|
+
|
|
405
|
+
`--format` argument can be one of:
|
|
406
|
+
- `csv`: CSV file format (default)
|
|
407
|
+
|
|
408
|
+
`--delimiter` argument can be:
|
|
409
|
+
- Any single character (only valid for CSV file format)
|
|
410
|
+
- Default is _comma_ (`','`)
|
|
411
|
+
|
|
412
|
+
`--encoding` argument can be:
|
|
413
|
+
- Any [standard Python encoding](https://docs.python.org/3/library/codecs.html#standard-encodings),
|
|
414
|
+
- Default is `'utf-8'`
|
|
415
|
+
|
|
286
416
|
#### View a given table
|
|
287
417
|
|
|
288
418
|
Using a simple query builder, the content of a table can be displayed.
|
|
@@ -362,6 +492,28 @@ $ laketower -c demo/laketower.yml tables query "select date_trunc('day', time) a
|
|
|
362
492
|
└───────────────────────────┴────────────────────┘
|
|
363
493
|
```
|
|
364
494
|
|
|
495
|
+
Use named parameters within a giving query (note: escape `$` prefixes properly!):
|
|
496
|
+
|
|
497
|
+
```bash
|
|
498
|
+
$ laketower -c demo/laketower.yml tables query "select date_trunc('day', time) as day, avg(temperature_2m) as mean_temperature from weather where day between \$start_date and \$end_date group by day order by day desc" -p start_date 2025-01-29 -p end_date 2025-01-31
|
|
499
|
+
|
|
500
|
+
┏━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━┓
|
|
501
|
+
┃ day ┃ mean_temperature ┃
|
|
502
|
+
┡━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━┩
|
|
503
|
+
│ 2025-01-31 00:00:00+01:00 │ 5.683333257834117 │
|
|
504
|
+
│ 2025-01-30 00:00:00+01:00 │ 8.900000015894571 │
|
|
505
|
+
│ 2025-01-29 00:00:00+01:00 │ 7.770833313465118 │
|
|
506
|
+
└───────────────────────────┴────────────────────┘
|
|
507
|
+
```
|
|
508
|
+
|
|
509
|
+
Export query results to CSV:
|
|
510
|
+
|
|
511
|
+
```bash
|
|
512
|
+
$ laketower -c demo/laketower.yml tables query --output results.csv "select date_trunc('day', time) as day, avg(temperature_2m) as mean_temperature from weather group by day order by day desc limit 3"
|
|
513
|
+
|
|
514
|
+
Query results written to: results.csv
|
|
515
|
+
```
|
|
516
|
+
|
|
365
517
|
#### List saved queries
|
|
366
518
|
|
|
367
519
|
```bash
|
|
@@ -401,6 +553,22 @@ $ laketower -c demo/laketower.yml queries view daily_avg_temperature
|
|
|
401
553
|
└───────────────────────────┴─────────────────┘
|
|
402
554
|
```
|
|
403
555
|
|
|
556
|
+
Executing a predefined query with parameters (here `start_date` and `end_date`):
|
|
557
|
+
|
|
558
|
+
```bash
|
|
559
|
+
$ laketower -c demo/laketower.yml queries view daily_avg_temperature_params -p start_date 2025-02-01 -p end_date 2025-02-05
|
|
560
|
+
|
|
561
|
+
┏━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━┓
|
|
562
|
+
┃ day ┃ avg_temperature ┃
|
|
563
|
+
┡━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━┩
|
|
564
|
+
│ 2025-02-01 00:00:00+01:00 │ 4.0 │
|
|
565
|
+
│ 2025-02-02 00:00:00+01:00 │ 4.0 │
|
|
566
|
+
│ 2025-02-03 00:00:00+01:00 │ 4.0 │
|
|
567
|
+
│ 2025-02-04 00:00:00+01:00 │ 3.0 │
|
|
568
|
+
│ 2025-02-05 00:00:00+01:00 │ 3.0 │
|
|
569
|
+
└───────────────────────────┴─────────────────┘
|
|
570
|
+
```
|
|
571
|
+
|
|
404
572
|
## License
|
|
405
573
|
|
|
406
574
|
Licensed under [Apache License 2.0](LICENSE)
|
|
@@ -0,0 +1,56 @@
|
|
|
1
|
+
tables:
|
|
2
|
+
- name: sample_table
|
|
3
|
+
uri: demo/sample_table
|
|
4
|
+
format: delta
|
|
5
|
+
- name: weather
|
|
6
|
+
uri: demo/weather
|
|
7
|
+
format: delta
|
|
8
|
+
- name: invalid_uri_table
|
|
9
|
+
uri: demo/invalid_uri_table
|
|
10
|
+
format: delta
|
|
11
|
+
|
|
12
|
+
queries:
|
|
13
|
+
- name: all_data
|
|
14
|
+
title: All data
|
|
15
|
+
description: Display all rows from `sample_table` and `weather` tables
|
|
16
|
+
sql: |
|
|
17
|
+
select
|
|
18
|
+
sample_table.*,
|
|
19
|
+
weather.*
|
|
20
|
+
from
|
|
21
|
+
sample_table,
|
|
22
|
+
weather
|
|
23
|
+
limit 10
|
|
24
|
+
- name: daily_avg_temperature
|
|
25
|
+
title: Daily average temperature
|
|
26
|
+
description: Display daily average temperature values from `weather` table
|
|
27
|
+
sql: |
|
|
28
|
+
select
|
|
29
|
+
date_trunc('day', time) as day,
|
|
30
|
+
round(avg(temperature_2m)) as avg_temperature
|
|
31
|
+
from
|
|
32
|
+
weather
|
|
33
|
+
group by
|
|
34
|
+
day
|
|
35
|
+
order by
|
|
36
|
+
day asc
|
|
37
|
+
- name: daily_avg_temperature_params
|
|
38
|
+
title: Daily average temperature with parameters
|
|
39
|
+
description: Display daily average temperature values from `weather` table, with dynamic filters for start and end dates
|
|
40
|
+
parameters:
|
|
41
|
+
start_date:
|
|
42
|
+
default: "2025-01-15"
|
|
43
|
+
end_date:
|
|
44
|
+
default: "2025-01-31"
|
|
45
|
+
sql: |
|
|
46
|
+
select
|
|
47
|
+
date_trunc('day', time) as day,
|
|
48
|
+
round(avg(temperature_2m)) as avg_temperature
|
|
49
|
+
from
|
|
50
|
+
weather
|
|
51
|
+
where
|
|
52
|
+
day between $start_date and $end_date
|
|
53
|
+
group by
|
|
54
|
+
day
|
|
55
|
+
order by
|
|
56
|
+
day asc
|
|
Binary file
|
|
Binary file
|
|
Binary file
|