laketower 0.5.1__tar.gz → 0.6.1__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of laketower might be problematic. Click here for more details.

Files changed (82) hide show
  1. {laketower-0.5.1 → laketower-0.6.1}/.github/workflows/ci-cd.yml +10 -10
  2. {laketower-0.5.1 → laketower-0.6.1}/.gitignore +3 -0
  3. {laketower-0.5.1 → laketower-0.6.1}/CHANGELOG.md +48 -1
  4. {laketower-0.5.1 → laketower-0.6.1}/PKG-INFO +173 -2
  5. {laketower-0.5.1 → laketower-0.6.1}/README.md +169 -1
  6. laketower-0.6.1/demo/laketower.yml +56 -0
  7. laketower-0.6.1/docs/static/queries_view.png +0 -0
  8. laketower-0.6.1/docs/static/tables_history.png +0 -0
  9. laketower-0.6.1/docs/static/tables_import.png +0 -0
  10. laketower-0.6.1/docs/static/tables_overview.png +0 -0
  11. laketower-0.6.1/docs/static/tables_query.png +0 -0
  12. laketower-0.6.1/docs/static/tables_statistics.png +0 -0
  13. laketower-0.6.1/docs/static/tables_view.png +0 -0
  14. laketower-0.6.1/laketower/__about__.py +1 -0
  15. laketower-0.6.1/laketower/cli.py +461 -0
  16. laketower-0.6.1/laketower/config.py +119 -0
  17. laketower-0.6.1/laketower/static/editor.bundle.js +27433 -0
  18. laketower-0.6.1/laketower/static/editor.js +74 -0
  19. laketower-0.6.1/laketower/static/vendor/bootstrap/bootstrap.bundle.min.js +7 -0
  20. laketower-0.6.1/laketower/static/vendor/bootstrap-icons/bootstrap-icons.min.css +5 -0
  21. laketower-0.6.1/laketower/static/vendor/bootstrap-icons/fonts/bootstrap-icons.woff +0 -0
  22. laketower-0.6.1/laketower/static/vendor/bootstrap-icons/fonts/bootstrap-icons.woff2 +0 -0
  23. laketower-0.6.1/laketower/static/vendor/halfmoon/halfmoon.min.css +22 -0
  24. laketower-0.6.1/laketower/static/vendor/halfmoon/halfmoon.modern.css +282 -0
  25. laketower-0.6.1/laketower/tables.py +311 -0
  26. {laketower-0.5.1 → laketower-0.6.1}/laketower/templates/_base.html +22 -19
  27. {laketower-0.5.1 → laketower-0.6.1}/laketower/templates/queries/view.html +24 -4
  28. {laketower-0.5.1 → laketower-0.6.1}/laketower/templates/tables/_macros.html +3 -0
  29. {laketower-0.5.1 → laketower-0.6.1}/laketower/templates/tables/history.html +6 -0
  30. laketower-0.6.1/laketower/templates/tables/import.html +71 -0
  31. {laketower-0.5.1 → laketower-0.6.1}/laketower/templates/tables/index.html +6 -0
  32. laketower-0.6.1/laketower/templates/tables/query.html +81 -0
  33. {laketower-0.5.1 → laketower-0.6.1}/laketower/templates/tables/statistics.html +6 -0
  34. {laketower-0.5.1 → laketower-0.6.1}/laketower/templates/tables/view.html +6 -0
  35. laketower-0.6.1/laketower/web.py +422 -0
  36. laketower-0.6.1/package-lock.json +1139 -0
  37. laketower-0.6.1/package.json +16 -0
  38. {laketower-0.5.1 → laketower-0.6.1}/pyproject.toml +15 -10
  39. {laketower-0.5.1 → laketower-0.6.1}/tasks.py +65 -7
  40. laketower-0.6.1/tests/conftest.py +119 -0
  41. {laketower-0.5.1 → laketower-0.6.1}/tests/test_cli.py +530 -28
  42. laketower-0.6.1/tests/test_config.py +229 -0
  43. laketower-0.6.1/tests/test_tables.py +111 -0
  44. laketower-0.6.1/tests/test_web.py +844 -0
  45. laketower-0.6.1/uv.lock +1812 -0
  46. laketower-0.5.1/demo/laketower.yml +0 -31
  47. laketower-0.5.1/docs/static/queries_view.png +0 -0
  48. laketower-0.5.1/docs/static/tables_history.png +0 -0
  49. laketower-0.5.1/docs/static/tables_overview.png +0 -0
  50. laketower-0.5.1/docs/static/tables_query.png +0 -0
  51. laketower-0.5.1/docs/static/tables_statistics.png +0 -0
  52. laketower-0.5.1/docs/static/tables_view.png +0 -0
  53. laketower-0.5.1/laketower/__about__.py +0 -1
  54. laketower-0.5.1/laketower/cli.py +0 -331
  55. laketower-0.5.1/laketower/config.py +0 -47
  56. laketower-0.5.1/laketower/tables.py +0 -141
  57. laketower-0.5.1/laketower/templates/tables/query.html +0 -51
  58. laketower-0.5.1/laketower/web.py +0 -236
  59. laketower-0.5.1/tests/conftest.py +0 -60
  60. laketower-0.5.1/tests/test_web.py +0 -425
  61. laketower-0.5.1/uv.lock +0 -1671
  62. {laketower-0.5.1 → laketower-0.6.1}/.python-version +0 -0
  63. {laketower-0.5.1 → laketower-0.6.1}/LICENSE +0 -0
  64. {laketower-0.5.1 → laketower-0.6.1}/demo/generate.py +0 -0
  65. {laketower-0.5.1 → laketower-0.6.1}/demo/sample_table/_delta_log/00000000000000000000.json +0 -0
  66. {laketower-0.5.1 → laketower-0.6.1}/demo/sample_table/_delta_log/00000000000000000001.json +0 -0
  67. {laketower-0.5.1 → laketower-0.6.1}/demo/sample_table/_delta_log/00000000000000000002.json +0 -0
  68. {laketower-0.5.1 → laketower-0.6.1}/demo/sample_table/_delta_log/00000000000000000003.json +0 -0
  69. {laketower-0.5.1 → laketower-0.6.1}/demo/sample_table/part-00001-1a31a393-6db6-4d1a-bf4e-81ea061ff8cd-c000.snappy.parquet +0 -0
  70. {laketower-0.5.1 → laketower-0.6.1}/demo/sample_table/part-00001-5af77102-9207-4c89-aaf6-37e1f815ec26-c000.snappy.parquet +0 -0
  71. {laketower-0.5.1 → laketower-0.6.1}/demo/sample_table/part-00001-b11bab55-43d0-4d05-ae88-5b9481ae57db-c000.snappy.parquet +0 -0
  72. {laketower-0.5.1 → laketower-0.6.1}/demo/weather/_delta_log/00000000000000000000.json +0 -0
  73. {laketower-0.5.1 → laketower-0.6.1}/demo/weather/_delta_log/00000000000000000001.json +0 -0
  74. {laketower-0.5.1 → laketower-0.6.1}/demo/weather/_delta_log/00000000000000000002.json +0 -0
  75. {laketower-0.5.1 → laketower-0.6.1}/demo/weather/part-00001-2323b963-be56-44e0-8c10-e237e7e6d4b9-c000.snappy.parquet +0 -0
  76. {laketower-0.5.1 → laketower-0.6.1}/demo/weather/part-00001-6360cbf8-f8a9-475f-8729-6f20b4ca64a9-c000.snappy.parquet +0 -0
  77. {laketower-0.5.1 → laketower-0.6.1}/laketower/__init__.py +0 -0
  78. {laketower-0.5.1 → laketower-0.6.1}/laketower/__main__.py +0 -0
  79. {laketower-0.5.1 → laketower-0.6.1}/laketower/static/.gitkeep +0 -0
  80. {laketower-0.5.1 → laketower-0.6.1}/laketower/templates/index.html +0 -0
  81. {laketower-0.5.1 → laketower-0.6.1}/renovate.json +0 -0
  82. {laketower-0.5.1 → laketower-0.6.1}/tests/__init__.py +0 -0
@@ -16,7 +16,7 @@ jobs:
16
16
  matrix:
17
17
  python-version: ["3.10", "3.11", "3.12", "3.13"]
18
18
  steps:
19
- - uses: actions/checkout@v4
19
+ - uses: actions/checkout@v5
20
20
  - name: Install uv
21
21
  uses: astral-sh/setup-uv@v6
22
22
  with:
@@ -47,10 +47,10 @@ jobs:
47
47
  CODECOV_MD_FILE: "coverage.md"
48
48
  CODECOV_REPORT_TITLE: "Code coverage report"
49
49
  steps:
50
- - uses: actions/checkout@v4
50
+ - uses: actions/checkout@v5
51
51
  with:
52
52
  ref: main
53
- - uses: actions/checkout@v4
53
+ - uses: actions/checkout@v5
54
54
  - name: Install uv
55
55
  uses: astral-sh/setup-uv@v6
56
56
  with:
@@ -59,7 +59,7 @@ jobs:
59
59
  cache-dependency-glob: "uv.lock"
60
60
  - name: Install dependencies
61
61
  run: uv sync --all-extras --dev
62
- - uses: actions/download-artifact@v4
62
+ - uses: actions/download-artifact@v5
63
63
  with:
64
64
  name: coverage
65
65
  - name: Generate Markdown code coverage report
@@ -78,7 +78,7 @@ jobs:
78
78
  - name: Export code coverage report to job summary
79
79
  run: cat "${CODECOV_MD_FILE}" >> "${GITHUB_STEP_SUMMARY}"
80
80
  - name: Add or update code coverage comment to pull request
81
- uses: actions/github-script@v7
81
+ uses: actions/github-script@v8
82
82
  if: ${{ github.event_name == 'pull_request' }}
83
83
  with:
84
84
  script: |
@@ -124,7 +124,7 @@ jobs:
124
124
  runs-on: ubuntu-latest
125
125
  needs: ci
126
126
  steps:
127
- - uses: actions/checkout@v4
127
+ - uses: actions/checkout@v5
128
128
  - name: Install uv
129
129
  uses: astral-sh/setup-uv@v6
130
130
  with:
@@ -154,7 +154,7 @@ jobs:
154
154
  id-token: write
155
155
  steps:
156
156
  - name: Download package build artifacts
157
- uses: actions/download-artifact@v4
157
+ uses: actions/download-artifact@v5
158
158
  with:
159
159
  name: build
160
160
  path: dist/
@@ -180,7 +180,7 @@ jobs:
180
180
  id-token: write
181
181
  steps:
182
182
  - name: Download package build artifacts
183
- uses: actions/download-artifact@v4
183
+ uses: actions/download-artifact@v5
184
184
  with:
185
185
  name: build
186
186
  path: dist/
@@ -202,9 +202,9 @@ jobs:
202
202
  permissions:
203
203
  contents: write
204
204
  steps:
205
- - uses: actions/checkout@v4
205
+ - uses: actions/checkout@v5
206
206
  - name: Download package build artifacts
207
- uses: actions/download-artifact@v4
207
+ uses: actions/download-artifact@v5
208
208
  with:
209
209
  name: build
210
210
  path: dist/
@@ -7,6 +7,9 @@ wheels/
7
7
  *.egg-info
8
8
  .coverage
9
9
 
10
+ # Node.js
11
+ node_modules/
12
+
10
13
  # Virtual environments
11
14
  .venv
12
15
 
@@ -7,6 +7,51 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
7
7
 
8
8
  ## [Unreleased]
9
9
 
10
+ ## [0.6.1] - 2025-09-11
11
+ Patch release with minor enhancements (SQL syntax highlighting, query parameters,
12
+ predefined query Markdown description) and quality of life improvements
13
+ (hide read-only SQL editor in predefined queries, web app offline usage).
14
+
15
+ ### Added
16
+ - web: display app version in sidebar
17
+ - web: use CodeMirror SQL query editor
18
+ - web: add support for tables query parameters
19
+ - web: add support for predefined queries parameters
20
+ - web: add optional markdown description for predefined queries
21
+ - cli: add support for tables query parameters
22
+ - cli: add support for predefined queries parameters
23
+
24
+ ### Changed
25
+ - web: hide SQL editor for predefined queries
26
+ - demo: add parameters to daily avg temperature query
27
+
28
+ ### Fixed
29
+ - handle empty SQL queries
30
+
31
+ ### Misc
32
+ - vendor static assets for offline usage
33
+
34
+ ## [0.6.0] - 2025-08-27
35
+ Minor release with new features (CSV import/export, S3/ADLS remote tables)
36
+ and quality of life improvements (tables lazy loading, quoted SQL identifiers).
37
+
38
+ ### Added
39
+ - cli: add csv export option to tables query command
40
+ - cli: add tables import command
41
+ - web: add csv export to query views
42
+ - web: add table import form
43
+ - allow environment variable substitution in YAML configuration
44
+ - support for remote Delta tables (S3, ADLS)
45
+
46
+ ### Changed
47
+ - cli: table uri lazy validation in app configuration
48
+ - web: table uri lazy validation in app configuration
49
+ - docs: update web application screenshots
50
+
51
+ ### Fixed
52
+ - cli: laketower python entrypoint script
53
+ - always use quoted SQL identifiers in query builder
54
+
10
55
  ## [0.5.1] - 2025-05-30
11
56
  Patch release with support for `deltalake` version 1.0.0.
12
57
 
@@ -85,7 +130,9 @@ Initial release of `laketower`.
85
130
  - View a given table with simple query builder
86
131
  - Query all registered tables with DuckDB SQL dialect
87
132
 
88
- [Unreleased]: https://github.com/datalpia/laketower/compare/0.5.1...HEAD
133
+ [Unreleased]: https://github.com/datalpia/laketower/compare/0.6.1...HEAD
134
+ [0.6.1]: https://github.com/datalpia/laketower/compare/0.6.0...0.6.1
135
+ [0.6.0]: https://github.com/datalpia/laketower/compare/0.5.1...0.6.0
89
136
  [0.5.1]: https://github.com/datalpia/laketower/compare/0.5.0...0.5.1
90
137
  [0.5.0]: https://github.com/datalpia/laketower/compare/0.4.1...0.5.0
91
138
  [0.4.1]: https://github.com/datalpia/laketower/compare/0.4.0...0.4.1
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: laketower
3
- Version: 0.5.1
3
+ Version: 0.6.1
4
4
  Summary: Oversee your lakehouse
5
5
  Project-URL: Repository, https://github.com/datalpia/laketower
6
6
  Project-URL: Issues, https://github.com/datalpia/laketower/issues
@@ -22,14 +22,17 @@ Classifier: Topic :: Database
22
22
  Classifier: Topic :: Software Development
23
23
  Classifier: Topic :: Utilities
24
24
  Requires-Python: <3.14,>=3.10
25
+ Requires-Dist: bleach
25
26
  Requires-Dist: deltalake<2,>=1
26
27
  Requires-Dist: duckdb
27
28
  Requires-Dist: fastapi
28
29
  Requires-Dist: jinja2!=3.1.5,>=3
30
+ Requires-Dist: markdown
29
31
  Requires-Dist: pandas
30
32
  Requires-Dist: pyarrow!=19.0.0
31
33
  Requires-Dist: pydantic-settings>=2
32
34
  Requires-Dist: pydantic>=2
35
+ Requires-Dist: python-multipart
33
36
  Requires-Dist: pyyaml
34
37
  Requires-Dist: rich
35
38
  Requires-Dist: sqlglot
@@ -50,13 +53,16 @@ Utility application to explore and manage tables in your data lakehouse, especia
50
53
  ## Features
51
54
 
52
55
  - Delta Lake table format support
56
+ - Remote tables support (S3, ADLS)
53
57
  - Inspect table metadata
54
58
  - Inspect table schema
55
59
  - Inspect table history
56
60
  - Get table statistics
61
+ - Import data into a table from CSV files
57
62
  - View table content with a simple query builder
58
63
  - Query all registered tables with DuckDB SQL dialect
59
64
  - Execute saved queries
65
+ - Export query results to CSV files
60
66
  - Static and versionable YAML configuration
61
67
  - Web application
62
68
  - CLI application
@@ -94,12 +100,18 @@ tables:
94
100
  queries:
95
101
  - name: <query_name>
96
102
  title: <Query name>
103
+ description: <Query description>
104
+ parameters:
105
+ <param_name_1>:
106
+ default: <default_value>
97
107
  sql: <sql expression>
98
108
  ```
99
109
 
100
110
  Current limitations:
101
111
 
102
- - `tables.uri`: only local paths are allowed
112
+ - `tables.uri`:
113
+ - Local paths are supported (`./path/to/table`, `/abs/path/to/table`, `file:///abs/path/to/table`)
114
+ - Remote paths to S3 (`s3://<bucket>/<path>`) and ADLS (`abfss://<container>/<path>`)
103
115
  - `tables.format`: only `delta` is allowed
104
116
 
105
117
  Example from the provided demo:
@@ -138,6 +150,103 @@ queries:
138
150
  day asc
139
151
  ```
140
152
 
153
+ Support for environment variables substitution is also supported within the YAML
154
+ configuration using a object containing a single key `env` with the name of the
155
+ environment variable to be injected. The value of the variable can contain JSON
156
+ and will be decoded in a best effort manner (default to string value). For instance:
157
+
158
+ ```yaml
159
+ # export TABLE_URI=path/to/table
160
+
161
+ tables:
162
+ - name: sample_table
163
+ uri:
164
+ env: TABLE_URI
165
+ format: delta
166
+ ```
167
+
168
+ #### Remote S3 Tables
169
+
170
+ Configuring S3 tables (AWS, MinIO, Cloudflare R2):
171
+
172
+ ```yaml
173
+ tables:
174
+ - name: delta_table_s3
175
+ uri: s3://<bucket>/path/to/table
176
+ format: delta
177
+ connection:
178
+ s3:
179
+ s3_access_key_id: access-key-id
180
+ s3_secret_access_key: secret-access-key
181
+ s3_region: s3-region
182
+ s3_endpoint_url: http://s3.domain.com
183
+ s3_allow_http: false
184
+ ```
185
+
186
+ Depending on your object storage location and configuration, one might have to
187
+ set part or all the available `connection.s3` parameters. The only required ones
188
+ are `s3_access_key_id` and `s3_secret_access_key`.
189
+
190
+ Also as a security best practice, it is best not to write secrets directly in
191
+ static configuration files, so one can use environment variables to all dynamic substitution,
192
+ e.g.
193
+
194
+ ```yaml
195
+ tables:
196
+ - name: delta_table_s3
197
+ uri: s3://<bucket>/path/to/table
198
+ format: delta
199
+ connection:
200
+ s3:
201
+ s3_access_key_id: access-key-id
202
+ s3_secret_access_key:
203
+ env: S3_SECRET_ACCESS_KEY
204
+ s3_region: s3-region
205
+ s3_endpoint_url: http://s3.domain.com
206
+ s3_allow_http: false
207
+ ```
208
+
209
+ #### Remote ADLS Tables
210
+
211
+ Configuring Azure ADLS tables:
212
+
213
+ ```yaml
214
+ tables:
215
+ - name: delta_table_adls
216
+ uri: abfss://<container>/path/to/table
217
+ format: delta
218
+ connection:
219
+ adls:
220
+ adls_account_name: adls-account-name
221
+ adls_access_key: adls-access-key
222
+ adls_sas_key: adls-sas-key
223
+ adls_tenant_id: adls-tenant-id
224
+ adls_client_id: adls-client-id
225
+ adls_client_secret: adls-client-secret
226
+ azure_msi_endpoint: https://msi.azure.com
227
+ use_azure_cli: false
228
+ ```
229
+
230
+ Depending on your object storage location and configuration, one might have to
231
+ set part or all the available `connection.adls` parameters. The only required one
232
+ is `adls_account_name`.
233
+
234
+ Also as a security best practice, it is best not to write secrets directly in
235
+ static configuration files, so one can use environment variables to all dynamic substitution,
236
+ e.g.
237
+
238
+ ```yaml
239
+ tables:
240
+ - name: delta_table_adls
241
+ uri: abfss://<container>/path/to/table
242
+ format: delta
243
+ connection:
244
+ adls:
245
+ adls_account_name: adls-account-name
246
+ adls_access_key:
247
+ env: ADLS_ACCESS_KEY
248
+ ```
249
+
141
250
  ### Web Application
142
251
 
143
252
  The easiest way to get started is to launch the Laketower web application:
@@ -152,6 +261,7 @@ $ laketower -c demo/laketower.yml web
152
261
  ![Laketower UI - Tables View](https://raw.githubusercontent.com/datalpia/laketower/refs/heads/main/docs/static/tables_view.png)
153
262
  ![Laketower UI - Tables Statistics](https://raw.githubusercontent.com/datalpia/laketower/refs/heads/main/docs/static/tables_statistics.png)
154
263
  ![Laketower UI - Tables History](https://raw.githubusercontent.com/datalpia/laketower/refs/heads/main/docs/static/tables_history.png)
264
+ ![Laketower UI - Tables Import](https://raw.githubusercontent.com/datalpia/laketower/refs/heads/main/docs/static/tables_import.png)
155
265
  ![Laketower UI - Tables Query](https://raw.githubusercontent.com/datalpia/laketower/refs/heads/main/docs/static/tables_query.png)
156
266
  ![Laketower UI - Queries View](https://raw.githubusercontent.com/datalpia/laketower/refs/heads/main/docs/static/queries_view.png)
157
267
 
@@ -321,6 +431,29 @@ $ laketower -c demo/laketower.yml tables statistics --version 0 weather
321
431
  └──────────────────────┴───────┴──────┴──────┴──────┴──────┘
322
432
  ```
323
433
 
434
+ #### Import data into a given table
435
+
436
+ Import a CSV dataset into a table in append mode:
437
+
438
+ ```bash
439
+ $ laketower -c demo/laketower.yml tables import weather --file data.csv --mode append --format csv --delimiter ',' --encoding 'utf-8'
440
+ ```
441
+
442
+ `--mode` argument can be one of:
443
+ - `append`: append rows to the table (default)
444
+ - `overwrite`: replace all rows with the ones from the input file
445
+
446
+ `--format` argument can be one of:
447
+ - `csv`: CSV file format (default)
448
+
449
+ `--delimiter` argument can be:
450
+ - Any single character (only valid for CSV file format)
451
+ - Default is _comma_ (`','`)
452
+
453
+ `--encoding` argument can be:
454
+ - Any [standard Python encoding](https://docs.python.org/3/library/codecs.html#standard-encodings),
455
+ - Default is `'utf-8'`
456
+
324
457
  #### View a given table
325
458
 
326
459
  Using a simple query builder, the content of a table can be displayed.
@@ -400,6 +533,28 @@ $ laketower -c demo/laketower.yml tables query "select date_trunc('day', time) a
400
533
  └───────────────────────────┴────────────────────┘
401
534
  ```
402
535
 
536
+ Use named parameters within a giving query (note: escape `$` prefixes properly!):
537
+
538
+ ```bash
539
+ $ laketower -c demo/laketower.yml tables query "select date_trunc('day', time) as day, avg(temperature_2m) as mean_temperature from weather where day between \$start_date and \$end_date group by day order by day desc" -p start_date 2025-01-29 -p end_date 2025-01-31
540
+
541
+ ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━┓
542
+ ┃ day ┃ mean_temperature ┃
543
+ ┡━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━┩
544
+ │ 2025-01-31 00:00:00+01:00 │ 5.683333257834117 │
545
+ │ 2025-01-30 00:00:00+01:00 │ 8.900000015894571 │
546
+ │ 2025-01-29 00:00:00+01:00 │ 7.770833313465118 │
547
+ └───────────────────────────┴────────────────────┘
548
+ ```
549
+
550
+ Export query results to CSV:
551
+
552
+ ```bash
553
+ $ laketower -c demo/laketower.yml tables query --output results.csv "select date_trunc('day', time) as day, avg(temperature_2m) as mean_temperature from weather group by day order by day desc limit 3"
554
+
555
+ Query results written to: results.csv
556
+ ```
557
+
403
558
  #### List saved queries
404
559
 
405
560
  ```bash
@@ -439,6 +594,22 @@ $ laketower -c demo/laketower.yml queries view daily_avg_temperature
439
594
  └───────────────────────────┴─────────────────┘
440
595
  ```
441
596
 
597
+ Executing a predefined query with parameters (here `start_date` and `end_date`):
598
+
599
+ ```bash
600
+ $ laketower -c demo/laketower.yml queries view daily_avg_temperature_params -p start_date 2025-02-01 -p end_date 2025-02-05
601
+
602
+ ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━┓
603
+ ┃ day ┃ avg_temperature ┃
604
+ ┡━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━┩
605
+ │ 2025-02-01 00:00:00+01:00 │ 4.0 │
606
+ │ 2025-02-02 00:00:00+01:00 │ 4.0 │
607
+ │ 2025-02-03 00:00:00+01:00 │ 4.0 │
608
+ │ 2025-02-04 00:00:00+01:00 │ 3.0 │
609
+ │ 2025-02-05 00:00:00+01:00 │ 3.0 │
610
+ └───────────────────────────┴─────────────────┘
611
+ ```
612
+
442
613
  ## License
443
614
 
444
615
  Licensed under [Apache License 2.0](LICENSE)
@@ -12,13 +12,16 @@ Utility application to explore and manage tables in your data lakehouse, especia
12
12
  ## Features
13
13
 
14
14
  - Delta Lake table format support
15
+ - Remote tables support (S3, ADLS)
15
16
  - Inspect table metadata
16
17
  - Inspect table schema
17
18
  - Inspect table history
18
19
  - Get table statistics
20
+ - Import data into a table from CSV files
19
21
  - View table content with a simple query builder
20
22
  - Query all registered tables with DuckDB SQL dialect
21
23
  - Execute saved queries
24
+ - Export query results to CSV files
22
25
  - Static and versionable YAML configuration
23
26
  - Web application
24
27
  - CLI application
@@ -56,12 +59,18 @@ tables:
56
59
  queries:
57
60
  - name: <query_name>
58
61
  title: <Query name>
62
+ description: <Query description>
63
+ parameters:
64
+ <param_name_1>:
65
+ default: <default_value>
59
66
  sql: <sql expression>
60
67
  ```
61
68
 
62
69
  Current limitations:
63
70
 
64
- - `tables.uri`: only local paths are allowed
71
+ - `tables.uri`:
72
+ - Local paths are supported (`./path/to/table`, `/abs/path/to/table`, `file:///abs/path/to/table`)
73
+ - Remote paths to S3 (`s3://<bucket>/<path>`) and ADLS (`abfss://<container>/<path>`)
65
74
  - `tables.format`: only `delta` is allowed
66
75
 
67
76
  Example from the provided demo:
@@ -100,6 +109,103 @@ queries:
100
109
  day asc
101
110
  ```
102
111
 
112
+ Support for environment variables substitution is also supported within the YAML
113
+ configuration using a object containing a single key `env` with the name of the
114
+ environment variable to be injected. The value of the variable can contain JSON
115
+ and will be decoded in a best effort manner (default to string value). For instance:
116
+
117
+ ```yaml
118
+ # export TABLE_URI=path/to/table
119
+
120
+ tables:
121
+ - name: sample_table
122
+ uri:
123
+ env: TABLE_URI
124
+ format: delta
125
+ ```
126
+
127
+ #### Remote S3 Tables
128
+
129
+ Configuring S3 tables (AWS, MinIO, Cloudflare R2):
130
+
131
+ ```yaml
132
+ tables:
133
+ - name: delta_table_s3
134
+ uri: s3://<bucket>/path/to/table
135
+ format: delta
136
+ connection:
137
+ s3:
138
+ s3_access_key_id: access-key-id
139
+ s3_secret_access_key: secret-access-key
140
+ s3_region: s3-region
141
+ s3_endpoint_url: http://s3.domain.com
142
+ s3_allow_http: false
143
+ ```
144
+
145
+ Depending on your object storage location and configuration, one might have to
146
+ set part or all the available `connection.s3` parameters. The only required ones
147
+ are `s3_access_key_id` and `s3_secret_access_key`.
148
+
149
+ Also as a security best practice, it is best not to write secrets directly in
150
+ static configuration files, so one can use environment variables to all dynamic substitution,
151
+ e.g.
152
+
153
+ ```yaml
154
+ tables:
155
+ - name: delta_table_s3
156
+ uri: s3://<bucket>/path/to/table
157
+ format: delta
158
+ connection:
159
+ s3:
160
+ s3_access_key_id: access-key-id
161
+ s3_secret_access_key:
162
+ env: S3_SECRET_ACCESS_KEY
163
+ s3_region: s3-region
164
+ s3_endpoint_url: http://s3.domain.com
165
+ s3_allow_http: false
166
+ ```
167
+
168
+ #### Remote ADLS Tables
169
+
170
+ Configuring Azure ADLS tables:
171
+
172
+ ```yaml
173
+ tables:
174
+ - name: delta_table_adls
175
+ uri: abfss://<container>/path/to/table
176
+ format: delta
177
+ connection:
178
+ adls:
179
+ adls_account_name: adls-account-name
180
+ adls_access_key: adls-access-key
181
+ adls_sas_key: adls-sas-key
182
+ adls_tenant_id: adls-tenant-id
183
+ adls_client_id: adls-client-id
184
+ adls_client_secret: adls-client-secret
185
+ azure_msi_endpoint: https://msi.azure.com
186
+ use_azure_cli: false
187
+ ```
188
+
189
+ Depending on your object storage location and configuration, one might have to
190
+ set part or all the available `connection.adls` parameters. The only required one
191
+ is `adls_account_name`.
192
+
193
+ Also as a security best practice, it is best not to write secrets directly in
194
+ static configuration files, so one can use environment variables to all dynamic substitution,
195
+ e.g.
196
+
197
+ ```yaml
198
+ tables:
199
+ - name: delta_table_adls
200
+ uri: abfss://<container>/path/to/table
201
+ format: delta
202
+ connection:
203
+ adls:
204
+ adls_account_name: adls-account-name
205
+ adls_access_key:
206
+ env: ADLS_ACCESS_KEY
207
+ ```
208
+
103
209
  ### Web Application
104
210
 
105
211
  The easiest way to get started is to launch the Laketower web application:
@@ -114,6 +220,7 @@ $ laketower -c demo/laketower.yml web
114
220
  ![Laketower UI - Tables View](https://raw.githubusercontent.com/datalpia/laketower/refs/heads/main/docs/static/tables_view.png)
115
221
  ![Laketower UI - Tables Statistics](https://raw.githubusercontent.com/datalpia/laketower/refs/heads/main/docs/static/tables_statistics.png)
116
222
  ![Laketower UI - Tables History](https://raw.githubusercontent.com/datalpia/laketower/refs/heads/main/docs/static/tables_history.png)
223
+ ![Laketower UI - Tables Import](https://raw.githubusercontent.com/datalpia/laketower/refs/heads/main/docs/static/tables_import.png)
117
224
  ![Laketower UI - Tables Query](https://raw.githubusercontent.com/datalpia/laketower/refs/heads/main/docs/static/tables_query.png)
118
225
  ![Laketower UI - Queries View](https://raw.githubusercontent.com/datalpia/laketower/refs/heads/main/docs/static/queries_view.png)
119
226
 
@@ -283,6 +390,29 @@ $ laketower -c demo/laketower.yml tables statistics --version 0 weather
283
390
  └──────────────────────┴───────┴──────┴──────┴──────┴──────┘
284
391
  ```
285
392
 
393
+ #### Import data into a given table
394
+
395
+ Import a CSV dataset into a table in append mode:
396
+
397
+ ```bash
398
+ $ laketower -c demo/laketower.yml tables import weather --file data.csv --mode append --format csv --delimiter ',' --encoding 'utf-8'
399
+ ```
400
+
401
+ `--mode` argument can be one of:
402
+ - `append`: append rows to the table (default)
403
+ - `overwrite`: replace all rows with the ones from the input file
404
+
405
+ `--format` argument can be one of:
406
+ - `csv`: CSV file format (default)
407
+
408
+ `--delimiter` argument can be:
409
+ - Any single character (only valid for CSV file format)
410
+ - Default is _comma_ (`','`)
411
+
412
+ `--encoding` argument can be:
413
+ - Any [standard Python encoding](https://docs.python.org/3/library/codecs.html#standard-encodings),
414
+ - Default is `'utf-8'`
415
+
286
416
  #### View a given table
287
417
 
288
418
  Using a simple query builder, the content of a table can be displayed.
@@ -362,6 +492,28 @@ $ laketower -c demo/laketower.yml tables query "select date_trunc('day', time) a
362
492
  └───────────────────────────┴────────────────────┘
363
493
  ```
364
494
 
495
+ Use named parameters within a giving query (note: escape `$` prefixes properly!):
496
+
497
+ ```bash
498
+ $ laketower -c demo/laketower.yml tables query "select date_trunc('day', time) as day, avg(temperature_2m) as mean_temperature from weather where day between \$start_date and \$end_date group by day order by day desc" -p start_date 2025-01-29 -p end_date 2025-01-31
499
+
500
+ ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━┓
501
+ ┃ day ┃ mean_temperature ┃
502
+ ┡━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━┩
503
+ │ 2025-01-31 00:00:00+01:00 │ 5.683333257834117 │
504
+ │ 2025-01-30 00:00:00+01:00 │ 8.900000015894571 │
505
+ │ 2025-01-29 00:00:00+01:00 │ 7.770833313465118 │
506
+ └───────────────────────────┴────────────────────┘
507
+ ```
508
+
509
+ Export query results to CSV:
510
+
511
+ ```bash
512
+ $ laketower -c demo/laketower.yml tables query --output results.csv "select date_trunc('day', time) as day, avg(temperature_2m) as mean_temperature from weather group by day order by day desc limit 3"
513
+
514
+ Query results written to: results.csv
515
+ ```
516
+
365
517
  #### List saved queries
366
518
 
367
519
  ```bash
@@ -401,6 +553,22 @@ $ laketower -c demo/laketower.yml queries view daily_avg_temperature
401
553
  └───────────────────────────┴─────────────────┘
402
554
  ```
403
555
 
556
+ Executing a predefined query with parameters (here `start_date` and `end_date`):
557
+
558
+ ```bash
559
+ $ laketower -c demo/laketower.yml queries view daily_avg_temperature_params -p start_date 2025-02-01 -p end_date 2025-02-05
560
+
561
+ ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━┓
562
+ ┃ day ┃ avg_temperature ┃
563
+ ┡━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━┩
564
+ │ 2025-02-01 00:00:00+01:00 │ 4.0 │
565
+ │ 2025-02-02 00:00:00+01:00 │ 4.0 │
566
+ │ 2025-02-03 00:00:00+01:00 │ 4.0 │
567
+ │ 2025-02-04 00:00:00+01:00 │ 3.0 │
568
+ │ 2025-02-05 00:00:00+01:00 │ 3.0 │
569
+ └───────────────────────────┴─────────────────┘
570
+ ```
571
+
404
572
  ## License
405
573
 
406
574
  Licensed under [Apache License 2.0](LICENSE)
@@ -0,0 +1,56 @@
1
+ tables:
2
+ - name: sample_table
3
+ uri: demo/sample_table
4
+ format: delta
5
+ - name: weather
6
+ uri: demo/weather
7
+ format: delta
8
+ - name: invalid_uri_table
9
+ uri: demo/invalid_uri_table
10
+ format: delta
11
+
12
+ queries:
13
+ - name: all_data
14
+ title: All data
15
+ description: Display all rows from `sample_table` and `weather` tables
16
+ sql: |
17
+ select
18
+ sample_table.*,
19
+ weather.*
20
+ from
21
+ sample_table,
22
+ weather
23
+ limit 10
24
+ - name: daily_avg_temperature
25
+ title: Daily average temperature
26
+ description: Display daily average temperature values from `weather` table
27
+ sql: |
28
+ select
29
+ date_trunc('day', time) as day,
30
+ round(avg(temperature_2m)) as avg_temperature
31
+ from
32
+ weather
33
+ group by
34
+ day
35
+ order by
36
+ day asc
37
+ - name: daily_avg_temperature_params
38
+ title: Daily average temperature with parameters
39
+ description: Display daily average temperature values from `weather` table, with dynamic filters for start and end dates
40
+ parameters:
41
+ start_date:
42
+ default: "2025-01-15"
43
+ end_date:
44
+ default: "2025-01-31"
45
+ sql: |
46
+ select
47
+ date_trunc('day', time) as day,
48
+ round(avg(temperature_2m)) as avg_temperature
49
+ from
50
+ weather
51
+ where
52
+ day between $start_date and $end_date
53
+ group by
54
+ day
55
+ order by
56
+ day asc