laketower 0.5.0__tar.gz → 0.6.0__tar.gz

{laketower-0.5.0 → laketower-0.6.0}/.github/workflows/ci-cd.yml

@@ -16,9 +16,9 @@ jobs:
       matrix:
         python-version: ["3.10", "3.11", "3.12", "3.13"]
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v5
       - name: Install uv
-        uses: astral-sh/setup-uv@v5
+        uses: astral-sh/setup-uv@v6
         with:
           python-version: ${{ matrix.python-version }}
           enable-cache: true
@@ -47,19 +47,19 @@ jobs:
       CODECOV_MD_FILE: "coverage.md"
       CODECOV_REPORT_TITLE: "Code coverage report"
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v5
         with:
           ref: main
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v5
       - name: Install uv
-        uses: astral-sh/setup-uv@v5
+        uses: astral-sh/setup-uv@v6
         with:
           python-version: "3.13"
           enable-cache: true
           cache-dependency-glob: "uv.lock"
       - name: Install dependencies
         run: uv sync --all-extras --dev
-      - uses: actions/download-artifact@v4
+      - uses: actions/download-artifact@v5
         with:
           name: coverage
       - name: Generate Markdown code coverage report
@@ -124,9 +124,9 @@ jobs:
     runs-on: ubuntu-latest
     needs: ci
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v5
       - name: Install uv
-        uses: astral-sh/setup-uv@v5
+        uses: astral-sh/setup-uv@v6
         with:
           python-version: '3.13'
           enable-cache: true
@@ -154,7 +154,7 @@ jobs:
       id-token: write
     steps:
       - name: Download package build artifacts
-        uses: actions/download-artifact@v4
+        uses: actions/download-artifact@v5
         with:
           name: build
           path: dist/
@@ -180,14 +180,14 @@ jobs:
       id-token: write
     steps:
       - name: Download package build artifacts
-        uses: actions/download-artifact@v4
+        uses: actions/download-artifact@v5
         with:
           name: build
           path: dist/
       - name: Publish package distributions to PyPI
         uses: pypa/gh-action-pypi-publish@release/v1
       - name: Install uv
-        uses: astral-sh/setup-uv@v5
+        uses: astral-sh/setup-uv@v6
       - name: Validate package is available with uvx
         run: uvx laketower --version
 
@@ -202,14 +202,14 @@ jobs:
     permissions:
       contents: write
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v5
       - name: Download package build artifacts
-        uses: actions/download-artifact@v4
+        uses: actions/download-artifact@v5
         with:
           name: build
           path: dist/
       - name: Install uv
-        uses: astral-sh/setup-uv@v5
+        uses: astral-sh/setup-uv@v6
         with:
           python-version: '3.13'
           enable-cache: true

{laketower-0.5.0 → laketower-0.6.0}/CHANGELOG.md

@@ -7,6 +7,33 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+## [0.6.0] - 2025-08-27
+Minor release with new features (CSV import/export, S3/ADLS remote tables)
+and quality of life improvements (tables lazy loading, quoted SQL identifiers).
+
+### Added
+- cli: add csv export option to tables query command
+- cli: add tables import command
+- web: add csv export to query views
+- web: add table import form
+- allow environment variable substitution in YAML configuration
+- support for remote Delta tables (S3, ADLS)
+
+### Changed
+- cli: table uri lazy validation in app configuration
+- web: table uri lazy validation in app configuration
+- docs: update web application screenshots
+
+### Fixed
+- cli: laketower python entrypoint script
+- always use quoted SQL identifiers in query builder
+
+## [0.5.1] - 2025-05-30
+Patch release with support for `deltalake` version 1.0.0.
+
+### Changes
+- deps: upgrade to `deltalake` version 1
+
 ## [0.5.0] - 2025-03-19
 **Announcement:** Laketower open-source license is switching from AGPLv3 to Apache 2.0.
 
@@ -79,7 +106,9 @@ Initial release of `laketower`.
     - View a given table with simple query builder
     - Query all registered tables with DuckDB SQL dialect
 
-[Unreleased]: https://github.com/datalpia/laketower/compare/0.5.0...HEAD
+[Unreleased]: https://github.com/datalpia/laketower/compare/0.6.0...HEAD
+[0.6.0]: https://github.com/datalpia/laketower/compare/0.5.1...0.6.0
+[0.5.1]: https://github.com/datalpia/laketower/compare/0.5.0...0.5.1
 [0.5.0]: https://github.com/datalpia/laketower/compare/0.4.1...0.5.0
 [0.4.1]: https://github.com/datalpia/laketower/compare/0.4.0...0.4.1
 [0.4.0]: https://github.com/datalpia/laketower/compare/0.3.0...0.4.0

{laketower-0.5.0 → laketower-0.6.0}/PKG-INFO

@@ -1,10 +1,10 @@
 Metadata-Version: 2.4
 Name: laketower
-Version: 0.5.0
+Version: 0.6.0
 Summary: Oversee your lakehouse
 Project-URL: Repository, https://github.com/datalpia/laketower
 Project-URL: Issues, https://github.com/datalpia/laketower/issues
-Project-URL: Changelog, https://github.com/datalpia/laketower/blob/master/CHANGELOG.md
+Project-URL: Changelog, https://github.com/datalpia/laketower/blob/main/CHANGELOG.md
 Author-email: Romain Clement <git@romain-clement.net>
 License: Apache-2.0
 License-File: LICENSE
@@ -22,7 +22,7 @@ Classifier: Topic :: Database
 Classifier: Topic :: Software Development
 Classifier: Topic :: Utilities
 Requires-Python: <3.14,>=3.10
-Requires-Dist: deltalake
+Requires-Dist: deltalake<2,>=1
 Requires-Dist: duckdb
 Requires-Dist: fastapi
 Requires-Dist: jinja2!=3.1.5,>=3
@@ -30,6 +30,7 @@ Requires-Dist: pandas
 Requires-Dist: pyarrow!=19.0.0
 Requires-Dist: pydantic-settings>=2
 Requires-Dist: pydantic>=2
+Requires-Dist: python-multipart
 Requires-Dist: pyyaml
 Requires-Dist: rich
 Requires-Dist: sqlglot
@@ -50,13 +51,16 @@ Utility application to explore and manage tables in your data lakehouse, especia
 ## Features
 
 - Delta Lake table format support
+- Remote tables support (S3, ADLS)
 - Inspect table metadata
 - Inspect table schema
 - Inspect table history
 - Get table statistics
+- Import data into a table from CSV files
 - View table content with a simple query builder
 - Query all registered tables with DuckDB SQL dialect
 - Execute saved queries
+- Export query results to CSV files
 - Static and versionable YAML configuration
 - Web application
 - CLI application
@@ -99,7 +103,9 @@ queries:
 
 Current limitations:
 
-- `tables.uri`: only local paths are allowed
+- `tables.uri`:
+    - Local paths are supported (`./path/to/table`, `/abs/path/to/table`, `file:///abs/path/to/table`)
+    - Remote paths to S3 (`s3://<bucket>/<path>`) and ADLS (`abfss://<container>/<path>`)
 - `tables.format`: only `delta` is allowed
 
 Example from the provided demo:
@@ -138,6 +144,103 @@ queries:
         day asc
 ```
 
+Support for environment variables substitution is also supported within the YAML
+configuration using a object containing a single key `env` with the name of the
+environment variable to be injected. The value of the variable can contain JSON
+and will be decoded in a best effort manner (default to string value). For instance:
+
+```yaml
+# export TABLE_URI=path/to/table
+
+tables:
+  - name: sample_table
+    uri:
+      env: TABLE_URI
+    format: delta
+```
+
+#### Remote S3 Tables
+
+Configuring S3 tables (AWS, MinIO, Cloudflare R2):
+
+```yaml
+tables:
+  - name: delta_table_s3
+    uri: s3://<bucket>/path/to/table
+    format: delta
+    connection:
+      s3:
+        s3_access_key_id: access-key-id
+        s3_secret_access_key: secret-access-key
+        s3_region: s3-region
+        s3_endpoint_url: http://s3.domain.com
+        s3_allow_http: false
+```
+
+Depending on your object storage location and configuration, one might have to
+set part or all the available `connection.s3` parameters. The only required ones
+are `s3_access_key_id` and `s3_secret_access_key`.
+
+Also as a security best practice, it is best not to write secrets directly in
+static configuration files, so one can use environment variables to all dynamic substitution,
+e.g.
+
+```yaml
+tables:
+  - name: delta_table_s3
+    uri: s3://<bucket>/path/to/table
+    format: delta
+    connection:
+      s3:
+        s3_access_key_id: access-key-id
+        s3_secret_access_key:
+          env: S3_SECRET_ACCESS_KEY
+        s3_region: s3-region
+        s3_endpoint_url: http://s3.domain.com
+        s3_allow_http: false
+```
+
+#### Remote ADLS Tables
+
+Configuring Azure ADLS tables:
+
+```yaml
+tables:
+  - name: delta_table_adls
+    uri: abfss://<container>/path/to/table
+    format: delta
+    connection:
+      adls:
+        adls_account_name: adls-account-name
+        adls_access_key: adls-access-key
+        adls_sas_key: adls-sas-key
+        adls_tenant_id: adls-tenant-id
+        adls_client_id: adls-client-id
+        adls_client_secret: adls-client-secret
+        azure_msi_endpoint: https://msi.azure.com
+        use_azure_cli: false
+```
+
+Depending on your object storage location and configuration, one might have to
+set part or all the available `connection.adls` parameters. The only required one
+is `adls_account_name`.
+
+Also as a security best practice, it is best not to write secrets directly in
+static configuration files, so one can use environment variables to all dynamic substitution,
+e.g.
+
+```yaml
+tables:
+  - name: delta_table_adls
+    uri: abfss://<container>/path/to/table
+    format: delta
+    connection:
+      adls:
+        adls_account_name: adls-account-name
+        adls_access_key:
+          env: ADLS_ACCESS_KEY
+```
+
 ### Web Application
 
 The easiest way to get started is to launch the Laketower web application:
@@ -148,12 +251,13 @@ $ laketower -c demo/laketower.yml web
 
 #### Screenshots
 
-![Laketower UI - Tables Overview](docs/static/tables_overview.png)
-![Laketower UI - Tables View](docs/static/tables_view.png)
-![Laketower UI - Tables Statistics](docs/static/tables_statistics.png)
-![Laketower UI - Tables History](docs/static/tables_history.png)
-![Laketower UI - Tables Query](docs/static/tables_query.png)
-![Laketower UI - Queries View](docs/static/queries_view.png)
+![Laketower UI - Tables Overview](https://raw.githubusercontent.com/datalpia/laketower/refs/heads/main/docs/static/tables_overview.png)
+![Laketower UI - Tables View](https://raw.githubusercontent.com/datalpia/laketower/refs/heads/main/docs/static/tables_view.png)
+![Laketower UI - Tables Statistics](https://raw.githubusercontent.com/datalpia/laketower/refs/heads/main/docs/static/tables_statistics.png)
+![Laketower UI - Tables History](https://raw.githubusercontent.com/datalpia/laketower/refs/heads/main/docs/static/tables_history.png)
+![Laketower UI - Tables Import](https://raw.githubusercontent.com/datalpia/laketower/refs/heads/main/docs/static/tables_import.png)
+![Laketower UI - Tables Query](https://raw.githubusercontent.com/datalpia/laketower/refs/heads/main/docs/static/tables_query.png)
+![Laketower UI - Queries View](https://raw.githubusercontent.com/datalpia/laketower/refs/heads/main/docs/static/queries_view.png)
 
 ### CLI
 
@@ -321,6 +425,29 @@ $ laketower -c demo/laketower.yml tables statistics --version 0 weather
 └──────────────────────┴───────┴──────┴──────┴──────┴──────┘
 ```
 
+#### Import data into a given table
+
+Import a CSV dataset into a table in append mode:
+
+```bash
+$ laketower -c demo/laketower.yml tables import weather --file data.csv --mode append --format csv --delimiter ',' --encoding 'utf-8'
+```
+
+`--mode` argument can be one of:
+- `append`: append rows to the table (default)
+- `overwrite`: replace all rows with the ones from the input file
+
+`--format` argument can be one of:
+- `csv`: CSV file format (default)
+
+`--delimiter` argument can be:
+- Any single character (only valid for CSV file format)
+- Default is _comma_ (`','`)
+
+`--encoding` argument can be:
+- Any [standard Python encoding](https://docs.python.org/3/library/codecs.html#standard-encodings),
+- Default is `'utf-8'`
+
 #### View a given table
 
 Using a simple query builder, the content of a table can be displayed.
@@ -400,6 +527,14 @@ $ laketower -c demo/laketower.yml tables query "select date_trunc('day', time) a
 └───────────────────────────┴────────────────────┘
 ```
 
+Export query results to CSV:
+
+```bash
+$ laketower -c demo/laketower.yml tables query --output results.csv "select date_trunc('day', time) as day, avg(temperature_2m) as mean_temperature from weather group by day order by day desc limit 3"
+
+Query results written to: results.csv
+```
+
 #### List saved queries
 
 ```bash

{laketower-0.5.0 → laketower-0.6.0}/README.md

@@ -12,13 +12,16 @@ Utility application to explore and manage tables in your data lakehouse, especia
 ## Features
 
 - Delta Lake table format support
+- Remote tables support (S3, ADLS)
 - Inspect table metadata
 - Inspect table schema
 - Inspect table history
 - Get table statistics
+- Import data into a table from CSV files
 - View table content with a simple query builder
 - Query all registered tables with DuckDB SQL dialect
 - Execute saved queries
+- Export query results to CSV files
 - Static and versionable YAML configuration
 - Web application
 - CLI application
@@ -61,7 +64,9 @@ queries:
 
 Current limitations:
 
-- `tables.uri`: only local paths are allowed
+- `tables.uri`:
+    - Local paths are supported (`./path/to/table`, `/abs/path/to/table`, `file:///abs/path/to/table`)
+    - Remote paths to S3 (`s3://<bucket>/<path>`) and ADLS (`abfss://<container>/<path>`)
 - `tables.format`: only `delta` is allowed
 
 Example from the provided demo:
@@ -100,6 +105,103 @@ queries:
         day asc
 ```
 
+Support for environment variables substitution is also supported within the YAML
+configuration using a object containing a single key `env` with the name of the
+environment variable to be injected. The value of the variable can contain JSON
+and will be decoded in a best effort manner (default to string value). For instance:
+
+```yaml
+# export TABLE_URI=path/to/table
+
+tables:
+  - name: sample_table
+    uri:
+      env: TABLE_URI
+    format: delta
+```
+
+#### Remote S3 Tables
+
+Configuring S3 tables (AWS, MinIO, Cloudflare R2):
+
+```yaml
+tables:
+  - name: delta_table_s3
+    uri: s3://<bucket>/path/to/table
+    format: delta
+    connection:
+      s3:
+        s3_access_key_id: access-key-id
+        s3_secret_access_key: secret-access-key
+        s3_region: s3-region
+        s3_endpoint_url: http://s3.domain.com
+        s3_allow_http: false
+```
+
+Depending on your object storage location and configuration, one might have to
+set part or all the available `connection.s3` parameters. The only required ones
+are `s3_access_key_id` and `s3_secret_access_key`.
+
+Also as a security best practice, it is best not to write secrets directly in
+static configuration files, so one can use environment variables to all dynamic substitution,
+e.g.
+
+```yaml
+tables:
+  - name: delta_table_s3
+    uri: s3://<bucket>/path/to/table
+    format: delta
+    connection:
+      s3:
+        s3_access_key_id: access-key-id
+        s3_secret_access_key:
+          env: S3_SECRET_ACCESS_KEY
+        s3_region: s3-region
+        s3_endpoint_url: http://s3.domain.com
+        s3_allow_http: false
+```
+
+#### Remote ADLS Tables
+
+Configuring Azure ADLS tables:
+
+```yaml
+tables:
+  - name: delta_table_adls
+    uri: abfss://<container>/path/to/table
+    format: delta
+    connection:
+      adls:
+        adls_account_name: adls-account-name
+        adls_access_key: adls-access-key
+        adls_sas_key: adls-sas-key
+        adls_tenant_id: adls-tenant-id
+        adls_client_id: adls-client-id
+        adls_client_secret: adls-client-secret
+        azure_msi_endpoint: https://msi.azure.com
+        use_azure_cli: false
+```
+
+Depending on your object storage location and configuration, one might have to
+set part or all the available `connection.adls` parameters. The only required one
+is `adls_account_name`.
+
+Also as a security best practice, it is best not to write secrets directly in
+static configuration files, so one can use environment variables to all dynamic substitution,
+e.g.
+
+```yaml
+tables:
+  - name: delta_table_adls
+    uri: abfss://<container>/path/to/table
+    format: delta
+    connection:
+      adls:
+        adls_account_name: adls-account-name
+        adls_access_key:
+          env: ADLS_ACCESS_KEY
+```
+
 ### Web Application
 
 The easiest way to get started is to launch the Laketower web application:
@@ -110,12 +212,13 @@ $ laketower -c demo/laketower.yml web
 
 #### Screenshots
 
-![Laketower UI - Tables Overview](docs/static/tables_overview.png)
-![Laketower UI - Tables View](docs/static/tables_view.png)
-![Laketower UI - Tables Statistics](docs/static/tables_statistics.png)
-![Laketower UI - Tables History](docs/static/tables_history.png)
-![Laketower UI - Tables Query](docs/static/tables_query.png)
-![Laketower UI - Queries View](docs/static/queries_view.png)
+![Laketower UI - Tables Overview](https://raw.githubusercontent.com/datalpia/laketower/refs/heads/main/docs/static/tables_overview.png)
+![Laketower UI - Tables View](https://raw.githubusercontent.com/datalpia/laketower/refs/heads/main/docs/static/tables_view.png)
+![Laketower UI - Tables Statistics](https://raw.githubusercontent.com/datalpia/laketower/refs/heads/main/docs/static/tables_statistics.png)
+![Laketower UI - Tables History](https://raw.githubusercontent.com/datalpia/laketower/refs/heads/main/docs/static/tables_history.png)
+![Laketower UI - Tables Import](https://raw.githubusercontent.com/datalpia/laketower/refs/heads/main/docs/static/tables_import.png)
+![Laketower UI - Tables Query](https://raw.githubusercontent.com/datalpia/laketower/refs/heads/main/docs/static/tables_query.png)
+![Laketower UI - Queries View](https://raw.githubusercontent.com/datalpia/laketower/refs/heads/main/docs/static/queries_view.png)
 
 ### CLI
 
@@ -283,6 +386,29 @@ $ laketower -c demo/laketower.yml tables statistics --version 0 weather
 └──────────────────────┴───────┴──────┴──────┴──────┴──────┘
 ```
 
+#### Import data into a given table
+
+Import a CSV dataset into a table in append mode:
+
+```bash
+$ laketower -c demo/laketower.yml tables import weather --file data.csv --mode append --format csv --delimiter ',' --encoding 'utf-8'
+```
+
+`--mode` argument can be one of:
+- `append`: append rows to the table (default)
+- `overwrite`: replace all rows with the ones from the input file
+
+`--format` argument can be one of:
+- `csv`: CSV file format (default)
+
+`--delimiter` argument can be:
+- Any single character (only valid for CSV file format)
+- Default is _comma_ (`','`)
+
+`--encoding` argument can be:
+- Any [standard Python encoding](https://docs.python.org/3/library/codecs.html#standard-encodings),
+- Default is `'utf-8'`
+
 #### View a given table
 
 Using a simple query builder, the content of a table can be displayed.
@@ -362,6 +488,14 @@ $ laketower -c demo/laketower.yml tables query "select date_trunc('day', time) a
 └───────────────────────────┴────────────────────┘
 ```
 
+Export query results to CSV:
+
+```bash
+$ laketower -c demo/laketower.yml tables query --output results.csv "select date_trunc('day', time) as day, avg(temperature_2m) as mean_temperature from weather group by day order by day desc limit 3"
+
+Query results written to: results.csv
+```
+
 #### List saved queries
 
 ```bash

{laketower-0.5.0 → laketower-0.6.0}/demo/laketower.yml

@@ -5,6 +5,9 @@ tables:
   - name: weather
     uri: demo/weather
     format: delta
+  - name: invalid_uri_table
+    uri: demo/invalid_uri_table
+    format: delta
 
 queries:
   - name: all_data

laketower-0.6.0/laketower/__about__.py

@@ -0,0 +1 @@
+__version__ = "0.6.0"