laketower 0.5.0__py3-none-any.whl → 0.6.0__py3-none-any.whl

laketower/web.py

@@ -3,17 +3,21 @@ from pathlib import Path
 from typing import Annotated
 
 import pydantic_settings
-from fastapi import APIRouter, FastAPI, Query, Request
-from fastapi.responses import HTMLResponse
+from fastapi import APIRouter, FastAPI, File, Form, Query, Request, UploadFile
+from fastapi.responses import HTMLResponse, Response
 from fastapi.staticfiles import StaticFiles
 from fastapi.templating import Jinja2Templates
 
 from laketower.config import Config, load_yaml_config
 from laketower.tables import (
     DEFAULT_LIMIT,
+    ImportFileFormatEnum,
+    ImportModeEnum,
     execute_query,
     generate_table_statistics_query,
     generate_table_query,
+    import_file_to_table,
+    load_datasets,
     load_table,
 )
 
@@ -55,10 +59,7 @@ def index(request: Request) -> HTMLResponse:
 @router.get("/tables/query", response_class=HTMLResponse)
 def get_tables_query(request: Request, sql: str) -> HTMLResponse:
     config: Config = request.app.state.config
-    tables_dataset = {
-        table_config.name: load_table(table_config).dataset()
-        for table_config in config.tables
-    }
+    tables_dataset = load_datasets(config.tables)
 
     try:
         results = execute_query(tables_dataset, sql)
@@ -80,13 +81,36 @@ def get_tables_query(request: Request, sql: str) -> HTMLResponse:
     )
 
 
+@router.get("/tables/query/csv")
+def export_tables_query_csv(request: Request, sql: str) -> Response:
+    config: Config = request.app.state.config
+    tables_dataset = load_datasets(config.tables)
+
+    results = execute_query(tables_dataset, sql)
+    csv_content = results.to_csv(header=True, index=False, sep=",")
+
+    return Response(
+        content=csv_content,
+        media_type="text/csv",
+        headers={"Content-Disposition": "attachment; filename=query_results.csv"},
+    )
+
+
 @router.get("/tables/{table_id}", response_class=HTMLResponse)
 def get_table_index(request: Request, table_id: str) -> HTMLResponse:
     config: Config = request.app.state.config
     table_config = next(
         filter(lambda table_config: table_config.name == table_id, config.tables)
     )
-    table = load_table(table_config)
+    try:
+        table = load_table(table_config)
+        table_metadata = table.metadata()
+        table_schema = table.schema()
+        error = None
+    except ValueError as e:
+        error = {"message": str(e)}
+        table_metadata = None
+        table_schema = None
 
     return templates.TemplateResponse(
         request=request,
@@ -95,8 +119,9 @@ def get_table_index(request: Request, table_id: str) -> HTMLResponse:
             "tables": config.tables,
             "queries": config.queries,
             "table_id": table_id,
-            "table_metadata": table.metadata(),
-            "table_schema": table.schema(),
+            "table_metadata": table_metadata,
+            "table_schema": table_schema,
+            "error": error,
         },
     )
 
@@ -107,7 +132,13 @@ def get_table_history(request: Request, table_id: str) -> HTMLResponse:
     table_config = next(
         filter(lambda table_config: table_config.name == table_id, config.tables)
     )
-    table = load_table(table_config)
+    try:
+        table = load_table(table_config)
+        table_history = table.history()
+        error = None
+    except ValueError as e:
+        error = {"message": str(e)}
+        table_history = None
 
     return templates.TemplateResponse(
         request=request,
@@ -116,7 +147,8 @@ def get_table_history(request: Request, table_id: str) -> HTMLResponse:
             "tables": config.tables,
             "queries": config.queries,
             "table_id": table_id,
-            "table_history": table.history(),
+            "table_history": table_history,
+            "error": error,
         },
     )
 
@@ -131,12 +163,18 @@ def get_table_statistics(
     table_config = next(
         filter(lambda table_config: table_config.name == table_id, config.tables)
     )
-    table = load_table(table_config)
-    table_name = table_config.name
-    table_metadata = table.metadata()
-    table_dataset = table.dataset(version=version)
-    sql_query = generate_table_statistics_query(table_name)
-    query_results = execute_query({table_name: table_dataset}, sql_query)
+    try:
+        table = load_table(table_config)
+        table_name = table_config.name
+        table_metadata = table.metadata()
+        table_dataset = table.dataset(version=version)
+        sql_query = generate_table_statistics_query(table_name)
+        query_results = execute_query({table_name: table_dataset}, sql_query)
+        error = None
+    except ValueError as e:
+        error = {"message": str(e)}
+        table_metadata = None
+        query_results = None
 
     return templates.TemplateResponse(
         request=request,
@@ -147,6 +185,7 @@ def get_table_statistics(
             "table_id": table_id,
             "table_metadata": table_metadata,
             "table_results": query_results,
+            "error": error,
         },
     )
 
@@ -165,14 +204,21 @@ def get_table_view(
     table_config = next(
         filter(lambda table_config: table_config.name == table_id, config.tables)
     )
-    table = load_table(table_config)
-    table_name = table_config.name
-    table_metadata = table.metadata()
-    table_dataset = table.dataset(version=version)
-    sql_query = generate_table_query(
-        table_name, limit=limit, cols=cols, sort_asc=sort_asc, sort_desc=sort_desc
-    )
-    results = execute_query({table_name: table_dataset}, sql_query)
+    try:
+        table = load_table(table_config)
+        table_name = table_config.name
+        table_metadata = table.metadata()
+        table_dataset = table.dataset(version=version)
+        sql_query = generate_table_query(
+            table_name, limit=limit, cols=cols, sort_asc=sort_asc, sort_desc=sort_desc
+        )
+        results = execute_query({table_name: table_dataset}, sql_query)
+        error = None
+    except ValueError as e:
+        error = {"message": str(e)}
+        table_metadata = None
+        sql_query = None
+        results = None
 
     return templates.TemplateResponse(
         request=request,
@@ -185,6 +231,78 @@ def get_table_view(
             "table_results": results,
             "sql_query": sql_query,
             "default_limit": DEFAULT_LIMIT,
+            "error": error,
+        },
+    )
+
+
+@router.get("/tables/{table_id}/import", response_class=HTMLResponse)
+def get_table_import(
+    request: Request,
+    table_id: str,
+) -> HTMLResponse:
+    config: Config = request.app.state.config
+    table_config = next(
+        filter(lambda table_config: table_config.name == table_id, config.tables)
+    )
+    try:
+        table = load_table(table_config)
+        table_metadata = table.metadata()
+        message = None
+    except ValueError as e:
+        message = {"type": "error", "body": str(e)}
+        table_metadata = None
+
+    return templates.TemplateResponse(
+        request=request,
+        name="tables/import.html",
+        context={
+            "tables": config.tables,
+            "queries": config.queries,
+            "table_id": table_id,
+            "table_metadata": table_metadata,
+            "message": message,
+        },
+    )
+
+
+@router.post("/tables/{table_id}/import", response_class=HTMLResponse)
+def post_table_import(
+    request: Request,
+    table_id: str,
+    input_file: Annotated[UploadFile, File()],
+    mode: Annotated[ImportModeEnum, Form()],
+    file_format: Annotated[ImportFileFormatEnum, Form()],
+    delimiter: Annotated[str, Form()],
+    encoding: Annotated[str, Form()],
+) -> HTMLResponse:
+    config: Config = request.app.state.config
+    table_config = next(
+        filter(lambda table_config: table_config.name == table_id, config.tables)
+    )
+    try:
+        table = load_table(table_config)
+        table_metadata = table.metadata()
+        rows_imported = import_file_to_table(
+            table_config, input_file.file, mode, file_format, delimiter, encoding
+        )
+        message = {
+            "type": "success",
+            "body": f"Successfully imported {rows_imported} rows",
+        }
+    except Exception as e:
+        message = {"type": "error", "body": str(e)}
+        table_metadata = None
+
+    return templates.TemplateResponse(
+        request=request,
+        name="tables/import.html",
+        context={
+            "tables": config.tables,
+            "queries": config.queries,
+            "table_id": table_id,
+            "table_metadata": table_metadata,
+            "message": message,
         },
     )
 
@@ -195,10 +313,7 @@ def get_query_view(request: Request, query_id: str) -> HTMLResponse:
     query_config = next(
         filter(lambda query_config: query_config.name == query_id, config.queries)
     )
-    tables_dataset = {
-        table_config.name: load_table(table_config).dataset()
-        for table_config in config.tables
-    }
+    tables_dataset = load_datasets(config.tables)
 
     try:
         results = execute_query(tables_dataset, query_config.sql)

{laketower-0.5.0.dist-info → laketower-0.6.0.dist-info}/METADATA

@@ -1,10 +1,10 @@
 Metadata-Version: 2.4
 Name: laketower
-Version: 0.5.0
+Version: 0.6.0
 Summary: Oversee your lakehouse
 Project-URL: Repository, https://github.com/datalpia/laketower
 Project-URL: Issues, https://github.com/datalpia/laketower/issues
-Project-URL: Changelog, https://github.com/datalpia/laketower/blob/master/CHANGELOG.md
+Project-URL: Changelog, https://github.com/datalpia/laketower/blob/main/CHANGELOG.md
 Author-email: Romain Clement <git@romain-clement.net>
 License: Apache-2.0
 License-File: LICENSE
@@ -22,7 +22,7 @@ Classifier: Topic :: Database
 Classifier: Topic :: Software Development
 Classifier: Topic :: Utilities
 Requires-Python: <3.14,>=3.10
-Requires-Dist: deltalake
+Requires-Dist: deltalake<2,>=1
 Requires-Dist: duckdb
 Requires-Dist: fastapi
 Requires-Dist: jinja2!=3.1.5,>=3
@@ -30,6 +30,7 @@ Requires-Dist: pandas
 Requires-Dist: pyarrow!=19.0.0
 Requires-Dist: pydantic-settings>=2
 Requires-Dist: pydantic>=2
+Requires-Dist: python-multipart
 Requires-Dist: pyyaml
 Requires-Dist: rich
 Requires-Dist: sqlglot
@@ -50,13 +51,16 @@ Utility application to explore and manage tables in your data lakehouse, especia
 ## Features
 
 - Delta Lake table format support
+- Remote tables support (S3, ADLS)
 - Inspect table metadata
 - Inspect table schema
 - Inspect table history
 - Get table statistics
+- Import data into a table from CSV files
 - View table content with a simple query builder
 - Query all registered tables with DuckDB SQL dialect
 - Execute saved queries
+- Export query results to CSV files
 - Static and versionable YAML configuration
 - Web application
 - CLI application
@@ -99,7 +103,9 @@ queries:
 
 Current limitations:
 
-- `tables.uri`: only local paths are allowed
+- `tables.uri`:
+    - Local paths are supported (`./path/to/table`, `/abs/path/to/table`, `file:///abs/path/to/table`)
+    - Remote paths to S3 (`s3://<bucket>/<path>`) and ADLS (`abfss://<container>/<path>`)
 - `tables.format`: only `delta` is allowed
 
 Example from the provided demo:
@@ -138,6 +144,103 @@ queries:
         day asc
 ```
 
+Support for environment variables substitution is also supported within the YAML
+configuration using a object containing a single key `env` with the name of the
+environment variable to be injected. The value of the variable can contain JSON
+and will be decoded in a best effort manner (default to string value). For instance:
+
+```yaml
+# export TABLE_URI=path/to/table
+
+tables:
+  - name: sample_table
+    uri:
+      env: TABLE_URI
+    format: delta
+```
+
+#### Remote S3 Tables
+
+Configuring S3 tables (AWS, MinIO, Cloudflare R2):
+
+```yaml
+tables:
+  - name: delta_table_s3
+    uri: s3://<bucket>/path/to/table
+    format: delta
+    connection:
+      s3:
+        s3_access_key_id: access-key-id
+        s3_secret_access_key: secret-access-key
+        s3_region: s3-region
+        s3_endpoint_url: http://s3.domain.com
+        s3_allow_http: false
+```
+
+Depending on your object storage location and configuration, one might have to
+set part or all the available `connection.s3` parameters. The only required ones
+are `s3_access_key_id` and `s3_secret_access_key`.
+
+Also as a security best practice, it is best not to write secrets directly in
+static configuration files, so one can use environment variables to all dynamic substitution,
+e.g.
+
+```yaml
+tables:
+  - name: delta_table_s3
+    uri: s3://<bucket>/path/to/table
+    format: delta
+    connection:
+      s3:
+        s3_access_key_id: access-key-id
+        s3_secret_access_key:
+          env: S3_SECRET_ACCESS_KEY
+        s3_region: s3-region
+        s3_endpoint_url: http://s3.domain.com
+        s3_allow_http: false
+```
+
+#### Remote ADLS Tables
+
+Configuring Azure ADLS tables:
+
+```yaml
+tables:
+  - name: delta_table_adls
+    uri: abfss://<container>/path/to/table
+    format: delta
+    connection:
+      adls:
+        adls_account_name: adls-account-name
+        adls_access_key: adls-access-key
+        adls_sas_key: adls-sas-key
+        adls_tenant_id: adls-tenant-id
+        adls_client_id: adls-client-id
+        adls_client_secret: adls-client-secret
+        azure_msi_endpoint: https://msi.azure.com
+        use_azure_cli: false
+```
+
+Depending on your object storage location and configuration, one might have to
+set part or all the available `connection.adls` parameters. The only required one
+is `adls_account_name`.
+
+Also as a security best practice, it is best not to write secrets directly in
+static configuration files, so one can use environment variables to all dynamic substitution,
+e.g.
+
+```yaml
+tables:
+  - name: delta_table_adls
+    uri: abfss://<container>/path/to/table
+    format: delta
+    connection:
+      adls:
+        adls_account_name: adls-account-name
+        adls_access_key:
+          env: ADLS_ACCESS_KEY
+```
+
 ### Web Application
 
 The easiest way to get started is to launch the Laketower web application:
@@ -148,12 +251,13 @@ $ laketower -c demo/laketower.yml web
 
 #### Screenshots
 
-![Laketower UI - Tables Overview](docs/static/tables_overview.png)
-![Laketower UI - Tables View](docs/static/tables_view.png)
-![Laketower UI - Tables Statistics](docs/static/tables_statistics.png)
-![Laketower UI - Tables History](docs/static/tables_history.png)
-![Laketower UI - Tables Query](docs/static/tables_query.png)
-![Laketower UI - Queries View](docs/static/queries_view.png)
+![Laketower UI - Tables Overview](https://raw.githubusercontent.com/datalpia/laketower/refs/heads/main/docs/static/tables_overview.png)
+![Laketower UI - Tables View](https://raw.githubusercontent.com/datalpia/laketower/refs/heads/main/docs/static/tables_view.png)
+![Laketower UI - Tables Statistics](https://raw.githubusercontent.com/datalpia/laketower/refs/heads/main/docs/static/tables_statistics.png)
+![Laketower UI - Tables History](https://raw.githubusercontent.com/datalpia/laketower/refs/heads/main/docs/static/tables_history.png)
+![Laketower UI - Tables Import](https://raw.githubusercontent.com/datalpia/laketower/refs/heads/main/docs/static/tables_import.png)
+![Laketower UI - Tables Query](https://raw.githubusercontent.com/datalpia/laketower/refs/heads/main/docs/static/tables_query.png)
+![Laketower UI - Queries View](https://raw.githubusercontent.com/datalpia/laketower/refs/heads/main/docs/static/queries_view.png)
 
 ### CLI
 
@@ -321,6 +425,29 @@ $ laketower -c demo/laketower.yml tables statistics --version 0 weather
 └──────────────────────┴───────┴──────┴──────┴──────┴──────┘
 ```
 
+#### Import data into a given table
+
+Import a CSV dataset into a table in append mode:
+
+```bash
+$ laketower -c demo/laketower.yml tables import weather --file data.csv --mode append --format csv --delimiter ',' --encoding 'utf-8'
+```
+
+`--mode` argument can be one of:
+- `append`: append rows to the table (default)
+- `overwrite`: replace all rows with the ones from the input file
+
+`--format` argument can be one of:
+- `csv`: CSV file format (default)
+
+`--delimiter` argument can be:
+- Any single character (only valid for CSV file format)
+- Default is _comma_ (`','`)
+
+`--encoding` argument can be:
+- Any [standard Python encoding](https://docs.python.org/3/library/codecs.html#standard-encodings),
+- Default is `'utf-8'`
+
 #### View a given table
 
 Using a simple query builder, the content of a table can be displayed.
@@ -400,6 +527,14 @@ $ laketower -c demo/laketower.yml tables query "select date_trunc('day', time) a
 └───────────────────────────┴────────────────────┘
 ```
 
+Export query results to CSV:
+
+```bash
+$ laketower -c demo/laketower.yml tables query --output results.csv "select date_trunc('day', time) as day, avg(temperature_2m) as mean_temperature from weather group by day order by day desc limit 3"
+
+Query results written to: results.csv
+```
+
 #### List saved queries
 
 ```bash

laketower-0.6.0.dist-info/RECORD

@@ -0,0 +1,23 @@
+laketower/__about__.py,sha256=cID1jLnC_vj48GgMN6Yb1FA3JsQ95zNmCHmRYE8TFhY,22
+laketower/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+laketower/__main__.py,sha256=czKxJKG8OfncnxWmpaOWx7b1JBwFnZNQi7wKSTncB4M,108
+laketower/cli.py,sha256=tvCr90q4jRVAoP2qzwhTLG7PUsae0QOGQwJRid3GVLc,15324
+laketower/config.py,sha256=uIQSE1MjEuA-kp0TwA0QREwPbaNGL9hLGmKWqNaA8VY,3298
+laketower/tables.py,sha256=gs4klWJkyyS7_oIDz1HKFicXAF6jbGfvzJWrDw8r-rQ,10235
+laketower/web.py,sha256=-cXg_8pUCdhU5QF6WH_3luXi545F0Et9rpSC05J63Ng,10736
+laketower/static/.gitkeep,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+laketower/templates/_base.html,sha256=S-8kjAfYBx3Btb4FwzM2qyfkGYrOBHhpvCWR32mCvOw,3729
+laketower/templates/index.html,sha256=dLF2Og0qgzBkvGyVRidRNzTv0u4o97ifOx1jVeig8Kg,59
+laketower/templates/queries/view.html,sha256=naqU3XGyVVW6Er8wQ95-DYlp38Czolvh-h5noIAWs84,1978
+laketower/templates/tables/_macros.html,sha256=sCI1TOFW0QA74oSXW87H6dNTudOs7n-FretnTPFcRh4,1174
+laketower/templates/tables/history.html,sha256=a5GBLXCiLlbWno5eR0XT5i_oMAghylUBBFOpr27NB3Q,1853
+laketower/templates/tables/import.html,sha256=bQZwRrv84tDBuf0AHJyc7L-PjW-XSoZhMHNDIo6TP4c,2604
+laketower/templates/tables/index.html,sha256=saNdQbJAjMJAzayTk4rA5Mmw_bCXvor2WpghVmoWSAI,2507
+laketower/templates/tables/query.html,sha256=ymWcqZj4TtJgUeCIMseJD0PIOqy0gf1SVzrQzN9UD5Q,1652
+laketower/templates/tables/statistics.html,sha256=h6TiQtFwiRWvPqDphcRRF1rZ886FP00UbJuMHuW5l6U,1827
+laketower/templates/tables/view.html,sha256=ruiAX_S--wpodmgEbcQ-GT7BQzz-vzSCk4NpzlO3I80,3985
+laketower-0.6.0.dist-info/METADATA,sha256=HMP4sBtBKVgvO0Ay_KIJd6que4EKaICfXLXYA9TQ12o,25496
+laketower-0.6.0.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+laketower-0.6.0.dist-info/entry_points.txt,sha256=sJpQgRwdeZhRBudNqBTqtHPCE-uLC9YgFXJY2CTEyCk,53
+laketower-0.6.0.dist-info/licenses/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+laketower-0.6.0.dist-info/RECORD,,

laketower-0.6.0.dist-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+laketower = laketower.__main__:cli

laketower-0.5.0.dist-info/RECORD

@@ -1,22 +0,0 @@
-laketower/__about__.py,sha256=LBK46heutvn3KmsCrKIYu8RQikbfnjZaj2xFrXaeCzQ,22
-laketower/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-laketower/__main__.py,sha256=czKxJKG8OfncnxWmpaOWx7b1JBwFnZNQi7wKSTncB4M,108
-laketower/cli.py,sha256=U4gI12egcOs51wxjmQlU70XhA2QGcowc0AmTYpUKEFE,11962
-laketower/config.py,sha256=NdUDF7lr2hEW9Gujp0OpkOKcDP46ju1y_r0IM4Hrx2M,1100
-laketower/tables.py,sha256=QwqoK73Q9pDRzyuoN9pwmwP_WWj3Rg2qPJaIcCdnJbw,4402
-laketower/web.py,sha256=5NMKj26aVz3cKnUAe-3sLDJ_4Ue3u0VXhATrDQ8GVF8,7205
-laketower/static/.gitkeep,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-laketower/templates/_base.html,sha256=S-8kjAfYBx3Btb4FwzM2qyfkGYrOBHhpvCWR32mCvOw,3729
-laketower/templates/index.html,sha256=dLF2Og0qgzBkvGyVRidRNzTv0u4o97ifOx1jVeig8Kg,59
-laketower/templates/queries/view.html,sha256=2jW08X-PflXmsfy7u9ibX1BF8G5dx9X_n_Eoq3jzizA,1686
-laketower/templates/tables/_macros.html,sha256=fnj_8nBco0iS6mlBmGmfT2PZhI2Y82yP0cm8tRxchpU,965
-laketower/templates/tables/history.html,sha256=yAW0xw9_Uxp0QZYKje6qhcbpeznxI3fb740hfNyILZ8,1740
-laketower/templates/tables/index.html,sha256=oY13l_p8qozlLONanLpga1WhEo4oTP92pRf9sBSuFZI,2394
-laketower/templates/tables/query.html,sha256=YAFnW8Q5abDsbeglFHHZmJfGJXPjI2s4Nxf6gF_-Eg0,1360
-laketower/templates/tables/statistics.html,sha256=rgIOuF2PlHo2jvcYDAnxa5ObNortwyALlrURpM7qxMw,1714
-laketower/templates/tables/view.html,sha256=psfeRKkN19Q3Ko5Sm2570qRhehvuoEBPG89zFU5KQlc,3872
-laketower-0.5.0.dist-info/METADATA,sha256=AKQgJ1YSUgNA3bXyyBweGY3BAMDtR2Mlr2SzQFZZ8P8,20935
-laketower-0.5.0.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-laketower-0.5.0.dist-info/entry_points.txt,sha256=OL_4klopvyEzasJOFJ-sKu54lv24Jvomni32h1WVUjk,48
-laketower-0.5.0.dist-info/licenses/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
-laketower-0.5.0.dist-info/RECORD,,

laketower-0.5.0.dist-info/entry_points.txt

@@ -1,2 +0,0 @@
-[console_scripts]
-laketower = laketower:cli.cli