laketower 0.5.1__py3-none-any.whl → 0.6.5__py3-none-any.whl

laketower/static/vendor/halfmoon/halfmoon.modern.css

@@ -0,0 +1,282 @@
+
+/*!
+ * ----------------------------------------------------------------------------
+ * Halfmoon CSS - Modern theme
+ * Copyright (c) 2023, Tahmid Khan | MIT License | https://www.gethalfmoon.com
+ * ----------------------------------------------------------------------------
+ * The above notice must be included in its entirety when this file is used.
+ */
+
+/* Color palette */
+
+[data-bs-core=modern] {
+	/* Gray */
+
+	--bs-slate-hue: 216;
+	--bs-slate-saturation: 20%;
+
+	/* Light gray */
+
+	--bs-lightgray-hue: var(--bs-slate-hue);
+	--bs-lightgray-saturation: var(--bs-slate-saturation);
+
+	/* Sable (almost black) */
+
+	--bs-sable-hue: var(--bs-darkgray-hue);
+	--bs-sable-saturation: var(--bs-darkgray-saturation);
+	--bs-sable-100-hsl: var(--bs-sable-hue), var(--bs-sable-saturation), 31%;
+	--bs-sable-200-hsl: var(--bs-sable-hue), var(--bs-sable-saturation), 29%;
+	--bs-sable-300-hsl: var(--bs-sable-hue), var(--bs-sable-saturation), 27%;
+	--bs-sable-400-hsl: var(--bs-sable-hue), var(--bs-sable-saturation), 25%;
+	--bs-sable-500-hsl: var(--bs-sable-hue), var(--bs-sable-saturation), 23%;
+	--bs-sable-600-hsl: var(--bs-sable-hue), var(--bs-sable-saturation), 21%;
+	--bs-sable-700-hsl: var(--bs-sable-hue), var(--bs-sable-saturation), 19%;
+	--bs-sable-800-hsl: var(--bs-sable-hue), var(--bs-sable-saturation), 17%;
+	--bs-sable-900-hsl: var(--bs-sable-hue), var(--bs-sable-saturation), 15%;
+	--bs-sable-100: hsl(var(--bs-sable-100-hsl));
+	--bs-sable-200: hsl(var(--bs-sable-200-hsl));
+	--bs-sable-300: hsl(var(--bs-sable-300-hsl));
+	--bs-sable-400: hsl(var(--bs-sable-400-hsl));
+	--bs-sable-500: hsl(var(--bs-sable-500-hsl));
+	--bs-sable-600: hsl(var(--bs-sable-600-hsl));
+	--bs-sable-700: hsl(var(--bs-sable-700-hsl));
+	--bs-sable-800: hsl(var(--bs-sable-800-hsl));
+	--bs-sable-900: hsl(var(--bs-sable-900-hsl));
+	--bs-sable-hsl: var(--bs-sable-500-hsl);
+	--bs-sable: hsl(var(--bs-sable-hsl));
+	--bs-sable-foreground-hsl: var(--bs-white-hsl);
+	--bs-sable-foreground: hsl(var(--bs-sable-foreground-hsl));
+	--bs-sable-text-emphasis-hsl: var(--bs-sable-600-hsl);
+	--bs-sable-text-emphasis: hsl(var(--bs-sable-text-emphasis-hsl));
+	--bs-sable-hover-bg: var(--bs-sable-600);
+	--bs-sable-active-bg: var(--bs-sable-700);
+	--bs-sable-bg-subtle: hsl(var(--bs-sable-hue), var(--bs-sable-saturation), 70%);
+	--bs-sable-border-subtle: var(--bs-sable-400);
+	--bs-sable-checkbox-svg: var(--bs-checkbox-svg-light);
+	--bs-sable-dash-svg: var(--bs-dash-svg-light);
+	--bs-sable-radio-svg: var(--bs-radio-svg-light);
+	--bs-sable-switch-svg: var(--bs-switch-svg-light);
+
+	/* Primary */
+
+	--bs-primary-hue: var(--bs-navy-hue);
+	--bs-primary-saturation: var(--bs-navy-saturation);
+	--bs-primary-100-hsl: var(--bs-navy-100-hsl);
+	--bs-primary-200-hsl: var(--bs-navy-200-hsl);
+	--bs-primary-300-hsl: var(--bs-navy-300-hsl);
+	--bs-primary-400-hsl: var(--bs-navy-400-hsl);
+	--bs-primary-500-hsl: var(--bs-navy-500-hsl);
+	--bs-primary-600-hsl: var(--bs-navy-600-hsl);
+	--bs-primary-700-hsl: var(--bs-navy-700-hsl);
+	--bs-primary-800-hsl: var(--bs-navy-800-hsl);
+	--bs-primary-900-hsl: var(--bs-navy-900-hsl);
+	--bs-primary-100: var(--bs-navy-100);
+	--bs-primary-200: var(--bs-navy-200);
+	--bs-primary-300: var(--bs-navy-300);
+	--bs-primary-400: var(--bs-navy-400);
+	--bs-primary-500: var(--bs-navy-500);
+	--bs-primary-600: var(--bs-navy-600);
+	--bs-primary-700: var(--bs-navy-700);
+	--bs-primary-800: var(--bs-navy-800);
+	--bs-primary-900: var(--bs-navy-900);
+	--bs-primary-hsl: var(--bs-navy-hsl);
+	--bs-primary: var(--bs-navy);
+	--bs-primary-foreground-hsl: var(--bs-navy-foreground-hsl);
+	--bs-primary-foreground: var(--bs-navy-foreground);
+	--bs-primary-text-emphasis-hsl: var(--bs-navy-text-emphasis-hsl);
+	--bs-primary-text-emphasis: var(--bs-navy-text-emphasis);
+	--bs-primary-hover-bg: var(--bs-navy-hover-bg);
+	--bs-primary-active-bg: var(--bs-navy-active-bg);
+	--bs-primary-bg-subtle: var(--bs-navy-bg-subtle);
+	--bs-primary-border-subtle: var(--bs-navy-border-subtle);
+	--bs-primary-checkbox-svg: var(--bs-navy-checkbox-svg);
+	--bs-primary-dash-svg: var(--bs-navy-dash-svg);
+	--bs-primary-radio-svg: var(--bs-navy-radio-svg);
+	--bs-primary-switch-svg: var(--bs-navy-switch-svg);
+
+	/* Info */
+
+	--bs-info-hue: var(--bs-blue-hue);
+	--bs-info-saturation: var(--bs-blue-saturation);
+	--bs-info-100-hsl: var(--bs-blue-100-hsl);
+	--bs-info-200-hsl: var(--bs-blue-200-hsl);
+	--bs-info-300-hsl: var(--bs-blue-300-hsl);
+	--bs-info-400-hsl: var(--bs-blue-400-hsl);
+	--bs-info-500-hsl: var(--bs-blue-500-hsl);
+	--bs-info-600-hsl: var(--bs-blue-600-hsl);
+	--bs-info-700-hsl: var(--bs-blue-700-hsl);
+	--bs-info-800-hsl: var(--bs-blue-800-hsl);
+	--bs-info-900-hsl: var(--bs-blue-900-hsl);
+	--bs-info-100: var(--bs-blue-100);
+	--bs-info-200: var(--bs-blue-200);
+	--bs-info-300: var(--bs-blue-300);
+	--bs-info-400: var(--bs-blue-400);
+	--bs-info-500: var(--bs-blue-500);
+	--bs-info-600: var(--bs-blue-600);
+	--bs-info-700: var(--bs-blue-700);
+	--bs-info-800: var(--bs-blue-800);
+	--bs-info-900: var(--bs-blue-900);
+	--bs-info-hsl: var(--bs-blue-hsl);
+	--bs-info: var(--bs-blue);
+	--bs-info-foreground-hsl: var(--bs-blue-foreground-hsl);
+	--bs-info-foreground: var(--bs-blue-foreground);
+	--bs-info-text-emphasis-hsl: var(--bs-blue-text-emphasis-hsl);
+	--bs-info-text-emphasis: var(--bs-blue-text-emphasis);
+	--bs-info-hover-bg: var(--bs-blue-hover-bg);
+	--bs-info-active-bg: var(--bs-blue-active-bg);
+	--bs-info-bg-subtle: var(--bs-blue-bg-subtle);
+	--bs-info-border-subtle: var(--bs-blue-border-subtle);
+	--bs-info-checkbox-svg: var(--bs-blue-checkbox-svg);
+	--bs-info-dash-svg: var(--bs-blue-dash-svg);
+	--bs-info-radio-svg: var(--bs-blue-radio-svg);
+	--bs-info-switch-svg: var(--bs-blue-switch-svg);
+}
+
+[data-bs-core=modern][data-bs-theme=dark] {
+	/* Dark gray */
+
+	--bs-darkgray-text-emphasis-hsl: var(--bs-darkgray-200-hsl);
+	--bs-darkgray-text-emphasis: hsl(var(--bs-darkgray-text-emphasis-hsl));
+
+	/* Sable (black) */
+
+	--bs-sable-text-emphasis-hsl: var(--bs-sable-400-hsl);
+	--bs-sable-text-emphasis: hsl(var(--bs-sable-text-emphasis-hsl));
+	--bs-sable-bg-subtle: hsl(var(--bs-sable-hue), var(--bs-sable-saturation), 14%);
+	--bs-sable-border-subtle: var(--bs-sable-600);
+
+	/* Blue */
+
+	--bs-blue-text-emphasis-hsl: var(--bs-blue-300-hsl);
+	--bs-blue-text-emphasis: hsl(var(--bs-blue-text-emphasis-hsl));
+
+	/* Primary */
+
+	--bs-primary-hue: var(--bs-sky-hue);
+	--bs-primary-saturation: var(--bs-sky-saturation);
+	--bs-primary-100-hsl: var(--bs-sky-100-hsl);
+	--bs-primary-200-hsl: var(--bs-sky-200-hsl);
+	--bs-primary-300-hsl: var(--bs-sky-300-hsl);
+	--bs-primary-400-hsl: var(--bs-sky-400-hsl);
+	--bs-primary-500-hsl: var(--bs-sky-500-hsl);
+	--bs-primary-600-hsl: var(--bs-sky-600-hsl);
+	--bs-primary-700-hsl: var(--bs-sky-700-hsl);
+	--bs-primary-800-hsl: var(--bs-sky-800-hsl);
+	--bs-primary-900-hsl: var(--bs-sky-900-hsl);
+	--bs-primary-100: var(--bs-sky-100);
+	--bs-primary-200: var(--bs-sky-200);
+	--bs-primary-300: var(--bs-sky-300);
+	--bs-primary-400: var(--bs-sky-400);
+	--bs-primary-500: var(--bs-sky-500);
+	--bs-primary-600: var(--bs-sky-600);
+	--bs-primary-700: var(--bs-sky-700);
+	--bs-primary-800: var(--bs-sky-800);
+	--bs-primary-900: var(--bs-sky-900);
+	--bs-primary-hsl: var(--bs-sky-hsl);
+	--bs-primary: var(--bs-sky);
+	--bs-primary-foreground-hsl: var(--bs-sky-foreground-hsl);
+	--bs-primary-foreground: var(--bs-sky-foreground);
+	--bs-primary-text-emphasis-hsl: var(--bs-sky-text-emphasis-hsl);
+	--bs-primary-text-emphasis: var(--bs-sky-text-emphasis);
+	--bs-primary-hover-bg: var(--bs-sky-hover-bg);
+	--bs-primary-active-bg: var(--bs-sky-active-bg);
+	--bs-primary-bg-subtle: var(--bs-sky-bg-subtle);
+	--bs-primary-border-subtle: var(--bs-sky-border-subtle);
+	--bs-primary-checkbox-svg: var(--bs-sky-checkbox-svg);
+	--bs-primary-dash-svg: var(--bs-sky-dash-svg);
+	--bs-primary-radio-svg: var(--bs-sky-radio-svg);
+	--bs-primary-switch-svg: var(--bs-sky-switch-svg);
+
+	/* Info */
+
+	--bs-info-text-emphasis-hsl: var(--bs-blue-text-emphasis-hsl);
+	--bs-info-text-emphasis: var(--bs-blue-text-emphasis);
+	--bs-info-bg-subtle: var(--bs-blue-bg-subtle);
+	--bs-info-border-subtle: var(--bs-blue-border-subtle);
+}
+
+/* Variables */
+
+[data-bs-core=modern] {
+	/* Link */
+
+	--bs-link-color-hsl: var(--bs-info-text-emphasis-hsl);
+	--bs-link-hover-color-hsl: var(--bs-info-hsl);
+
+	/* Content (used as needed in cards, panels, menus, etc.) */
+
+	--bs-content-bg-hsl: var(--bs-body-bg-hsl);
+	--bs-content-border-color: var(--bs-border-color);
+
+	/* Form */
+
+	--bs-form-focus-border-color: var(--bs-info-border-subtle);
+	--bs-form-focus-shadow-hsl: var(--bs-info-hsl);
+	--bs-form-check-focus-border-color: var(--bs-info-border-subtle);
+}
+
+[data-bs-core=modern]:not([data-bs-theme=dark]) {
+	/* Background */
+
+	--bs-body-bg-hsl: var(--bs-white-hsl);
+	--bs-secondary-bg-hsl: var(--bs-lightgray-hue), var(--bs-lightgray-saturation), 98.75%;
+	--bs-tertiary-bg-hsl: var(--bs-lightgray-hue), var(--bs-lightgray-saturation), 97.5%;
+
+	/* Border */
+
+	--bs-border-color: var(--bs-lightgray-700);
+	--bs-border-color-light: var(--bs-lightgray-500);
+}
+
+[data-bs-core=modern][data-bs-theme=dark] {
+	/* Background */
+
+	--bs-body-bg-hsl: var(--bs-sable-900-hsl);
+	--bs-secondary-bg-hsl: var(--bs-sable-800-hsl);
+	--bs-tertiary-bg-hsl: var(--bs-sable-700-hsl);
+
+	/* Border */
+
+	--bs-border-color: var(--bs-gray-900);
+
+	/* Content (used as needed in cards, panels, menus, etc.) */
+
+	--bs-content-floating-bg-hsl: var(--bs-sable-hue), var(--bs-sable-saturation), 16.5%;
+
+	/* Action (used as needed in buttons, inputs, menu items, page links, etc.) */
+
+	--bs-action-border-color: var(--bs-border-color);
+
+	/* Contextual buttons */
+
+	--bs-ctx-btn-border-color: transparent;
+	--bs-ctx-btn-bg-clip: border-box;
+
+	/* Action bar (used as needed in range, progress, etc.) */
+
+	--bs-actionbar-border-color: hsla(var(--bs-white-hsl), 0.075);
+	--bs-progresstrack-border-width: 0;
+	--bs-progresstrack-box-shadow: inset 0 0 0 var(--bs-border-width) var(--bs-actionbar-border-color);
+	--bs-progresstrack-bg-clip: border-box;
+}
+
+/* Sidebar */
+
+[data-bs-core=modern] .sidebar {
+	--bs-sidebar-item-padding-x: 1rem;
+	--bs-sidebar-item-padding-y: 0.25rem;
+	--bs-sidebar-header-font-weight: var(--bs-font-weight-bold);
+	--bs-sidebar-divider-bg: var(--bs-sidebar-bg);
+}
+
+[data-bs-core=modern] .sidebar-nav .nav-link {
+	border-left: var(--bs-border-width) solid var(--bs-border-color-light);
+}
+
+[data-bs-core=modern] .sidebar-nav .nav-link.active,
+[data-bs-core=modern] .sidebar-nav .nav-link.show {
+	font-weight: var(--bs-font-weight-bold);
+	border-color: currentColor;
+	-webkit-font-smoothing: antialiased;
+	-moz-osx-font-smoothing: grayscale;
+}

laketower/tables.py

@@ -1,14 +1,16 @@
+import enum
 from datetime import datetime, timezone
-from typing import Any, Protocol
+from typing import Any, BinaryIO, Protocol, TextIO
 
 import deltalake
 import duckdb
-import pandas as pd
 import pyarrow as pa
+import pyarrow.csv as csv
 import pyarrow.dataset as padataset
 import pydantic
 import sqlglot
 import sqlglot.dialects.duckdb
+import sqlglot.errors
 import sqlglot.expressions
 
 from laketower.config import ConfigTable, TableFormats
@@ -17,6 +19,15 @@ from laketower.config import ConfigTable, TableFormats
 DEFAULT_LIMIT = 10
 
 
+class ImportModeEnum(str, enum.Enum):
+    append = "append"
+    overwrite = "overwrite"
+
+
+class ImportFileFormatEnum(str, enum.Enum):
+    csv = "csv"
+
+
 class TableMetadata(pydantic.BaseModel):
     table_format: TableFormats
     name: str | None = None
@@ -43,17 +54,112 @@ class TableHistory(pydantic.BaseModel):
 
 
 class TableProtocol(Protocol):  # pragma: no cover
+    @classmethod
+    def is_valid(cls, table_config: ConfigTable) -> bool: ...
+    def __init__(self, table_config: ConfigTable) -> None: ...
     def metadata(self) -> TableMetadata: ...
     def schema(self) -> pa.Schema: ...
     def history(self) -> TableHistory: ...
     def dataset(self, version: int | str | None = None) -> padataset.Dataset: ...
+    def import_data(
+        self, data: pa.Table, mode: ImportModeEnum = ImportModeEnum.append
+    ) -> None: ...
 
 
 class DeltaTable:
     def __init__(self, table_config: ConfigTable):
         super().__init__()
         self.table_config = table_config
-        self._impl = deltalake.DeltaTable(table_config.uri)
+        storage_options = self._generate_storage_options(table_config)
+        self._impl = deltalake.DeltaTable(
+            table_config.uri, storage_options=storage_options
+        )
+
+    @classmethod
+    def _generate_storage_options(
+        cls, table_config: ConfigTable
+    ) -> dict[str, str] | None:
+        # documentation from `object-store` Rust crate:
+        # - s3: https://docs.rs/object_store/latest/object_store/aws/enum.AmazonS3ConfigKey.html
+        # - adls: https://docs.rs/object_store/latest/object_store/azure/enum.AzureConfigKey.html
+        storage_options = None
+        conn_s3 = (
+            table_config.connection.s3
+            if table_config.connection and table_config.connection.s3
+            else None
+        )
+        conn_adls = (
+            table_config.connection.adls
+            if table_config.connection and table_config.connection.adls
+            else None
+        )
+        if conn_s3:
+            storage_options = (
+                {
+                    "aws_access_key_id": conn_s3.s3_access_key_id,
+                    "aws_secret_access_key": conn_s3.s3_secret_access_key.get_secret_value(),
+                    "aws_allow_http": str(conn_s3.s3_allow_http).lower(),
+                }
+                | ({"aws_region": conn_s3.s3_region} if conn_s3.s3_region else {})
+                | (
+                    {"aws_endpoint_url": str(conn_s3.s3_endpoint_url).rstrip("/")}
+                    if conn_s3.s3_endpoint_url
+                    else {}
+                )
+            )
+        elif conn_adls:
+            storage_options = (
+                {
+                    "azure_storage_account_name": conn_adls.adls_account_name,
+                    "azure_use_azure_cli": str(conn_adls.use_azure_cli).lower(),
+                }
+                | (
+                    {
+                        "azure_storage_access_key": conn_adls.adls_access_key.get_secret_value()
+                    }
+                    if conn_adls.adls_access_key
+                    else {}
+                )
+                | (
+                    {"azure_storage_sas_key": conn_adls.adls_sas_key.get_secret_value()}
+                    if conn_adls.adls_sas_key
+                    else {}
+                )
+                | (
+                    {"azure_storage_tenant_id": conn_adls.adls_tenant_id}
+                    if conn_adls.adls_tenant_id
+                    else {}
+                )
+                | (
+                    {"azure_storage_client_id": conn_adls.adls_client_id}
+                    if conn_adls.adls_client_id
+                    else {}
+                )
+                | (
+                    {
+                        "azure_storage_client_secret": conn_adls.adls_client_secret.get_secret_value()
+                    }
+                    if conn_adls.adls_client_secret
+                    else {}
+                )
+                | (
+                    {
+                        "azure_msi_endpoint": str(conn_adls.azure_msi_endpoint).rstrip(
+                            "/"
+                        )
+                    }
+                    if conn_adls.azure_msi_endpoint
+                    else {}
+                )
+            )
+        return storage_options
+
+    @classmethod
+    def is_valid(cls, table_config: ConfigTable) -> bool:
+        storage_options = cls._generate_storage_options(table_config)
+        return deltalake.DeltaTable.is_deltatable(
+            table_config.uri, storage_options=storage_options
+        )
 
     def metadata(self) -> TableMetadata:
         metadata = self._impl.metadata()
@@ -96,10 +202,47 @@ class DeltaTable:
             self._impl.load_as_version(version)
         return self._impl.to_pyarrow_dataset()
 
+    def import_data(
+        self, data: pa.Table, mode: ImportModeEnum = ImportModeEnum.append
+    ) -> None:
+        deltalake.write_deltalake(
+            self.table_config.uri, data, mode=mode.value, schema_mode="merge"
+        )
+
 
 def load_table(table_config: ConfigTable) -> TableProtocol:
-    format_handler = {TableFormats.delta: DeltaTable}
-    return format_handler[table_config.table_format](table_config)
+    format_handler: dict[TableFormats, type[TableProtocol]] = {
+        TableFormats.delta: DeltaTable
+    }
+    table_handler = format_handler[table_config.table_format]
+    if not table_handler.is_valid(table_config):
+        raise ValueError(f"Invalid table: {table_config.uri}")
+    return table_handler(table_config)
+
+
+def load_datasets(table_configs: list[ConfigTable]) -> dict[str, padataset.Dataset]:
+    tables_dataset = {}
+    for table_config in table_configs:
+        try:
+            tables_dataset[table_config.name] = load_table(table_config).dataset()
+        except ValueError:
+            pass
+    return tables_dataset
+
+
+def extract_query_parameter_names(sql: str) -> set[str]:
+    try:
+        parsed_sql = sqlglot.parse(sql, dialect=sqlglot.dialects.duckdb.DuckDB)
+    except sqlglot.errors.SqlglotError as e:
+        raise ValueError(f"Error: {e}") from e
+
+    return {
+        str(node.this)
+        for statement in parsed_sql
+        if statement is not None
+        for node in statement.walk()
+        if isinstance(node, sqlglot.expressions.Placeholder)
+    }
 
 
 def generate_table_query(
@@ -110,32 +253,91 @@ def generate_table_query(
     sort_desc: str | None = None,
 ) -> str:
     query_expr = (
-        sqlglot.select(*(cols or ["*"])).from_(table_name).limit(limit or DEFAULT_LIMIT)
+        sqlglot.select(*([f'"{col}"' for col in cols] if cols else ["*"]))
+        .from_(f'"{table_name}"')
+        .limit(limit or DEFAULT_LIMIT)
     )
     if sort_asc:
         query_expr = query_expr.order_by(f"{sort_asc} asc")
     elif sort_desc:
         query_expr = query_expr.order_by(f"{sort_desc} desc")
-    return sqlglot.Generator(dialect=sqlglot.dialects.duckdb.DuckDB).generate(
-        query_expr
-    )
+    return query_expr.sql(dialect=sqlglot.dialects.duckdb.DuckDB, identify="always")
 
 
 def generate_table_statistics_query(table_name: str) -> str:
-    return (
-        f"SELECT column_name, count, avg, std, min, max FROM (SUMMARIZE {table_name})"  # nosec B608
+    summarize_expr = sqlglot.expressions.Summarize(
+        this=sqlglot.expressions.Table(this=f'"{table_name}"')
+    )
+    subquery_expr = sqlglot.expressions.Subquery(this=summarize_expr)
+    query_expr = sqlglot.select(
+        "column_name", "count", "avg", "std", "min", "max"
+    ).from_(subquery_expr)
+    return query_expr.sql(dialect=sqlglot.dialects.duckdb.DuckDB, identify="always")
+
+
+def limit_query(sql_query: str, max_limit: int) -> str:
+    try:
+        query_ast = sqlglot.parse(sql_query, dialect=sqlglot.dialects.duckdb.DuckDB)
+    except sqlglot.errors.SqlglotError as e:
+        raise ValueError(f"Error: {e}") from e
+
+    if query_ast and isinstance(query_ast[-1], sqlglot.expressions.Select):
+        limit_wrapper = (
+            sqlglot.select("*")
+            .from_(sqlglot.expressions.Subquery(this=query_ast[-1]))
+            .limit(max_limit)
+        )
+        query_ast[-1] = limit_wrapper
+
+    return "; ".join(
+        [
+            stmt.sql(dialect=sqlglot.dialects.duckdb.DuckDB, identify="always")
+            for stmt in query_ast
+            if stmt is not None
+        ]
     )
 
 
 def execute_query(
-    tables_datasets: dict[str, padataset.Dataset], sql_query: str
-) -> pd.DataFrame:
+    tables_datasets: dict[str, padataset.Dataset],
+    sql_query: str,
+    sql_params: dict[str, str] = {},
+) -> pa.Table:
+    if not sql_query:
+        raise ValueError("Error: Cannot execute empty SQL query")
+
     try:
         conn = duckdb.connect()
         for table_name, table_dataset in tables_datasets.items():
+            # ATTACH IF NOT EXISTS ':memory:' AS {catalog.name};
+            # CREATE SCHEMA IF NOT EXISTS {catalog.name}.{database.name};
+            # USE {catalog.name}.{database.name};
+            # CREATE VIEW IF NOT EXISTS {table.name} AS FROM {table.name}_dataset;
+
             view_name = f"{table_name}_view"
             conn.register(view_name, table_dataset)
-            conn.execute(f"create table {table_name} as select * from {view_name}")  # nosec B608
-        return conn.execute(sql_query).df()
+            conn.execute(f'create view "{table_name}" as select * from "{view_name}"')  # nosec B608
+        return conn.execute(sql_query, parameters=sql_params).fetch_arrow_table()
     except duckdb.Error as e:
-        raise ValueError(str(e)) from e
+        raise ValueError(f"Error: {e}") from e
+
+
+def import_file_to_table(
+    table_config: ConfigTable,
+    file_path: BinaryIO | TextIO,
+    mode: ImportModeEnum = ImportModeEnum.append,
+    file_format: ImportFileFormatEnum = ImportFileFormatEnum.csv,
+    delimiter: str = ",",
+    encoding: str = "utf-8",
+) -> int:
+    file_format_handler = {
+        ImportFileFormatEnum.csv: lambda f, d, e: csv.read_csv(
+            f,
+            read_options=csv.ReadOptions(encoding=e),
+            parse_options=csv.ParseOptions(delimiter=d),
+        )
+    }
+    table = load_table(table_config)
+    df = file_format_handler[file_format](file_path, delimiter, encoding)
+    table.import_data(df, mode=mode)
+    return len(df)