squirrels 0.5.0rc0__py3-none-any.whl → 0.5.1__py3-none-any.whl

squirrels/_utils.py

@@ -1,11 +1,9 @@
-from typing import Sequence, Optional, Union, TypeVar, Callable, Any, Iterable
+from typing import Sequence, Optional, Union, TypeVar, Callable, Iterable, Literal, Any
 from datetime import datetime
 from pathlib import Path
-from functools import lru_cache
-from pydantic import BaseModel
 import os, time, logging, json, duckdb, polars as pl, yaml
 import jinja2 as j2, jinja2.nodes as j2_nodes
-import sqlglot, sqlglot.expressions, asyncio
+import sqlglot, sqlglot.expressions, asyncio, hashlib, inspect, base64
 
 from . import _constants as c
 from ._exceptions import ConfigurationError
@@ -20,7 +18,7 @@ polars_dtypes_to_sqrl_dtypes: dict[type[pl.DataType], list[str]] = {
     pl.Int32: ["integer", "int", "int4"],
     pl.Int64: ["bigint", "long", "int8"],
     pl.Float32: ["float", "float4", "real"],
-    pl.Float64: ["double", "float8"],
+    pl.Float64: ["double", "float8", "decimal"], # Note: Polars Decimal type is considered unstable, so we use Float64 for "decimal"
     pl.Boolean: ["boolean", "bool", "logical"],
     pl.Date: ["date"],
     pl.Time: ["time"],
@@ -35,12 +33,20 @@ sqrl_dtypes_to_polars_dtypes: dict[str, type[pl.DataType]] = {sqrl_type: k for k
 ## Other utility classes
 
 class Logger(logging.Logger):
-    def log_activity_time(self, activity: str, start_timestamp: float, *, request_id: str | None = None) -> None:
+    def info(self, msg: str, *, data: dict[str, Any] = {}, **kwargs) -> None:
+        super().info(msg, extra={"data": data}, **kwargs)
+
+    def log_activity_time(self, activity: str, start_timestamp: float, *, additional_data: dict[str, Any] = {}) -> None:
         end_timestamp = time.time()
         time_taken = round((end_timestamp-start_timestamp) * 10**3, 3)
-        data = { "activity": activity, "start_timestamp": start_timestamp, "end_timestamp": end_timestamp, "time_taken_ms": time_taken }
-        info = { "request_id": request_id } if request_id else {}
-        self.info(f'Time taken for "{activity}": {time_taken}ms', extra={"data": data, "info": info})
+        data = {
+            "activity": activity,
+            "start_timestamp": start_timestamp, 
+            "end_timestamp": end_timestamp, 
+            "time_taken_ms": time_taken,
+            **additional_data
+        }
+        self.info(f'Time taken for "{activity}": {time_taken}ms', data=data)
 
 
 class EnvironmentWithMacros(j2.Environment):
@@ -85,14 +91,6 @@ class EnvironmentWithMacros(j2.Environment):
 
 ## Utility functions/variables
 
-def log_activity_time(logger: logging.Logger, activity: str, start_timestamp: float, *, request_id: str | None = None) -> None:
-    end_timestamp = time.time()
-    time_taken = round((end_timestamp-start_timestamp) * 10**3, 3)
-    data = { "activity": activity, "start_timestamp": start_timestamp, "end_timestamp": end_timestamp, "time_taken_ms": time_taken }
-    info = { "request_id": request_id } if request_id else {}
-    logger.debug(f'Time taken for "{activity}": {time_taken}ms', extra={"data": data, "info": info})
-
-
 def render_string(raw_str: str, *, base_path: str = ".", **kwargs) -> str:
     """
     Given a template string, render it with the given keyword arguments
@@ -128,7 +126,7 @@ def read_file(filepath: FilePath) -> str:
 
 def normalize_name(name: str) -> str:
     """
-    Normalizes names to the convention of the squirrels manifest file.
+    Normalizes names to the convention of the squirrels manifest file (with underscores instead of dashes).
 
     Arguments:
         name: The name to normalize.
@@ -141,7 +139,7 @@ def normalize_name(name: str) -> str:
 
 def normalize_name_for_api(name: str) -> str:
     """
-    Normalizes names to the REST API convention.
+    Normalizes names to the REST API convention (with dashes instead of underscores).
 
     Arguments:
         name: The name to normalize.
@@ -196,8 +194,10 @@ def process_if_not_none(input_val: Optional[X], processor: Callable[[X], Y]) ->
     return processor(input_val)
 
 
-@lru_cache(maxsize=1)
-def _read_duckdb_init_sql() -> tuple[str, Path | None]:
+def _read_duckdb_init_sql(
+    *,
+    datalake_db_path: str | None = None,
+) -> str:
     """
     Reads and caches the duckdb init file content.
     Returns None if file doesn't exist or is empty.
@@ -212,35 +212,38 @@ def _read_duckdb_init_sql() -> tuple[str, Path | None]:
         if Path(c.DUCKDB_INIT_FILE).exists():
             with open(c.DUCKDB_INIT_FILE, 'r') as f:
                 init_contents.append(f.read())
-        
-        init_sql = "\n".join(init_contents).strip()
-        target_init_path = None
-        if init_sql:
-            target_init_path = Path(c.TARGET_FOLDER, c.DUCKDB_INIT_FILE)
-            target_init_path.parent.mkdir(parents=True, exist_ok=True)
-            target_init_path.write_text(init_sql)
-        
-        return init_sql, target_init_path
+
+        if datalake_db_path:
+            attach_stmt = f"ATTACH '{datalake_db_path}' AS vdl (READ_ONLY);"
+            init_contents.append(attach_stmt)
+            use_stmt = f"USE vdl;"
+            init_contents.append(use_stmt)
+                
+        init_sql = "\n\n".join(init_contents).strip()
+        return init_sql
     except Exception as e:
         raise ConfigurationError(f"Failed to read {c.DUCKDB_INIT_FILE}: {str(e)}") from e
 
-def create_duckdb_connection(filepath: str | Path = ":memory:", *, read_only: bool = False) -> duckdb.DuckDBPyConnection:
+def create_duckdb_connection(
+    db_path: str | Path = ":memory:", 
+    *, 
+    datalake_db_path: str | None = None
+) -> duckdb.DuckDBPyConnection:
     """
     Creates a DuckDB connection and initializes it with statements from duckdb init file
 
     Arguments:
         filepath: Path to the DuckDB database file. Defaults to in-memory database.
-        read_only: Whether to open the database in read-only mode. Defaults to False.
+        datalake_db_path: The path to the VDL catalog database if applicable. If exists, this is attached as 'vdl' (READ_ONLY). Default is None.
     
     Returns:
         A DuckDB connection (which must be closed after use)
     """
-    conn = duckdb.connect(filepath, read_only=read_only)
+    conn = duckdb.connect(db_path)
     
     try:
-        init_sql, _ = _read_duckdb_init_sql()
-        if init_sql:
-            conn.execute(init_sql)
+        init_sql = _read_duckdb_init_sql(datalake_db_path=datalake_db_path)
+        conn.execute(init_sql)
     except Exception as e:
         conn.close()
         raise ConfigurationError(f"Failed to execute {c.DUCKDB_INIT_FILE}: {str(e)}") from e
@@ -284,13 +287,20 @@ def load_yaml_config(filepath: FilePath) -> dict:
     """
     try:
         with open(filepath, 'r') as f:
-            return yaml.safe_load(f)
+            content = yaml.safe_load(f)
+            content = content if content else {}
+        
+        if not isinstance(content, dict):
+            raise yaml.YAMLError(f"Parsed content from YAML file must be a dictionary. Got: {content}")
+        
+        return content
     except yaml.YAMLError as e:
         raise ConfigurationError(f"Failed to parse yaml file: {filepath}") from e
 
 
 def run_duckdb_stmt(
-    logger: Logger, duckdb_conn: duckdb.DuckDBPyConnection, stmt: str, *, params: dict[str, Any] | None = None, redacted_values: list[str] = []
+    logger: Logger, duckdb_conn: duckdb.DuckDBPyConnection, stmt: str, *, params: dict[str, Any] | None = None, 
+    model_name: str | None = None, redacted_values: list[str] = []
 ) -> duckdb.DuckDBPyConnection:
     """
     Runs a statement on a DuckDB connection
@@ -306,7 +316,8 @@ def run_duckdb_stmt(
     for value in redacted_values:
         redacted_stmt = redacted_stmt.replace(value, "[REDACTED]")
     
-    logger.info(f"Running statement: {redacted_stmt}", extra={"data": {"params": params}})
+    for_model_name = f" for model '{model_name}'" if model_name is not None else ""
+    logger.debug(f"Running SQL statement{for_model_name}:\n{redacted_stmt}")
     try:
         return duckdb_conn.execute(stmt, params)
     except duckdb.ParserException as e:
@@ -357,3 +368,74 @@ async def asyncio_gather(coroutines: list):
         # Wait for tasks to be cancelled
         await asyncio.gather(*tasks, return_exceptions=True)
         raise
+
+
+def hash_string(input_str: str, salt: str) -> str:
+    """
+    Hashes a string using SHA-256
+    """
+    return hashlib.sha256((input_str + salt).encode()).hexdigest()
+
+
+T = TypeVar('T')
+def call_func(func: Callable[..., T], **kwargs) -> T:
+    """
+    Calls a function with the given arguments if func expects arguments, otherwise calls func without arguments
+    """
+    sig = inspect.signature(func)
+    # Filter kwargs to only include parameters that the function accepts
+    filtered_kwargs = {k: v for k, v in kwargs.items() if k in sig.parameters}
+    return func(**filtered_kwargs)
+
+
+def generate_pkce_challenge(code_verifier: str) -> str:
+    """Generate PKCE code challenge from code verifier"""
+    # Generate SHA256 hash of code_verifier
+    verifier_hash = hashlib.sha256(code_verifier.encode('utf-8')).digest()
+    # Base64 URL encode (without padding)
+    expected_challenge = base64.urlsafe_b64encode(verifier_hash).decode('utf-8').rstrip('=')
+    return expected_challenge
+
+def validate_pkce_challenge(code_verifier: str, code_challenge: str) -> bool:
+    """Validate PKCE code verifier against code challenge"""
+    # Generate expected challenge
+    expected_challenge = generate_pkce_challenge(code_verifier)
+    return expected_challenge == code_challenge
+
+
+def get_scheme(hostname: str | None) -> str:
+    """Get the scheme of the request"""
+    return "http" if hostname in ("localhost", "127.0.0.1") else "https"
+
+
+def to_title_case(input_str: str) -> str:
+    """Convert a string to title case"""
+    spaced_str = input_str.replace('_', ' ').replace('-', ' ')
+    return spaced_str.title()
+
+
+def to_bool(val: object) -> bool:
+    """Convert common truthy/falsey representations to a boolean.
+
+    Accepted truthy values (case-insensitive): "1", "true", "t", "yes", "y", "on".
+    All other values are considered falsey. None is falsey.
+    """
+    if isinstance(val, bool):
+        return val
+    if val is None:
+        return False
+    s = str(val).strip().lower()
+    return s in ("1", "true", "t", "yes", "y", "on")
+
+
+ACCESS_LEVEL = Literal["admin", "member", "guest"]
+
+def get_access_level_rank(access_level: ACCESS_LEVEL) -> int:
+    """Get the rank of an access level. Lower ranks have more privileges."""
+    return { "admin": 1, "member": 2, "guest": 3 }.get(access_level.lower(), 1)
+
+def user_has_elevated_privileges(user_access_level: ACCESS_LEVEL, required_access_level: ACCESS_LEVEL) -> bool:
+    """Check if a user has privilege to access a resource"""
+    user_access_level_rank = get_access_level_rank(user_access_level)
+    required_access_level_rank = get_access_level_rank(required_access_level)
+    return user_access_level_rank <= required_access_level_rank

squirrels/_version.py

@@ -1,3 +1,3 @@
-__version__ = '0.5.0'
+__version__ = '0.5.1'
 
 sq_major_version, sq_minor_version, sq_patch_version = __version__.split('.')[:3]

squirrels/arguments.py

@@ -0,0 +1,7 @@
+from ._arguments.init_time_args import ConnectionsArgs, AuthProviderArgs, ParametersArgs, BuildModelArgs
+from ._arguments.run_time_args import ContextArgs, ModelArgs, DashboardArgs
+
+__all__ = [
+    "ConnectionsArgs", "AuthProviderArgs", "ParametersArgs", "BuildModelArgs", 
+    "ContextArgs", "ModelArgs", "DashboardArgs"
+]

squirrels/auth.py

@@ -0,0 +1,4 @@
+from ._schemas.auth_models import CustomUserFields, RegisteredUser
+from ._auth import ProviderConfigs, provider
+
+__all__ = ["CustomUserFields", "RegisteredUser", "ProviderConfigs", "provider"]

squirrels/connections.py

@@ -0,0 +1,3 @@
+from ._manifest import ConnectionProperties, ConnectionTypeEnum
+
+__all__ = ["ConnectionProperties", "ConnectionTypeEnum"]

squirrels/dashboards.py

@@ -1,82 +1,3 @@
-import matplotlib.figure as _figure, io as _io, abc as _abc, typing as _t
+from ._dashboards import PngDashboard, HtmlDashboard
 
-from . import _constants as c
-
-
-class Dashboard(metaclass=_abc.ABCMeta):
-    """
-    Abstract parent class for all Dashboard classes.
-    """
-    
-    @property
-    @_abc.abstractmethod
-    def _content(self) -> bytes | str:
-        pass
-    
-    @property
-    @_abc.abstractmethod
-    def _format(self) -> str:
-        pass
-
-
-class PngDashboard(Dashboard):
-    """
-    Instantiate a Dashboard in PNG format from a matplotlib figure or bytes
-    """
-    
-    def __init__(self, content: _figure.Figure | _io.BytesIO | bytes) -> None:
-        """
-        Constructor for PngDashboard
-
-        Arguments:
-            content: The content of the dashboard as a matplotlib.figure.Figure or bytes
-        """
-        if isinstance(content, _figure.Figure):
-            buffer = _io.BytesIO()
-            content.savefig(buffer, format=c.PNG)
-            content = buffer.getvalue()
-        
-        if isinstance(content, _io.BytesIO):
-            content = content.getvalue()
-        
-        self.__content = content
-
-    @property
-    def _content(self) -> bytes:
-        return self.__content
-    
-    @property
-    def _format(self) -> _t.Literal['png']:
-        return c.PNG
-    
-    def _repr_png_(self):
-        return self._content
-    
-
-class HtmlDashboard(Dashboard):
-    """
-    Instantiate a Dashboard from an HTML string
-    """
-
-    def __init__(self, content: _io.StringIO | str) -> None:
-        """
-        Constructor for HtmlDashboard
-
-        Arguments:
-            content: The content of the dashboard as HTML string
-        """
-        if isinstance(content, _io.StringIO):
-            content = content.getvalue()
-        
-        self.__content = content
-
-    @property
-    def _content(self) -> str:
-        return self.__content
-    
-    @property
-    def _format(self) -> _t.Literal['html']:
-        return c.HTML
-    
-    def _repr_html_(self):
-        return self._content
+__all__ = ["PngDashboard", "HtmlDashboard"]