squirrels 0.4.0__py3-none-any.whl → 0.5.0__py3-none-any.whl

squirrels/_py_module.py

@@ -2,6 +2,7 @@ from typing import Type, Optional, Any
 import importlib.util
 
 from . import _constants as c, _utils as u
+from ._exceptions import ConfigurationError, FileExecutionError
 
 
 class PyModule:
@@ -21,7 +22,7 @@ class PyModule:
             spec.loader.exec_module(self.module)
         except FileNotFoundError as e:
             if is_required:
-                raise u.ConfigurationError(f"Required file not found: '{self.filepath}'") from e
+                raise ConfigurationError(f"Required file not found: '{self.filepath}'") from e
             self.module = default_class
     
     def get_func_or_class(self, attr_name: str, *, default_attr: Any = None, is_required: bool = True) -> Any:
@@ -38,15 +39,16 @@ class PyModule:
         """
         func_or_class = getattr(self.module, attr_name, default_attr)
         if func_or_class is None and is_required:
-            raise u.ConfigurationError(f"Module '{self.filepath}' missing required attribute '{attr_name}'")
+            raise ConfigurationError(f"Module '{self.filepath}' missing required attribute '{attr_name}'")
         return func_or_class
 
 
-def run_pyconfig_main(base_path: str, filename: str, kwargs: dict[str, Any] = {}) -> None:
+def run_pyconfig_main(base_path: str, filename: str, kwargs: dict[str, Any] = {}) -> Any | None:
     """
     Given a python file in the 'pyconfigs' folder, run its main function
     
     Arguments:
+        base_path: The base path of the project
         filename: The name of the file to run main function
         kwargs: Dictionary of the main function arguments
     """
@@ -55,6 +57,6 @@ def run_pyconfig_main(base_path: str, filename: str, kwargs: dict[str, Any] = {}
     main_function = module.get_func_or_class(c.MAIN_FUNC, is_required=False)
     if main_function:
         try:
-            main_function(**kwargs)
+            return main_function(**kwargs)
         except Exception as e:
-            raise u.FileExecutionError(f'Failed to run python file "{filepath}"', e) from e
+            raise FileExecutionError(f'Failed to run python file "{filepath}"', e) from e

squirrels/_schemas/auth_models.py

@@ -0,0 +1,167 @@
+from typing import Callable, Any, Literal
+from datetime import datetime
+from pydantic import BaseModel, ConfigDict, Field, field_serializer
+
+
+class CustomUserFields(BaseModel):
+    """
+    Extend this class to add custom user fields.
+    - Only the following types are supported: [str, int, float, bool, typing.Literal]
+    - Add "| None" after the type to make it nullable. 
+    - Always set a default value for the column (use None if default is null).
+    """
+    pass
+
+
+class AbstractUser(BaseModel):
+    model_config = ConfigDict(from_attributes=True)
+    username: str
+    access_level: Literal["admin", "member", "guest"]
+    custom_fields: CustomUserFields
+    
+    def __hash__(self):
+        return hash(self.username)
+    
+    def __str__(self):
+        return self.username
+
+
+class GuestUser(AbstractUser):
+    access_level: Literal["guest"] = "guest"
+
+
+class RegisteredUser(AbstractUser):
+    access_level: Literal["admin", "member"] = "member"
+
+
+class ApiKey(BaseModel):
+    model_config = ConfigDict(from_attributes=True)
+    id: str
+    title: str
+    username: str
+    created_at: datetime
+    expires_at: datetime
+    
+    @field_serializer('created_at', 'expires_at')
+    def serialize_datetime(self, dt: datetime) -> str:
+        return dt.strftime("%Y-%m-%dT%H:%M:%S.%fZ")
+
+
+class UserField(BaseModel):
+    name: str
+    type: str
+    nullable: bool
+    enum: list[str] | None
+    default: Any | None
+
+
+class ProviderConfigs(BaseModel):
+    client_id: str
+    client_secret: str
+    server_url: str
+    server_metadata_path: str = Field(default="/.well-known/openid-configuration")
+    client_kwargs: dict = Field(default_factory=dict)
+    get_user: Callable[[dict], RegisteredUser]
+
+    @property
+    def server_metadata_url(self) -> str:
+        return f"{self.server_url}{self.server_metadata_path}"
+
+
+class AuthProvider(BaseModel):
+    name: str
+    label: str
+    icon: str
+    provider_configs: ProviderConfigs
+
+
+# OAuth 2.1 Models
+
+class OAuthClientModel(BaseModel):
+    """OAuth client details"""
+    model_config = ConfigDict(from_attributes=True)
+    client_id: str
+    client_name: str
+    redirect_uris: list[str]
+    scope: str
+    grant_types: list[str]
+    response_types: list[str]
+    created_at: datetime
+    is_active: bool
+    
+    @field_serializer('created_at')
+    def serialize_datetime(self, dt: datetime) -> str:
+        return dt.strftime("%Y-%m-%dT%H:%M:%S.%fZ")
+
+
+class ClientRegistrationRequest(BaseModel):
+    """Request model for OAuth client registration"""
+    client_name: str = Field(description="Human-readable name for the OAuth client")
+    redirect_uris: list[str] = Field(description="List of allowed redirect URIs for the client")
+    scope: str = Field(default="read", description="Default scope for the client")
+    grant_types: list[str] = Field(default=["authorization_code", "refresh_token"], description="Allowed grant types")
+    response_types: list[str] = Field(default=["code"], description="Allowed response types")
+
+
+class ClientUpdateRequest(BaseModel):
+    """Request model for OAuth client update"""
+    client_name: str | None = Field(default=None, description="Human-readable name for the OAuth client")
+    redirect_uris: list[str] | None = Field(default=None, description="List of allowed redirect URIs for the client")
+    scope: str | None = Field(default=None, description="Default scope for the client")
+    grant_types: list[str] | None = Field(default=None, description="Allowed grant types")
+    response_types: list[str] | None = Field(default=None, description="Allowed response types")
+    is_active: bool | None = Field(default=None, description="Whether the client is active")
+
+
+class ClientDetailsResponse(BaseModel):
+    """Response model for OAuth client details (without client_secret)"""
+    client_id: str = Field(description="Client ID")
+    client_name: str = Field(description="Client name")
+    redirect_uris: list[str] = Field(description="Registered redirect URIs")
+    scope: str = Field(description="Default scope")
+    grant_types: list[str] = Field(description="Allowed grant types")
+    response_types: list[str] = Field(description="Allowed response types")
+    created_at: datetime = Field(description="Registration timestamp")
+    is_active: bool = Field(description="Whether the client is active")
+    
+    @field_serializer('created_at')
+    def serialize_datetime(self, dt: datetime) -> str:
+        return dt.strftime("%Y-%m-%dT%H:%M:%S.%fZ")
+
+
+class ClientUpdateResponse(ClientDetailsResponse):
+    """Response model for OAuth client update"""
+    registration_access_token: str | None = Field(default=None, description="Token for managing this client registration (store securely)")
+
+
+class ClientRegistrationResponse(ClientUpdateResponse):
+    """Response model for OAuth client registration"""
+    client_secret: str = Field(description="Generated client secret (store securely)")
+    registration_client_uri: str | None = Field(default=None, description="URI for managing this client registration")
+
+
+class TokenResponse(BaseModel):
+    access_token: str
+    token_type: str = "bearer"
+    expires_in: int
+    refresh_token: str | None = None
+
+
+class OAuthServerMetadata(BaseModel):
+    """OAuth 2.1 Authorization Server Metadata (RFC 8414)"""
+    issuer: str = Field(description="Authorization server's issuer identifier URL")
+    authorization_endpoint: str = Field(description="URL of the authorization endpoint")
+    token_endpoint: str = Field(description="URL of the token endpoint")
+    revocation_endpoint: str = Field(description="URL of the token revocation endpoint")
+    registration_endpoint: str = Field(description="URL of the client registration endpoint")
+    scopes_supported: list[str] = Field(description="List of OAuth 2.1 scope values supported")
+    response_types_supported: list[str] = Field(description="List of OAuth 2.1 response_type values supported")
+    grant_types_supported: list[str] = Field(description="List of OAuth 2.1 grant type values supported")
+    token_endpoint_auth_methods_supported: list[str] = Field(
+        default=["client_secret_basic", "client_secret_post"], 
+        description="List of client authentication methods supported by the token endpoint"
+    )
+    code_challenge_methods_supported: list[str] = Field(
+        default=["S256"], 
+        description="List of PKCE code challenge methods supported"
+    )

squirrels/_schemas/query_param_models.py

@@ -0,0 +1,75 @@
+"""
+Query model generation utilities for API routes
+"""
+from typing import Annotated
+from dataclasses import make_dataclass
+from fastapi import Depends
+from pydantic import create_model
+
+from .._parameter_configs import APIParamFieldInfo
+
+
+def _get_query_models_helper(widget_parameters: list[str] | None, predefined_params: list[APIParamFieldInfo], param_fields: dict):
+    """Helper function to generate query models"""
+    if widget_parameters is None:
+        widget_parameters = list(param_fields.keys())
+    
+    QueryModelForGetRaw = make_dataclass("QueryParams", [
+        param_fields[param].as_query_info() for param in widget_parameters
+    ] + [param.as_query_info() for param in predefined_params])
+    QueryModelForGet = Annotated[QueryModelForGetRaw, Depends()]
+
+    field_definitions = {param: param_fields[param].as_body_info() for param in widget_parameters}
+    for param in predefined_params:
+        field_definitions[param.name] = param.as_body_info()
+    QueryModelForPost = create_model("RequestBodyParams", **field_definitions) # type: ignore
+    return QueryModelForGet, QueryModelForPost
+
+
+def get_query_models_for_parameters(widget_parameters: list[str] | None, param_fields: dict):
+    """Generate query models for parameter endpoints"""
+    predefined_params = [
+        APIParamFieldInfo("x_verify_params", bool, default=False, description="If true, the query parameters are verified to be valid for the dataset"),
+        APIParamFieldInfo("x_parent_param", str, description="The parameter name used for parameter updates. If not provided, then all parameters are retrieved"),
+    ]
+    return _get_query_models_helper(widget_parameters, predefined_params, param_fields)
+
+
+def get_query_models_for_dataset(widget_parameters: list[str] | None, param_fields: dict):
+    """Generate query models for dataset endpoints"""
+    predefined_params = [
+        APIParamFieldInfo("x_verify_params", bool, default=False, description="If true, the query parameters are verified to be valid for the dataset"),
+        APIParamFieldInfo("x_orientation", str, default="records", description="The orientation of the data to return, one of: 'records', 'rows', or 'columns'"),
+        APIParamFieldInfo("x_sql_query", str, description="Optional DuckDB SQL to transform the final dataset. Use table name 'result' to reference the dataset."),
+        APIParamFieldInfo("x_offset", int, default=0, description="The number of rows to skip before returning data (applied after data caching)"),
+        APIParamFieldInfo("x_limit", int, default=1000, description="The maximum number of rows to return (applied after data caching and offset)"),
+    ]
+    return _get_query_models_helper(widget_parameters, predefined_params, param_fields)
+
+
+def get_query_models_for_dashboard(widget_parameters: list[str] | None, param_fields: dict):
+    """Generate query models for dashboard endpoints"""
+    predefined_params = [
+        APIParamFieldInfo("x_verify_params", bool, default=False, description="If true, the query parameters are verified to be valid for the dashboard"),
+    ]
+    return _get_query_models_helper(widget_parameters, predefined_params, param_fields)
+
+
+def get_query_models_for_querying_models(param_fields: dict):
+    """Generate query models for querying data models"""
+    predefined_params = [
+        APIParamFieldInfo("x_verify_params", bool, default=False, description="If true, the query parameters are verified to be valid"),
+        APIParamFieldInfo("x_orientation", str, default="records", description="The orientation of the data to return, one of: 'records', 'rows', or 'columns'"),
+        APIParamFieldInfo("x_offset", int, default=0, description="The number of rows to skip before returning data (applied after data caching)"),
+        APIParamFieldInfo("x_limit", int, default=1000, description="The maximum number of rows to return (applied after data caching and offset)"),
+        APIParamFieldInfo("x_sql_query", str, description="The SQL query to execute on the data models"),
+    ]
+    return _get_query_models_helper(None, predefined_params, param_fields) 
+
+
+def get_query_models_for_compiled_models(param_fields: dict):
+    """Generate query models for fetching compiled model SQL"""
+    predefined_params = [
+        APIParamFieldInfo("x_verify_params", bool, default=False, description="If true, the query parameters are verified to be valid for the model"),
+    ]
+    return _get_query_models_helper(None, predefined_params, param_fields)

squirrels/{_api_response_models.py → _schemas/response_models.py}

@@ -1,41 +1,20 @@
-from typing import Annotated
+from typing import Annotated, Literal, Any
 from pydantic import BaseModel, Field
-from datetime import datetime, date
+from datetime import date
 
+from .. import _model_configs as mc, _sources as s
 
-class LoginReponse(BaseModel):
-    access_token: Annotated[str, Field(examples=["encoded_jwt_token"], description="An encoded JSON web token to use subsequent API requests")]
-    token_type: Annotated[str, Field(examples=["bearer"], description='Always "bearer" for Bearer token')]
-    username: Annotated[str, Field(examples=["johndoe"], description='The username authenticated with from the form data')]
-    expiry_time: Annotated[datetime, Field(examples=["2023-08-01T12:00:00.000000Z"], description="The expiry time of the access token in yyyy-MM-dd'T'hh:mm:ss.SSSSSS'Z' format")]
 
+## Simple Auth Response Models
 
-## Datasets / Dashboards Catalog Response Models
+class ApiKeyResponse(BaseModel):
+    api_key: Annotated[str, Field(examples=["sqrl-12345678"], description="The API key to use subsequent API requests")]
 
-name_description = "The name of the dataset / dashboard (usually in snake case)"
-label_description = "The human-friendly display name for the dataset / dashboard"
-description_description = "The description of the dataset / dashboard"
-parameters_path_description = "The API path to the parameters for the dataset / dashboard"
-result_path_description = "The API path to the results for the dataset / dashboard"
-
-class DatasetItemModel(BaseModel):
-    name: Annotated[str, Field(examples=["mydataset"], description=name_description)]
-    label: Annotated[str, Field(examples=["My Dataset"], description=label_description)]
-    description: Annotated[str, Field(examples=[""], description=description_description)]
-    parameters_path: Annotated[str, Field(examples=["/squirrels-v0/myproject/v1/dataset/mydataset/parameters"], description=parameters_path_description)]
-    result_path: Annotated[str, Field(examples=["/squirrels-v0/myproject/v1/dataset/mydataset"], description=result_path_description)]
-
-class DashboardItemModel(BaseModel):
-    name: Annotated[str, Field(examples=["mydashboard"], description=name_description)]
-    label: Annotated[str, Field(examples=["My Dashboard"], description=label_description)]
-    description: Annotated[str, Field(examples=[""], description=description_description)]
-    parameters_path: Annotated[str, Field(examples=["/squirrels-v0/myproject/v1/dashboard/mydashboard/parameters"], description=parameters_path_description)]
-    result_path: Annotated[str, Field(examples=["/squirrels-v0/myproject/v1/dashboard/mydashboard"], description=result_path_description)]
-    result_format: Annotated[str, Field(examples=["png", "html"], description="The format of the dashboard's result API response (one of 'png' or 'html')")]
-
-class CatalogModel(BaseModel):
-    datasets: Annotated[list[DatasetItemModel], Field(description="The list of accessible datasets")]
-    dashboards: Annotated[list[DashboardItemModel], Field(description="The list of accessible dashboards")]
+class ProviderResponse(BaseModel):
+    name: Annotated[str, Field(examples=["my_provider"], description="The name of the provider")]
+    label: Annotated[str, Field(examples=["My Provider"], description="The human-friendly display name for the provider")]
+    icon: Annotated[str, Field(examples=["https://example.com/my_provider_icon.png"], description="The URL of the provider's icon")]
+    login_url: Annotated[str, Field(examples=["https://example.com/my_provider_login"], description="The URL to redirect to for provider login")]
 
 
 ## Parameters Response Models
@@ -45,7 +24,7 @@ class ParameterOptionModel(BaseModel):
     label: Annotated[str, Field(examples=["My Option"], description="The human-friendly display name for the option")]
 
 class ParameterModelBase(BaseModel):
-    widget_type: Annotated[str, Field(examples=["none"], description="The parameter type (set to 'none' for this model)")]
+    widget_type: Annotated[str, Field(examples=["disabled"], description="The parameter type")]
     name: Annotated[str, Field(examples=["my_unique_param_name"], description="The name of the parameter. Use this as the key when providing the API request parameters")]
     label: Annotated[str, Field(examples=["My Parameter"], description="The human-friendly display name for the parameter")]
     description: Annotated[str, Field(examples=[""], description="The description of the parameter")]
@@ -102,40 +81,140 @@ class TextParameterModel(ParameterModelBase):
         description='A string for the input type (one of "text", "textarea", "number", "date", "datetime-local", "month", "time", "color", or "password")'
     )]
 
+ParametersListType = list[
+    NoneParameterModel | SingleSelectParameterModel | MultiSelectParameterModel | DateParameterModel | DateRangeParameterModel | 
+    NumberParameterModel | NumberRangeParameterModel | TextParameterModel
+]
+
 class ParametersModel(BaseModel):
-    parameters: list[
-        NoneParameterModel | SingleSelectParameterModel | MultiSelectParameterModel | DateParameterModel | DateRangeParameterModel | 
-        NumberParameterModel | NumberRangeParameterModel | TextParameterModel
-    ]
+    parameters: Annotated[ParametersListType, Field(description="The list of parameters for the dataset / dashboard")]
 
 
-## Dataset Results Response Models
+## Datasets / Dashboards Catalog Response Models
+
+name_description = "The name of the dataset / dashboard (usually in snake case)"
+label_description = "The human-friendly display name for the dataset / dashboard"
+description_description = "The description of the dataset / dashboard"
+parameters_path_description = "The API path to the parameters for the dataset / dashboard"
+metadata_path_description = "The API path to the metadata (i.e., description and schema) for the dataset"
+result_path_description = "The API path to the results for the dataset / dashboard"
+
+class ConfigurableDefaultModel(BaseModel):
+    name: str
+    default: str
+
+class ConfigurableItemModel(ConfigurableDefaultModel):
+    label: str
+    description: str
 
 class ColumnModel(BaseModel):
     name: Annotated[str, Field(examples=["mycol"], description="Name of column")]
-    type: Annotated[str, Field(examples=["string", "number", "integer", "boolean", "datetime"], description='Column type. One of "string", "number", "integer", "boolean", and "datetime"')]
+    type: Annotated[str, Field(examples=["string", "integer", "boolean", "datetime"], description='Column type (such as "string", "integer", "boolean", "datetime", etc.)')]
+    description: Annotated[str, Field(examples=["My column description"], description="The description of the column")]
+    category: Annotated[str, Field(examples=["dimension", "measure", "misc"], description="The category of the column (such as 'dimension', 'measure', or 'misc')")]
+
+class ColumnWithConditionModel(ColumnModel):
+    condition: Annotated[str | None, Field(None, examples=["My condition"], description="The condition of when the column is included (such as based on a parameter selection)")]
 
 class SchemaModel(BaseModel):
     fields: Annotated[list[ColumnModel], Field(description="A list of JSON objects containing the 'name' and 'type' for each of the columns in the result")]
-    dimensions: Annotated[list[str], Field(examples=[["mycol"]], description="A list of column names that are dimensions")]
+
+class SchemaWithConditionModel(BaseModel):
+    fields: Annotated[list[ColumnWithConditionModel], Field(description="A list of JSON objects containing the 'name' and 'type' for each of the columns in the result")]
+
+class DatasetItemModel(BaseModel):
+    name: Annotated[str, Field(examples=["mydataset"], description=name_description)]
+    label: Annotated[str, Field(examples=["My Dataset"], description=label_description)]
+    description: Annotated[str, Field(examples=[""], description=description_description)]
+    configurables: Annotated[list[ConfigurableDefaultModel], Field(default_factory=list, description="The list of configurables with their default values")]
+    parameters: Annotated[list[str], Field(examples=["myparam1", "myparam2"], description="The list of parameter names used by the dataset. If the list is empty, the dataset does not accept any parameters.")]
+    data_schema: Annotated[SchemaWithConditionModel, Field(alias="schema", description="JSON object describing the schema of the dataset")]
+    parameters_path: Annotated[str, Field(examples=["/squirrels/v0/myproject/v1/dataset/mydataset/parameters"], description=parameters_path_description)]
+    result_path: Annotated[str, Field(examples=["/squirrels/v0/myproject/v1/dataset/mydataset"], description=result_path_description)]
+    
+class DashboardItemModel(ParametersModel):
+    name: Annotated[str, Field(examples=["mydashboard"], description=name_description)]
+    label: Annotated[str, Field(examples=["My Dashboard"], description=label_description)]
+    description: Annotated[str, Field(examples=[""], description=description_description)]
+    parameters: Annotated[list[str], Field(examples=["myparam1", "myparam2"], description="The list of parameter names used by the dashboard")]
+    parameters_path: Annotated[str, Field(examples=["/squirrels/v0/myproject/v1/dashboard/mydashboard/parameters"], description=parameters_path_description)]
+    result_path: Annotated[str, Field(examples=["/squirrels/v0/myproject/v1/dashboard/mydashboard"], description=result_path_description)]
+    result_format: Annotated[str, Field(examples=["png", "html"], description="The format of the dashboard's result API response (one of 'png' or 'html')")]
+
+ModelConfigType = mc.ModelConfig | s.Source | mc.SeedConfig | mc.BuildModelConfig | mc.DbviewModelConfig | mc.FederateModelConfig
+
+class ConnectionItemModel(BaseModel):
+    name: Annotated[str, Field(examples=["myconnection"], description="The name of the connection")]
+    label: Annotated[str, Field(examples=["My Connection"], description="The human-friendly display name for the connection")]
+
+class DataModelItem(BaseModel):
+    name: Annotated[str, Field(examples=["model_name"], description="The name of the model")]
+    model_type: Annotated[Literal["source", "dbview", "federate", "seed", "build"], Field(
+        examples=["source", "dbview", "federate", "seed", "build"], description="The type of the model"
+    )]
+    config: Annotated[ModelConfigType, Field(description="The configuration of the model")]
+    is_queryable: Annotated[bool, Field(examples=[True], description="Whether the model is queryable")]
+
+class LineageNode(BaseModel):
+    name: str
+    type: Literal["model", "dataset", "dashboard"]
+
+class LineageRelation(BaseModel):
+    type: Literal["buildtime", "runtime"]
+    source: LineageNode
+    target: LineageNode
+
+class CatalogModelForTool(BaseModel):
+    parameters: Annotated[ParametersListType, Field(description="The list of all parameters in the project. It is possible that not all parameters are used by a dataset.")]
+    datasets: Annotated[list[DatasetItemModel], Field(description="The list of accessible datasets")]
+    
+class CatalogModel(CatalogModelForTool):
+    dashboards: Annotated[list[DashboardItemModel], Field(description="The list of accessible dashboards")]
+    connections: Annotated[list[ConnectionItemModel], Field(description="The list of connections in the project (only provided for admin users)")]
+    models: Annotated[list[DataModelItem], Field(description="The list of data models in the project (only provided for admin users)")]
+    lineage: Annotated[list[LineageRelation], Field(description="The lineage information between data assets (only provided for admin users)")]
+    configurables: Annotated[list[ConfigurableItemModel], Field(description="The list of configurables (only provided for admin users)")]
+
+
+## Dataset Results Response Models
+
+class DataDetailsModel(BaseModel):
+    num_rows: Annotated[int, Field(examples=[2], description="The number of rows in the data field")]
+    orientation: Annotated[Literal["records", "rows", "columns"], Field(examples=["records", "rows", "columns"], description="The orientation of the data field")]
 
 class DatasetResultModel(BaseModel):
     data_schema: Annotated[SchemaModel, Field(alias="schema", description="JSON object describing the schema of the dataset")]
-    data: Annotated[list[dict], Field(
-        examples=[[{"mycol": "myval"}]],
+    total_num_rows: Annotated[int, Field(examples=[2], description="The total number of rows for the dataset")]
+    data_details: Annotated[DataDetailsModel, Field(description="A JSON object containing the details of the data field")]
+    data: Annotated[list[dict] | list[list] | dict[str, list], Field(
+        examples=[[{"mycol": "col_value1"}, {"mycol": "col_value2"}], [["col_value1"], ["col_value2"]], {"mycol": ["col_value1", "col_value2"]}],
         description="A list of JSON objects where each object is a row of the tabular results. The keys and values of the object are column names (described in fields) and values of the row."
     )]
 
 
+## Compiled Query Response Model
+
+class CompiledQueryModel(BaseModel):
+    language: Annotated[Literal["sql", "python"], Field(examples=["sql"], description="The language of the data model query: 'sql' or 'python'")]
+    definition: Annotated[str, Field("", description="The compiled SQL or Python definition of the data model.")]
+    placeholders: Annotated[dict[str, Any], Field({}, description="The placeholders for the data model.")]
+
+
 ## Project Metadata Response Models
 
 class ProjectVersionModel(BaseModel):
-    major_version: int
-    minor_versions: list[int]
-    token_path: Annotated[str, Field(examples=["/squirrels-v0/myproject/v1/token"])]
-    data_catalog_path: Annotated[str, Field(examples=["/squirrels-v0/myproject/v1/datasets"])]
+    major_version: Annotated[int, Field(examples=[1])]
+    data_catalog_path: Annotated[str, Field(examples=["/squirrels/v0/project/myproject/v1/data-catalog"])]
 
 class ProjectModel(BaseModel):
     name: Annotated[str, Field(examples=["myproject"])]
+    version: Annotated[str, Field(examples=["v1"])]
     label: Annotated[str, Field(examples=["My Project"])]
-    versions: list[ProjectVersionModel]
+    description: Annotated[str, Field(examples=["My project description"])]
+    elevated_access_level: Annotated[Literal["admin", "member", "guest"], Field(
+        examples=["admin"], description="The access level required to access elevated features (such as configurables and data lineage)"
+    )]
+    redoc_path: Annotated[str, Field(examples=["/squirrels/v0/project/myproject/v1/redoc"])]
+    swagger_path: Annotated[str, Field(examples=["/squirrels/v0/project/myproject/v1/docs"])]
+    mcp_server_path: Annotated[str, Field(examples=["/squirrels/v0/project/myproject/v1/mcp"])]
+    squirrels_version: Annotated[str, Field(examples=["0.1.0"])]

squirrels/_seeds.py

@@ -1,39 +1,58 @@
 from dataclasses import dataclass
-import os, time, glob, pandas as pd
+import os, time, glob, polars as pl, json
 
-from ._manifest import ManifestConfig
-from . import _utils as _u, _constants as c
+from . import _utils as u, _constants as c, _model_configs as mc
+
+
+@dataclass
+class Seed:
+    config: mc.SeedConfig
+    df: pl.LazyFrame
+
+    def __post_init__(self):
+        if self.config.cast_column_types:
+            exprs = []
+            for col_config in self.config.columns:
+                sqrl_dtype = "double" if col_config.type.lower().startswith("decimal") else col_config.type
+                polars_dtype = u.sqrl_dtypes_to_polars_dtypes.get(sqrl_dtype, pl.String)
+                exprs.append(pl.col(col_config.name).cast(polars_dtype))
+
+            self.df = self.df.with_columns(*exprs)
 
 
 @dataclass
 class Seeds:
-    _data: dict[str, pd.DataFrame]
-    _manifest_cfg: ManifestConfig
+    _data: dict[str, Seed]
     
-    def run_query(self, sql_query: str) -> pd.DataFrame:
-        use_duckdb = self._manifest_cfg.settings_obj.do_use_duckdb()
-        return _u.run_sql_on_dataframes(sql_query, self._data, use_duckdb)
+    def run_query(self, sql_query: str) -> pl.DataFrame:
+        dataframes = {key: seed.df for key, seed in self._data.items()}
+        return u.run_sql_on_dataframes(sql_query, dataframes)
     
-    def get_dataframes(self) -> dict[str, pd.DataFrame]:
+    def get_dataframes(self) -> dict[str, Seed]:
         return self._data.copy()
 
 
 class SeedsIO:
 
     @classmethod
-    def load_files(cls, logger: _u.Logger, base_path: str, manifest_cfg: ManifestConfig) -> Seeds:
+    def load_files(cls, logger: u.Logger, base_path: str, env_vars: dict[str, str]) -> Seeds:
         start = time.time()
-        infer_schema: bool = manifest_cfg.settings.get(c.SEEDS_INFER_SCHEMA_SETTING, True)
-        na_values: list[str] = manifest_cfg.settings.get(c.SEEDS_NA_VALUES_SETTING, ["NA"])
-        csv_dtype = None if infer_schema else str
+        infer_schema_setting: bool = u.to_bool(env_vars.get(c.SQRL_SEEDS_INFER_SCHEMA, "true"))
+        na_values_setting: list[str] = json.loads(env_vars.get(c.SQRL_SEEDS_NA_VALUES, "[]"))
         
         seeds_dict = {}
         csv_files = glob.glob(os.path.join(base_path, c.SEEDS_FOLDER, '**/*.csv'), recursive=True)
         for csv_file in csv_files:
+            config_file = os.path.splitext(csv_file)[0] + '.yml'
+            config_dict = u.load_yaml_config(config_file) if os.path.exists(config_file) else {}
+            config = mc.SeedConfig(**config_dict)
+
             file_stem = os.path.splitext(os.path.basename(csv_file))[0]
-            df = pd.read_csv(csv_file, dtype=csv_dtype, keep_default_na=False, na_values=na_values)
-            seeds_dict[file_stem] = df
+            infer_schema = not config.cast_column_types and infer_schema_setting
+            df = pl.read_csv(csv_file, try_parse_dates=True, infer_schema=infer_schema, null_values=na_values_setting).lazy()
+            
+            seeds_dict[file_stem] = Seed(config, df)
         
-        seeds = Seeds(seeds_dict, manifest_cfg)
+        seeds = Seeds(seeds_dict)
         logger.log_activity_time("loading seed files", start)
         return seeds