squirrels 0.5.0rc0__py3-none-any.whl → 0.5.1__py3-none-any.whl

squirrels/_py_module.py

@@ -43,11 +43,12 @@ class PyModule:
         return func_or_class
 
 
-def run_pyconfig_main(base_path: str, filename: str, kwargs: dict[str, Any] = {}) -> None:
+def run_pyconfig_main(base_path: str, filename: str, kwargs: dict[str, Any] = {}) -> Any | None:
     """
     Given a python file in the 'pyconfigs' folder, run its main function
     
     Arguments:
+        base_path: The base path of the project
         filename: The name of the file to run main function
         kwargs: Dictionary of the main function arguments
     """
@@ -56,6 +57,6 @@ def run_pyconfig_main(base_path: str, filename: str, kwargs: dict[str, Any] = {}
     main_function = module.get_func_or_class(c.MAIN_FUNC, is_required=False)
     if main_function:
         try:
-            main_function(**kwargs)
+            return main_function(**kwargs)
         except Exception as e:
             raise FileExecutionError(f'Failed to run python file "{filepath}"', e) from e

squirrels/_request_context.py

@@ -0,0 +1,33 @@
+"""
+Request context management using ContextVars for request-scoped data.
+Provides thread-safe and async-safe access to request IDs throughout the request lifecycle.
+"""
+from contextvars import ContextVar
+import uuid
+import base64
+
+# ContextVar for storing the current request ID
+_request_id: ContextVar[str | None] = ContextVar("request_id", default=None)
+
+
+def get_request_id() -> str | None:
+    """
+    Get the current request ID from the context.
+    
+    Returns:
+        The request ID string if available, None otherwise (e.g., in background tasks).
+    """
+    return _request_id.get()
+
+
+def set_request_id() -> str:
+    """
+    Set a new request ID in the context.
+    Uses base64 URL-safe encoding of UUID bytes to create a shorter ID (22 chars vs 36).
+    
+    Returns:
+        The request ID that was set.
+    """
+    request_id = base64.urlsafe_b64encode(uuid.uuid4().bytes).decode().rstrip('=')
+    _request_id.set(request_id)
+    return request_id

squirrels/_schemas/auth_models.py

@@ -0,0 +1,167 @@
+from typing import Callable, Any, Literal
+from datetime import datetime
+from pydantic import BaseModel, ConfigDict, Field, field_serializer
+
+
+class CustomUserFields(BaseModel):
+    """
+    Extend this class to add custom user fields.
+    - Only the following types are supported: [str, int, float, bool, typing.Literal]
+    - Add "| None" after the type to make it nullable. 
+    - Always set a default value for the column (use None if default is null).
+    """
+    pass
+
+
+class AbstractUser(BaseModel):
+    model_config = ConfigDict(from_attributes=True)
+    username: str
+    access_level: Literal["admin", "member", "guest"]
+    custom_fields: CustomUserFields
+    
+    def __hash__(self):
+        return hash(self.username)
+    
+    def __str__(self):
+        return self.username
+
+
+class GuestUser(AbstractUser):
+    access_level: Literal["guest"] = "guest"
+
+
+class RegisteredUser(AbstractUser):
+    access_level: Literal["admin", "member"] = "member"
+
+
+class ApiKey(BaseModel):
+    model_config = ConfigDict(from_attributes=True)
+    id: str
+    title: str
+    username: str
+    created_at: datetime
+    expires_at: datetime
+    
+    @field_serializer('created_at', 'expires_at')
+    def serialize_datetime(self, dt: datetime) -> str:
+        return dt.strftime("%Y-%m-%dT%H:%M:%S.%fZ")
+
+
+class UserField(BaseModel):
+    name: str
+    type: str
+    nullable: bool
+    enum: list[str] | None
+    default: Any | None
+
+
+class ProviderConfigs(BaseModel):
+    client_id: str
+    client_secret: str
+    server_url: str
+    server_metadata_path: str = Field(default="/.well-known/openid-configuration")
+    client_kwargs: dict = Field(default_factory=dict)
+    get_user: Callable[[dict], RegisteredUser]
+
+    @property
+    def server_metadata_url(self) -> str:
+        return f"{self.server_url}{self.server_metadata_path}"
+
+
+class AuthProvider(BaseModel):
+    name: str
+    label: str
+    icon: str
+    provider_configs: ProviderConfigs
+
+
+# OAuth 2.1 Models
+
+class OAuthClientModel(BaseModel):
+    """OAuth client details"""
+    model_config = ConfigDict(from_attributes=True)
+    client_id: str
+    client_name: str
+    redirect_uris: list[str]
+    scope: str
+    grant_types: list[str]
+    response_types: list[str]
+    created_at: datetime
+    is_active: bool
+    
+    @field_serializer('created_at')
+    def serialize_datetime(self, dt: datetime) -> str:
+        return dt.strftime("%Y-%m-%dT%H:%M:%S.%fZ")
+
+
+class ClientRegistrationRequest(BaseModel):
+    """Request model for OAuth client registration"""
+    client_name: str = Field(description="Human-readable name for the OAuth client")
+    redirect_uris: list[str] = Field(description="List of allowed redirect URIs for the client")
+    scope: str = Field(default="read", description="Default scope for the client")
+    grant_types: list[str] = Field(default=["authorization_code", "refresh_token"], description="Allowed grant types")
+    response_types: list[str] = Field(default=["code"], description="Allowed response types")
+
+
+class ClientUpdateRequest(BaseModel):
+    """Request model for OAuth client update"""
+    client_name: str | None = Field(default=None, description="Human-readable name for the OAuth client")
+    redirect_uris: list[str] | None = Field(default=None, description="List of allowed redirect URIs for the client")
+    scope: str | None = Field(default=None, description="Default scope for the client")
+    grant_types: list[str] | None = Field(default=None, description="Allowed grant types")
+    response_types: list[str] | None = Field(default=None, description="Allowed response types")
+    is_active: bool | None = Field(default=None, description="Whether the client is active")
+
+
+class ClientDetailsResponse(BaseModel):
+    """Response model for OAuth client details (without client_secret)"""
+    client_id: str = Field(description="Client ID")
+    client_name: str = Field(description="Client name")
+    redirect_uris: list[str] = Field(description="Registered redirect URIs")
+    scope: str = Field(description="Default scope")
+    grant_types: list[str] = Field(description="Allowed grant types")
+    response_types: list[str] = Field(description="Allowed response types")
+    created_at: datetime = Field(description="Registration timestamp")
+    is_active: bool = Field(description="Whether the client is active")
+    
+    @field_serializer('created_at')
+    def serialize_datetime(self, dt: datetime) -> str:
+        return dt.strftime("%Y-%m-%dT%H:%M:%S.%fZ")
+
+
+class ClientUpdateResponse(ClientDetailsResponse):
+    """Response model for OAuth client update"""
+    registration_access_token: str | None = Field(default=None, description="Token for managing this client registration (store securely)")
+
+
+class ClientRegistrationResponse(ClientUpdateResponse):
+    """Response model for OAuth client registration"""
+    client_secret: str = Field(description="Generated client secret (store securely)")
+    registration_client_uri: str | None = Field(default=None, description="URI for managing this client registration")
+
+
+class TokenResponse(BaseModel):
+    access_token: str
+    token_type: str = "bearer"
+    expires_in: int
+    refresh_token: str | None = None
+
+
+class OAuthServerMetadata(BaseModel):
+    """OAuth 2.1 Authorization Server Metadata (RFC 8414)"""
+    issuer: str = Field(description="Authorization server's issuer identifier URL")
+    authorization_endpoint: str = Field(description="URL of the authorization endpoint")
+    token_endpoint: str = Field(description="URL of the token endpoint")
+    revocation_endpoint: str = Field(description="URL of the token revocation endpoint")
+    registration_endpoint: str = Field(description="URL of the client registration endpoint")
+    scopes_supported: list[str] = Field(description="List of OAuth 2.1 scope values supported")
+    response_types_supported: list[str] = Field(description="List of OAuth 2.1 response_type values supported")
+    grant_types_supported: list[str] = Field(description="List of OAuth 2.1 grant type values supported")
+    token_endpoint_auth_methods_supported: list[str] = Field(
+        default=["client_secret_basic", "client_secret_post"], 
+        description="List of client authentication methods supported by the token endpoint"
+    )
+    code_challenge_methods_supported: list[str] = Field(
+        default=["S256"], 
+        description="List of PKCE code challenge methods supported"
+    )

squirrels/_schemas/query_param_models.py

@@ -0,0 +1,75 @@
+"""
+Query model generation utilities for API routes
+"""
+from typing import Annotated
+from dataclasses import make_dataclass
+from fastapi import Depends
+from pydantic import create_model
+
+from .._parameter_configs import APIParamFieldInfo
+
+
+def _get_query_models_helper(widget_parameters: list[str] | None, predefined_params: list[APIParamFieldInfo], param_fields: dict):
+    """Helper function to generate query models"""
+    if widget_parameters is None:
+        widget_parameters = list(param_fields.keys())
+    
+    QueryModelForGetRaw = make_dataclass("QueryParams", [
+        param_fields[param].as_query_info() for param in widget_parameters
+    ] + [param.as_query_info() for param in predefined_params])
+    QueryModelForGet = Annotated[QueryModelForGetRaw, Depends()]
+
+    field_definitions = {param: param_fields[param].as_body_info() for param in widget_parameters}
+    for param in predefined_params:
+        field_definitions[param.name] = param.as_body_info()
+    QueryModelForPost = create_model("RequestBodyParams", **field_definitions) # type: ignore
+    return QueryModelForGet, QueryModelForPost
+
+
+def get_query_models_for_parameters(widget_parameters: list[str] | None, param_fields: dict):
+    """Generate query models for parameter endpoints"""
+    predefined_params = [
+        APIParamFieldInfo("x_verify_params", bool, default=False, description="If true, the query parameters are verified to be valid for the dataset"),
+        APIParamFieldInfo("x_parent_param", str, description="The parameter name used for parameter updates. If not provided, then all parameters are retrieved"),
+    ]
+    return _get_query_models_helper(widget_parameters, predefined_params, param_fields)
+
+
+def get_query_models_for_dataset(widget_parameters: list[str] | None, param_fields: dict):
+    """Generate query models for dataset endpoints"""
+    predefined_params = [
+        APIParamFieldInfo("x_verify_params", bool, default=False, description="If true, the query parameters are verified to be valid for the dataset"),
+        APIParamFieldInfo("x_orientation", str, default="records", description="The orientation of the data to return, one of: 'records', 'rows', or 'columns'"),
+        APIParamFieldInfo("x_sql_query", str, description="Optional DuckDB SQL to transform the final dataset. Use table name 'result' to reference the dataset."),
+        APIParamFieldInfo("x_offset", int, default=0, description="The number of rows to skip before returning data (applied after data caching)"),
+        APIParamFieldInfo("x_limit", int, default=1000, description="The maximum number of rows to return (applied after data caching and offset)"),
+    ]
+    return _get_query_models_helper(widget_parameters, predefined_params, param_fields)
+
+
+def get_query_models_for_dashboard(widget_parameters: list[str] | None, param_fields: dict):
+    """Generate query models for dashboard endpoints"""
+    predefined_params = [
+        APIParamFieldInfo("x_verify_params", bool, default=False, description="If true, the query parameters are verified to be valid for the dashboard"),
+    ]
+    return _get_query_models_helper(widget_parameters, predefined_params, param_fields)
+
+
+def get_query_models_for_querying_models(param_fields: dict):
+    """Generate query models for querying data models"""
+    predefined_params = [
+        APIParamFieldInfo("x_verify_params", bool, default=False, description="If true, the query parameters are verified to be valid"),
+        APIParamFieldInfo("x_orientation", str, default="records", description="The orientation of the data to return, one of: 'records', 'rows', or 'columns'"),
+        APIParamFieldInfo("x_offset", int, default=0, description="The number of rows to skip before returning data (applied after data caching)"),
+        APIParamFieldInfo("x_limit", int, default=1000, description="The maximum number of rows to return (applied after data caching and offset)"),
+        APIParamFieldInfo("x_sql_query", str, description="The SQL query to execute on the data models"),
+    ]
+    return _get_query_models_helper(None, predefined_params, param_fields) 
+
+
+def get_query_models_for_compiled_models(param_fields: dict):
+    """Generate query models for fetching compiled model SQL"""
+    predefined_params = [
+        APIParamFieldInfo("x_verify_params", bool, default=False, description="If true, the query parameters are verified to be valid for the model"),
+    ]
+    return _get_query_models_helper(None, predefined_params, param_fields)

squirrels/{_api_response_models.py → _schemas/response_models.py}

@@ -1,16 +1,20 @@
-from typing import Annotated, Literal
+from typing import Annotated, Literal, Any
 from pydantic import BaseModel, Field
-from datetime import datetime, date
+from datetime import date
 
-from . import _model_configs as mc, _sources as s
+from .. import _model_configs as mc, _sources as s
 
 
-class LoginReponse(BaseModel):
-    access_token: Annotated[str, Field(examples=["encoded_jwt_token"], description="An encoded JSON web token to use subsequent API requests")]
-    token_type: Annotated[str, Field(examples=["bearer"], description='Always "bearer" for Bearer token')]
-    username: Annotated[str, Field(examples=["johndoe"], description='The username authenticated with from the form data')]
-    is_admin: Annotated[bool, Field(examples=[False], description="A boolean for whether the user is an admin")]
-    expiry_time: Annotated[datetime, Field(examples=["2023-08-01T12:00:00.000000Z"], description="The expiry time of the access token in yyyy-MM-dd'T'hh:mm:ss.SSSSSS'Z' format")]
+## Simple Auth Response Models
+
+class ApiKeyResponse(BaseModel):
+    api_key: Annotated[str, Field(examples=["sqrl-12345678"], description="The API key to use subsequent API requests")]
+
+class ProviderResponse(BaseModel):
+    name: Annotated[str, Field(examples=["my_provider"], description="The name of the provider")]
+    label: Annotated[str, Field(examples=["My Provider"], description="The human-friendly display name for the provider")]
+    icon: Annotated[str, Field(examples=["https://example.com/my_provider_icon.png"], description="The URL of the provider's icon")]
+    login_url: Annotated[str, Field(examples=["https://example.com/my_provider_login"], description="The URL to redirect to for provider login")]
 
 
 ## Parameters Response Models
@@ -95,6 +99,14 @@ parameters_path_description = "The API path to the parameters for the dataset /
 metadata_path_description = "The API path to the metadata (i.e., description and schema) for the dataset"
 result_path_description = "The API path to the results for the dataset / dashboard"
 
+class ConfigurableDefaultModel(BaseModel):
+    name: str
+    default: str
+
+class ConfigurableItemModel(ConfigurableDefaultModel):
+    label: str
+    description: str
+
 class ColumnModel(BaseModel):
     name: Annotated[str, Field(examples=["mycol"], description="Name of column")]
     type: Annotated[str, Field(examples=["string", "integer", "boolean", "datetime"], description='Column type (such as "string", "integer", "boolean", "datetime", etc.)')]
@@ -114,18 +126,19 @@ class DatasetItemModel(BaseModel):
     name: Annotated[str, Field(examples=["mydataset"], description=name_description)]
     label: Annotated[str, Field(examples=["My Dataset"], description=label_description)]
     description: Annotated[str, Field(examples=[""], description=description_description)]
-    parameters: Annotated[list[str], Field(examples=["myparam1", "myparam2"], description="The list of parameter names used by the dataset")]
+    configurables: Annotated[list[ConfigurableDefaultModel], Field(default_factory=list, description="The list of configurables with their default values")]
+    parameters: Annotated[list[str], Field(examples=["myparam1", "myparam2"], description="The list of parameter names used by the dataset. If the list is empty, the dataset does not accept any parameters.")]
     data_schema: Annotated[SchemaWithConditionModel, Field(alias="schema", description="JSON object describing the schema of the dataset")]
-    parameters_path: Annotated[str, Field(examples=["/squirrels-v0/myproject/v1/dataset/mydataset/parameters"], description=parameters_path_description)]
-    result_path: Annotated[str, Field(examples=["/squirrels-v0/myproject/v1/dataset/mydataset"], description=result_path_description)]
-
+    parameters_path: Annotated[str, Field(examples=["/squirrels/v0/myproject/v1/dataset/mydataset/parameters"], description=parameters_path_description)]
+    result_path: Annotated[str, Field(examples=["/squirrels/v0/myproject/v1/dataset/mydataset"], description=result_path_description)]
+    
 class DashboardItemModel(ParametersModel):
     name: Annotated[str, Field(examples=["mydashboard"], description=name_description)]
     label: Annotated[str, Field(examples=["My Dashboard"], description=label_description)]
     description: Annotated[str, Field(examples=[""], description=description_description)]
     parameters: Annotated[list[str], Field(examples=["myparam1", "myparam2"], description="The list of parameter names used by the dashboard")]
-    parameters_path: Annotated[str, Field(examples=["/squirrels-v0/myproject/v1/dashboard/mydashboard/parameters"], description=parameters_path_description)]
-    result_path: Annotated[str, Field(examples=["/squirrels-v0/myproject/v1/dashboard/mydashboard"], description=result_path_description)]
+    parameters_path: Annotated[str, Field(examples=["/squirrels/v0/myproject/v1/dashboard/mydashboard/parameters"], description=parameters_path_description)]
+    result_path: Annotated[str, Field(examples=["/squirrels/v0/myproject/v1/dashboard/mydashboard"], description=result_path_description)]
     result_format: Annotated[str, Field(examples=["png", "html"], description="The format of the dashboard's result API response (one of 'png' or 'html')")]
 
 ModelConfigType = mc.ModelConfig | s.Source | mc.SeedConfig | mc.BuildModelConfig | mc.DbviewModelConfig | mc.FederateModelConfig
@@ -151,13 +164,16 @@ class LineageRelation(BaseModel):
     source: LineageNode
     target: LineageNode
 
-class CatalogModel(BaseModel):
-    parameters: Annotated[ParametersListType, Field(description="The list of all parameters in the project")]
+class CatalogModelForTool(BaseModel):
+    parameters: Annotated[ParametersListType, Field(description="The list of all parameters in the project. It is possible that not all parameters are used by a dataset.")]
     datasets: Annotated[list[DatasetItemModel], Field(description="The list of accessible datasets")]
+    
+class CatalogModel(CatalogModelForTool):
     dashboards: Annotated[list[DashboardItemModel], Field(description="The list of accessible dashboards")]
     connections: Annotated[list[ConnectionItemModel], Field(description="The list of connections in the project (only provided for admin users)")]
     models: Annotated[list[DataModelItem], Field(description="The list of data models in the project (only provided for admin users)")]
     lineage: Annotated[list[LineageRelation], Field(description="The lineage information between data assets (only provided for admin users)")]
+    configurables: Annotated[list[ConfigurableItemModel], Field(description="The list of configurables (only provided for admin users)")]
 
 
 ## Dataset Results Response Models
@@ -176,15 +192,29 @@ class DatasetResultModel(BaseModel):
     )]
 
 
+## Compiled Query Response Model
+
+class CompiledQueryModel(BaseModel):
+    language: Annotated[Literal["sql", "python"], Field(examples=["sql"], description="The language of the data model query: 'sql' or 'python'")]
+    definition: Annotated[str, Field("", description="The compiled SQL or Python definition of the data model.")]
+    placeholders: Annotated[dict[str, Any], Field({}, description="The placeholders for the data model.")]
+
+
 ## Project Metadata Response Models
 
 class ProjectVersionModel(BaseModel):
     major_version: Annotated[int, Field(examples=[1])]
-    data_catalog_path: Annotated[str, Field(examples=["/squirrels-v0/project/myproject/v1/data-catalog"])]
+    data_catalog_path: Annotated[str, Field(examples=["/squirrels/v0/project/myproject/v1/data-catalog"])]
 
 class ProjectModel(BaseModel):
     name: Annotated[str, Field(examples=["myproject"])]
     version: Annotated[str, Field(examples=["v1"])]
     label: Annotated[str, Field(examples=["My Project"])]
     description: Annotated[str, Field(examples=["My project description"])]
+    elevated_access_level: Annotated[Literal["admin", "member", "guest"], Field(
+        examples=["admin"], description="The access level required to access elevated features (such as configurables and data lineage)"
+    )]
+    redoc_path: Annotated[str, Field(examples=["/squirrels/v0/project/myproject/v1/redoc"])]
+    swagger_path: Annotated[str, Field(examples=["/squirrels/v0/project/myproject/v1/docs"])]
+    mcp_server_path: Annotated[str, Field(examples=["/squirrels/v0/project/myproject/v1/mcp"])]
     squirrels_version: Annotated[str, Field(examples=["0.1.0"])]

squirrels/_seeds.py

@@ -37,7 +37,7 @@ class SeedsIO:
     @classmethod
     def load_files(cls, logger: u.Logger, base_path: str, env_vars: dict[str, str]) -> Seeds:
         start = time.time()
-        infer_schema_setting: bool = (env_vars.get(c.SQRL_SEEDS_INFER_SCHEMA, "true").lower() == "true")
+        infer_schema_setting: bool = u.to_bool(env_vars.get(c.SQRL_SEEDS_INFER_SCHEMA, "true"))
         na_values_setting: list[str] = json.loads(env_vars.get(c.SQRL_SEEDS_NA_VALUES, "[]"))
         
         seeds_dict = {}

squirrels/_sources.py

@@ -1,19 +1,19 @@
 from typing import Any
 from pydantic import BaseModel, Field, model_validator
-import time, sqlglot
+import time, sqlglot, yaml
 
 from . import _utils as u, _constants as c, _model_configs as mc
 
 
 class UpdateHints(BaseModel):
     increasing_column: str | None = Field(default=None)
-    strictly_increasing: bool = Field(default=True, description="Delete the max value of the increasing column, ignored if value is set")
-    selective_overwrite_value: Any = Field(default=None)
+    strictly_increasing: bool = Field(default=True, description="Delete the max value of the increasing column, ignored if selective_overwrite_value is set")
+    selective_overwrite_value: Any = Field(default=None, description="Delete all values of the increasing column greater than or equal to this value")
 
 
 class Source(mc.ConnectionInterface, mc.ModelConfig):
     table: str | None = Field(default=None)
-    load_to_duckdb: bool = Field(default=False, description="Whether to load the data to DuckDB")
+    load_to_vdl: bool = Field(default=False, description="Whether to load the data to the 'virtual data lake' (VDL)")
     primary_key: list[str] = Field(default_factory=list)
     update_hints: UpdateHints = Field(default_factory=UpdateHints)
 
@@ -28,34 +28,28 @@ class Source(mc.ConnectionInterface, mc.ModelConfig):
     
     def get_cols_for_create_table_stmt(self) -> str:
         cols_clause = ", ".join([f"{col.name} {col.type}" for col in self.columns])
-        primary_key_clause = f", PRIMARY KEY ({', '.join(self.primary_key)})" if self.primary_key else ""
-        return f"{cols_clause}{primary_key_clause}"
-    
-    def get_cols_for_insert_stmt(self) -> str:
-        return ", ".join([col.name for col in self.columns])
+        return cols_clause
     
     def get_max_incr_col_query(self, source_name: str) -> str:
         return f"SELECT max({self.update_hints.increasing_column}) FROM {source_name}"
     
-    def get_query_for_insert(self, dialect: str, conn_name: str, table_name: str, max_value_of_increasing_col: Any | None, *, full_refresh: bool = True) -> str:
-        select_cols = self.get_cols_for_insert_stmt()
+    def get_query_for_upsert(self, dialect: str, conn_name: str, table_name: str, max_value_of_increasing_col: Any | None, *, full_refresh: bool = True) -> str:
+        select_cols = ", ".join([col.name for col in self.columns])
         if full_refresh or max_value_of_increasing_col is None:
             return f"SELECT {select_cols} FROM db_{conn_name}.{table_name}"
         
         increasing_col = self.update_hints.increasing_column
         increasing_col_type = next(col.type for col in self.columns if col.name == increasing_col)
         where_cond = f"{increasing_col}::{increasing_col_type} > '{max_value_of_increasing_col}'::{increasing_col_type}"
-        pushdown_query = f"SELECT {select_cols} FROM {table_name} WHERE {where_cond}"
         
-        if dialect in ['postgres', 'mysql']:
-            transpiled_query = sqlglot.transpile(pushdown_query, read='duckdb', write=dialect)[0].replace("'", "''")
-            return f"FROM {dialect}_query('db_{conn_name}', '{transpiled_query}')"
+        # TODO: figure out if using pushdown query is worth it
+        # if dialect in ['postgres', 'mysql']:
+        #     pushdown_query = f"SELECT {select_cols} FROM {table_name} WHERE {where_cond}"
+        #     transpiled_query = sqlglot.transpile(pushdown_query, read='duckdb', write=dialect)[0].replace("'", "''")
+        #     return f"FROM {dialect}_query('db_{conn_name}', '{transpiled_query}')"
         
         return f"SELECT {select_cols} FROM db_{conn_name}.{table_name} WHERE {where_cond}"
     
-    def get_insert_replace_clause(self) -> str:
-        return "" if len(self.primary_key) == 0 else "OR REPLACE"
-
 
 class Sources(BaseModel):
     sources: dict[str, Source] = Field(default_factory=dict)
@@ -98,7 +92,17 @@ class SourcesIO:
         start = time.time()
         
         sources_path = u.Path(base_path, c.MODELS_FOLDER, c.SOURCES_FILE)
-        sources_data = u.load_yaml_config(sources_path) if sources_path.exists() else {}
+        if sources_path.exists():
+            raw_content = u.read_file(sources_path)
+            rendered = u.render_string(raw_content, base_path=base_path, env_vars=env_vars)
+            sources_data = yaml.safe_load(rendered) or {}
+        else:
+            sources_data = {}
+        
+        if not isinstance(sources_data, dict):
+            raise u.ConfigurationError(
+                f"Parsed content from YAML file must be a dictionary. Got: {sources_data}"
+            )
         
         sources = Sources(**sources_data).finalize_null_fields(env_vars)