squirrels 0.2.1__py3-none-any.whl → 0.3.0__py3-none-any.whl

squirrels/__init__.py

@@ -1,8 +1,15 @@
-__version__ = '0.2.1'
+__version__ = '0.3.0'
 
 from .arguments.init_time_args import ConnectionsArgs, ParametersArgs
 from .arguments.run_time_args import AuthArgs, ContextArgs, ModelDepsArgs, ModelArgs
-from .parameter_options import SelectParameterOption, DateParameterOption, DateRangeParameterOption, NumberParameterOption, NumberRangeParameterOption
-from .parameters import SingleSelectParameter, MultiSelectParameter, DateParameter, DateRangeParameter, NumberParameter, NumberRangeParameter
-from .data_sources import SingleSelectDataSource, MultiSelectDataSource, DateDataSource, DateRangeDataSource, NumberDataSource, NumberRangeDataSource
+
+from .parameter_options import SelectParameterOption, DateParameterOption, DateRangeParameterOption
+from .parameter_options import NumberParameterOption, NumberRangeParameterOption, TextParameterOption
+
+from .parameters import SingleSelectParameter, MultiSelectParameter, DateParameter, DateRangeParameter
+from .parameters import NumberParameter, NumberRangeParameter, TextParameter
+
+from .data_sources import SingleSelectDataSource, MultiSelectDataSource, SelectDataSource, DateDataSource, DateRangeDataSource
+from .data_sources import NumberDataSource, NumberRangeDataSource, TextDataSource
+
 from .user_base import User, WrongPassword

squirrels/_api_response_models.py

@@ -0,0 +1,118 @@
+from typing import Annotated, Union, Optional
+from pydantic import BaseModel, Field
+from datetime import datetime, date
+
+
+class LoginReponse(BaseModel):
+    access_token: Annotated[str, Field(examples=["encoded_jwt_token"])]
+    token_type: Annotated[str, Field(examples=["bearer"])]
+    username: Annotated[str, Field(examples=["johndoe"])]
+    expiry_time: datetime
+
+
+## Parameters Response Models
+
+class ParameterOptionModel(BaseModel):
+    id: str
+    label: str
+
+class ParameterModelBase(BaseModel):
+    widget_type: str
+    name: str
+    label: str
+    description: str
+
+class SelectParameterModel(ParameterModelBase):
+    options: list[ParameterOptionModel]
+    trigger_refresh: bool
+
+class SingleSelectParameterModel(SelectParameterModel):
+    widget_type: Annotated[str, Field(examples=["single_select"])]
+    selected_id: Optional[str]
+
+class MultiSelectParameterModel(SelectParameterModel):
+    widget_type: Annotated[str, Field(examples=["multi_select"])]
+    show_select_all: bool
+    is_dropdown: bool
+    order_matters: bool
+    selected_ids: list[str]
+
+class DateParameterModel(ParameterModelBase):
+    widget_type: Annotated[str, Field(examples=["date"])]
+    selected_date: date
+
+class DateRangeParameterModel(ParameterModelBase):
+    widget_type: Annotated[str, Field(examples=["date_range"])]
+    selected_start_date: date
+    selected_end_date: date
+
+class NumericParameterModel(ParameterModelBase):
+    min_value: Annotated[float, Field(examples=[0])]
+    max_value: Annotated[float, Field(examples=[10])]
+    increment: Annotated[float, Field(examples=[1])]
+
+class NumberParameterModel(NumericParameterModel):
+    widget_type: Annotated[str, Field(examples=["number"])]
+    selected_value: Annotated[float, Field(examples=[2])]
+
+class NumberRangeParameterModel(NumericParameterModel):
+    widget_type: Annotated[str, Field(examples=["number_range"])]
+    selected_lower_value: Annotated[float, Field(examples=[2])]
+    selected_upper_value: Annotated[float, Field(examples=[8])]
+
+class TextParameterModel(ParameterModelBase):
+    widget_type: Annotated[str, Field(examples=["text"])]
+    entered_text: str
+    is_textarea: bool
+
+class ParametersModel(BaseModel):
+    parameters: list[
+        Union[
+            ParameterModelBase, SingleSelectParameterModel, MultiSelectParameterModel, DateParameterModel, DateRangeParameterModel,
+            NumberParameterModel, NumberRangeParameterModel, TextParameterModel
+        ]
+    ]
+
+
+## Dataset Results Response Models
+
+class ColumnModel(BaseModel):
+    name: Annotated[str, Field(examples=["mycol"])]
+    type: str
+
+class SchemaModel(BaseModel):
+    fields: list[ColumnModel]
+    dimensions: Annotated[list[str], Field(examples=[["mycol"]])]
+
+class DatasetResultModel(BaseModel):
+    data_schema: Annotated[SchemaModel, Field(alias='schema')]
+    data: Annotated[list[dict], Field(examples=[[{"mycol": "myval"}]])]
+
+
+## Catalog Response Models
+
+class ProjectVersionModel(BaseModel):
+    major_version: int
+    minor_versions: list[int]
+    token_path: Annotated[str, Field(examples=["/squirrels-v0/myproject/v1/token"])]
+    datasets_path: Annotated[str, Field(examples=["/squirrels-v0/myproject/v1/datasets"])]
+
+class ProjectModel(BaseModel):
+    name: Annotated[str, Field(examples=["myproject"])]
+    label: Annotated[str, Field(examples=["My Project"])]
+    versions: list[ProjectVersionModel]
+
+class CatalogModel(BaseModel):
+    projects: list[ProjectModel]
+
+
+## Datasets Catalog Response Models
+
+class DatasetInfoModel(BaseModel):
+    name: Annotated[str, Field(examples=["mydataset"])]
+    label: Annotated[str, Field(examples=["My Dataset"])]
+    parameters_path: Annotated[str, Field(examples=["/squirrels-v0/myproject/v1/dataset/mydataset/parameters"])]
+    result_path: Annotated[str, Field(examples=["/squirrels-v0/myproject/v1/dataset/mydataset"])]
+
+class DatasetsCatalogModel(BaseModel):
+    datasets: list[DatasetInfoModel]

squirrels/_api_server.py

@@ -1,16 +1,20 @@
-from typing import List, Iterable, Optional, Mapping, Callable, Coroutine, TypeVar, Any
+from typing import Iterable, Optional, Mapping, Callable, Coroutine, TypeVar, Annotated, Any
+from dataclasses import make_dataclass, asdict
 from fastapi import Depends, FastAPI, Request, HTTPException, Response, status
 from fastapi.responses import HTMLResponse, JSONResponse
 from fastapi.templating import Jinja2Templates
 from fastapi.staticfiles import StaticFiles
 from fastapi.security import OAuth2PasswordBearer, OAuth2PasswordRequestForm
 from fastapi.middleware.cors import CORSMiddleware
+from pydantic import create_model, BaseModel
 from cachetools import TTLCache
-import os, mimetypes, traceback, pandas as pd
+from pandas.api import types as pd_types
+import os, mimetypes, traceback, json, pandas as pd
 
-from . import _constants as c, _utils as u
+from . import _constants as c, _utils as u, _api_response_models as arm
 from ._version import sq_major_version
 from ._manifest import ManifestIO
+from ._parameter_sets import ParameterConfigsSetIO
 from ._authenticator import User, Authenticator
 from ._timer import timer, time
 from ._parameter_sets import ParameterSet
@@ -19,17 +23,42 @@ from ._models import ModelsIO
 mimetypes.add_type('application/javascript', '.js')
 
 
+def df_to_api_response0(df: pd.DataFrame, dimensions: list[str] = None) -> arm.DatasetResultModel:
+    """
+    Convert a pandas DataFrame to the response format that the dataset result API of Squirrels outputs.
+
+    Parameters:
+        df: The dataframe to convert into an API response
+        dimensions: The list of declared dimensions. If None, all non-numeric columns are assumed as dimensions
+
+    Returns:
+        The response of a Squirrels dataset result API
+    """
+    in_df_json = json.loads(df.to_json(orient='table', index=False))
+    out_fields = []
+    non_numeric_fields = []
+    for in_column in in_df_json["schema"]["fields"]:
+        col_name: str = in_column["name"]
+        out_column = arm.ColumnModel(name=col_name, type=in_column["type"])
+        out_fields.append(out_column)
+        
+        if not pd_types.is_numeric_dtype(df[col_name].dtype):
+            non_numeric_fields.append(col_name)
+    
+    out_dimensions = non_numeric_fields if dimensions is None else dimensions
+    out_schema = arm.SchemaModel(fields=out_fields, dimensions=out_dimensions)
+    return arm.DatasetResultModel(schema=out_schema, data=in_df_json["data"])
+
+
 class ApiServer:
-    def __init__(self, no_cache: bool, debug: bool) -> None:
+    def __init__(self, no_cache: bool) -> None:
         """
         Constructor for ApiServer
 
         Parameters:
             no_cache (bool): Whether to disable caching
-            debug (bool): Set to True to show "hidden" parameters in the /parameters endpoint response
         """
         self.no_cache = no_cache
-        self.debug = debug
         self.dataset_configs = ManifestIO.obj.datasets
         
         token_expiry_minutes = ManifestIO.obj.settings.get(c.AUTH_TOKEN_EXPIRE_SETTING, 30)
@@ -126,8 +155,13 @@ class ApiServer:
             # Changing selections into a cachable "frozenset" that will later be converted to dictionary
             selections = set()
             for key, val in params.items():
-                if isinstance(val, List):
-                    val = tuple(val)
+                if val is None:
+                    continue
+                if isinstance(val, (list, tuple)):
+                    if len(val) == 1: # for backward compatibility
+                        val = val[0]
+                    else:
+                        val = tuple(val)
                 selections.add((u.normalize_name(key), val))
             selections = frozenset(selections)
             
@@ -141,25 +175,24 @@ class ApiServer:
                 cache[cache_key] = result
             return result
         
+        def get_dataset_from_request_path(request: Request, section: int) -> str:
+            url_path: str = request.scope['route'].path
+            return url_path.split('/')[section]
+        
         # Login
         token_path = base_path + '/token'
 
         oauth2_scheme = OAuth2PasswordBearer(tokenUrl=token_path, auto_error=False)
 
         @app.post(token_path)
-        async def login_for_access_token(form_data: OAuth2PasswordRequestForm = Depends()):
+        async def login_for_access_token(form_data: OAuth2PasswordRequestForm = Depends()) -> arm.LoginReponse:
             user: Optional[User] = self.authenticator.authenticate_user(form_data.username, form_data.password)
             if not user:
                 raise HTTPException(status_code=status.HTTP_401_UNAUTHORIZED, 
                                     detail="Incorrect username or password",
                                     headers={"WWW-Authenticate": "Bearer"})
             access_token, expiry = self.authenticator.create_access_token(user)
-            return {
-                "access_token": access_token, 
-                "token_type": "bearer", 
-                "username": user.username,
-                "expiry_time": expiry
-            }
+            return arm.LoginReponse(access_token=access_token, token_type="bearer", username=user.username, expiry_time=expiry)
         
         async def get_current_user(response: Response, token: str = Depends(oauth2_scheme)) -> Optional[User]:
             user = self.authenticator.get_user_from_token(token)
@@ -167,11 +200,14 @@ class ApiServer:
             response.headers["Applied-Username"] = username
             return user
 
-        # Parameters API
+        # Parameters API Helpers
         parameters_path = base_path + '/dataset/{dataset}/parameters'
         
+        def get_dataset_for_parameters_request(request: Request) -> str:
+            return get_dataset_from_request_path(request, -2)
+        
         parameters_cache_size = ManifestIO.obj.settings.get(c.PARAMETERS_CACHE_SIZE_SETTING, 1024)
-        parameters_cache_ttl = ManifestIO.obj.settings.get(c.PARAMETERS_CACHE_TTL_SETTING, 0)
+        parameters_cache_ttl = ManifestIO.obj.settings.get(c.PARAMETERS_CACHE_TTL_SETTING, 60)
     
         async def get_parameters_helper(
             user: Optional[User], dataset: str, selections: Iterable[tuple[str, str]], request_version: Optional[int]
@@ -187,33 +223,21 @@ class ApiServer:
         async def get_parameters_cachable(*args) -> T:
             return await do_cachable_action(params_cache, get_parameters_helper, *args)
         
-        async def get_parameters_definition(dataset: str, user: Optional[User], headers: Mapping, params: Mapping):
+        async def get_parameters_definition(dataset: str, user: Optional[User], headers: Mapping, params: Mapping) -> arm.ParametersModel:
             api_function = get_parameters_helper if self.no_cache else get_parameters_cachable
             result = await apply_dataset_api_function(api_function, user, dataset, headers, params)
             return process_based_on_response_version_header(headers, {
-                0: result.to_json_dict0
+                0: result.to_api_response_model0
             })
-        
-        @app.get(parameters_path, response_class=JSONResponse)
-        async def get_parameters(dataset: str, request: Request, user: Optional[User] = Depends(get_current_user)):
-            start = time.time()
-            result = await get_parameters_definition(dataset, user, request.headers, request.query_params)
-            timer.add_activity_time("GET REQUEST total time for PARAMETERS", start)
-            return result
-
-        @app.post(parameters_path, response_class=JSONResponse)
-        async def get_parameters_with_post(dataset: str, request: Request, user: Optional[User] = Depends(get_current_user)):
-            start = time.time()
-            request_body = await request.json()
-            result = await get_parameters_definition(dataset, user, request.headers, request_body)
-            timer.add_activity_time("POST REQUEST total time for PARAMETERS", start)
-            return result
 
-        # Results API
+        # Results API Helpers
         results_path = base_path + '/dataset/{dataset}'
 
+        def get_dataset_for_results_request(request: Request) -> str:
+            return get_dataset_from_request_path(request, -1)
+
         results_cache_size = ManifestIO.obj.settings.get(c.RESULTS_CACHE_SIZE_SETTING, 128)
-        results_cache_ttl = ManifestIO.obj.settings.get(c.RESULTS_CACHE_TTL_SETTING, 0)
+        results_cache_ttl = ManifestIO.obj.settings.get(c.RESULTS_CACHE_TTL_SETTING, 60)
     
         async def get_results_helper(
             user: Optional[User], dataset: str, selections: Iterable[tuple[str, str]], request_version: Optional[int]
@@ -224,70 +248,111 @@ class ApiServer:
 
         results_cache = TTLCache(maxsize=results_cache_size, ttl=results_cache_ttl*60)
 
-        async def get_results_cachable(*args):
+        async def get_results_cachable(*args) -> pd.DataFrame:
             return await do_cachable_action(results_cache, get_results_helper, *args)
         
-        async def get_results_definition(dataset: str, user: Optional[User], headers: Mapping, params: Mapping):
+        async def get_results_definition(dataset: str, user: Optional[User], headers: Mapping, params: Mapping) -> arm.DatasetResultModel:
             api_function = get_results_helper if self.no_cache else get_results_cachable
             result = await apply_dataset_api_function(api_function, user, dataset, headers, params)
             return process_based_on_response_version_header(headers, {
-                0: lambda: u.df_to_json0(result)
+                0: lambda: df_to_api_response0(result)
             })
         
-        @app.get(results_path, response_class=JSONResponse)
-        async def get_results(dataset: str, request: Request, user: Optional[User] = Depends(get_current_user)):
-            start = time.time()
-            result = await get_results_definition(dataset, user, request.headers, request.query_params)
-            timer.add_activity_time("GET REQUEST total time for DATASET", start)
-            return result
+        param_fields = ParameterConfigsSetIO.obj.get_all_api_field_info()
         
-        @app.post(results_path, response_class=JSONResponse)
-        async def get_results_with_post(dataset: str, request: Request, user: Optional[User] = Depends(get_current_user)):
-            start = time.time()
-            request_body = await request.json()
-            result = await get_results_definition(dataset, user, request.headers, request_body)
-            timer.add_activity_time("POST REQUEST total time for DATASET", start)
-            return result
+        # Dataset Parameters and Results APIs
+        for dataset_name, dataset_cfg in self.dataset_configs.items():
+            dataset_normalized = u.normalize_name_for_api(dataset_name)
+            curr_parameters_path = parameters_path.format(dataset=dataset_normalized)
+            curr_results_path = results_path.format(dataset=dataset_normalized)
+
+            QueryModelGet = make_dataclass("QueryParams", [
+                param_fields[param].as_query_info() for param in dataset_cfg.parameters
+            ])
+            AnnotatedQueryModel = Annotated[QueryModelGet, Depends()]
+
+            QueryModelPost = create_model("RequestBodyParams", **{
+                param: param_fields[param].as_body_info() for param in dataset_cfg.parameters
+            })
+        
+            @app.get(curr_parameters_path, response_class=JSONResponse)
+            async def get_parameters(
+                request: Request, params: AnnotatedQueryModel, user: Optional[User] = Depends(get_current_user) # type: ignore
+            ) -> arm.ParametersModel:
+                start = time.time()
+                dataset = get_dataset_for_parameters_request(request)
+                result = await get_parameters_definition(dataset, user, request.headers, asdict(params))
+                timer.add_activity_time("GET REQUEST total time for PARAMETERS endpoint", start)
+                return result
+
+            @app.post(curr_parameters_path, response_class=JSONResponse)
+            async def get_parameters_with_post(
+                request: Request, params: QueryModelPost, user: Optional[User] = Depends(get_current_user) # type: ignore
+            ) -> arm.ParametersModel:
+                start = time.time()
+                dataset = get_dataset_for_parameters_request(request)
+                params: BaseModel = params
+                result = await get_parameters_definition(dataset, user, request.headers, params.model_dump())
+                timer.add_activity_time("POST REQUEST total time for PARAMETERS endpoint", start)
+                return result
+            
+            @app.get(curr_results_path, response_class=JSONResponse)
+            async def get_results(
+                request: Request, params: AnnotatedQueryModel, user: Optional[User] = Depends(get_current_user) # type: ignore
+            ) -> arm.DatasetResultModel:
+                start = time.time()
+                dataset = get_dataset_for_results_request(request)
+                result = await get_results_definition(dataset, user, request.headers, asdict(params))
+                timer.add_activity_time("GET REQUEST total time for DATASET endpoint", start)
+                return result
+            
+            @app.post(curr_results_path, response_class=JSONResponse)
+            async def get_results_with_post(
+                request: Request, params: QueryModelPost, user: Optional[User] = Depends(get_current_user) # type: ignore
+            ) -> arm.DatasetResultModel:
+                start = time.time()
+                dataset = get_dataset_for_results_request(request)
+                params: BaseModel = params
+                result = await get_results_definition(dataset, user, request.headers, params.model_dump())
+                timer.add_activity_time("POST REQUEST total time for DATASET endpoint", start)
+                return result
         
         # Datasets Catalog API
         datasets_path = base_path + '/datasets'
 
-        def get_datasets0(user: Optional[User]):
+        def get_datasets0(user: Optional[User]) -> arm.DatasetsCatalogModel:
             datasets_info = []
             for dataset_name, dataset_config in self.dataset_configs.items():
                 if can_user_access_dataset(user, dataset_name):
                     dataset_normalized = u.normalize_name_for_api(dataset_name)
-                    datasets_info.append({
-                        'name': dataset_name,
-                        'label': dataset_config.label,
-                        'parameters_path': parameters_path.format(dataset=dataset_normalized),
-                        'result_path': results_path.format(dataset=dataset_normalized)
-                    })
-            return {"datasets": datasets_info}
+                    datasets_info.append(arm.DatasetInfoModel(
+                        name=dataset_name, label=dataset_config.label,
+                        parameters_path=parameters_path.format(dataset=dataset_normalized),
+                        result_path=results_path.format(dataset=dataset_normalized)
+                    ))
+            return arm.DatasetsCatalogModel(datasets=datasets_info)
         
         @app.get(datasets_path)
-        def get_datasets(request: Request, user: Optional[User] = Depends(get_current_user)):
+        def get_datasets(request: Request, user: Optional[User] = Depends(get_current_user)) -> arm.DatasetsCatalogModel:
             return process_based_on_response_version_header(request.headers, {
                 0: lambda: get_datasets0(user)
             })
         
         # Projects Catalog API
-        def get_catalog0():
-            return {
-                'projects': [{
-                    'name': ManifestIO.obj.project_variables.get_name(),
-                    'label': ManifestIO.obj.project_variables.get_label(),
-                    'versions': [{
-                        'major_version': ManifestIO.obj.project_variables.get_major_version(),
-                        'minor_versions': [0],
-                        'token_path': token_path,
-                        'datasets_path': datasets_path
-                    }]
-                }]
-            }
+        def get_catalog0() -> arm.CatalogModel:
+            return arm.CatalogModel(projects=[arm.ProjectModel(
+                name=ManifestIO.obj.project_variables.get_name(),
+                label=ManifestIO.obj.project_variables.get_label(),
+                versions=[arm.ProjectVersionModel(
+                    major_version=ManifestIO.obj.project_variables.get_major_version(),
+                    minor_versions=[0],
+                    token_path=token_path,
+                    datasets_path=datasets_path
+                )]
+            )])
         
         @app.get(squirrels_version_path, response_class=JSONResponse)
-        async def get_catalog(request: Request):
+        async def get_catalog(request: Request) -> arm.CatalogModel:
             return process_based_on_response_version_header(request.headers, {
                 0: lambda: get_catalog0()
             })

squirrels/_authenticator.py

@@ -1,7 +1,7 @@
 from typing import Optional
 from datetime import datetime, timedelta, timezone
-from jose import JWTError, jwt
-import secrets
+from jwt.exceptions import InvalidTokenError
+import secrets, jwt
 
 from . import _utils as u, _constants as c
 from .arguments.run_time_args import AuthArgs
@@ -16,7 +16,7 @@ class Authenticator:
     
     @classmethod
     def get_auth_helper(cls, default_auth_helper = None):
-        auth_module_path = u.join_paths(c.PYCONFIG_FOLDER, c.AUTH_FILE)
+        auth_module_path = u.join_paths(c.PYCONFIGS_FOLDER, c.AUTH_FILE)
         return PyModule(auth_module_path, default_class=default_auth_helper)
 
     def __init__(self, token_expiry_minutes: int, auth_helper = None) -> None:
@@ -47,15 +47,17 @@ class Authenticator:
         if not isinstance(real_user, WrongPassword):
             fake_users = EnvironConfigIO.obj.get_users()
             if username in fake_users and secrets.compare_digest(fake_users[username][c.USER_PWD_KEY], password):
-                is_internal = fake_users[username].get("is_internal", False)
+                fake_user = fake_users[username].copy()
+                fake_user.pop("username", "")
+                is_internal = fake_user.pop("is_internal", False)
                 try:
-                    return user_cls.Create(username, fake_users[username], is_internal=is_internal)
+                    return user_cls.Create(username, is_internal=is_internal, **fake_user)
                 except Exception as e:
                     raise u.FileExecutionError(f'Failed to create user from User model in {c.AUTH_FILE}', e)
         
         return None
     
-    def create_access_token(self, user: User) -> str:
+    def create_access_token(self, user: User) -> tuple[str, datetime]:
         expire = datetime.now(timezone.utc) + timedelta(minutes=self.token_expiry_minutes)
         to_encode = {**vars(user), "exp": expire}
         encoded_jwt = jwt.encode(to_encode, self.secret_key, algorithm=self.algorithm)
@@ -64,11 +66,11 @@ class Authenticator:
     def get_user_from_token(self, token: Optional[str]) -> Optional[User]:
         if token is not None:
             try:
-                payload = jwt.decode(token, self.secret_key, algorithms=[self.algorithm])
+                payload: dict = jwt.decode(token, self.secret_key, algorithms=[self.algorithm])
                 payload.pop("exp")
                 user_cls: User = self.auth_helper.get_func_or_class("User", default_attr=User)
                 return user_cls._FromDict(payload)
-            except JWTError:
+            except InvalidTokenError:
                 return None
 
     def can_user_access_scope(self, user: Optional[User], scope: DatasetScope) -> bool:

squirrels/_command_line.py

@@ -31,18 +31,22 @@ def main():
     module_parser = subparsers.add_parser(c.DEPS_CMD, help=f'Load all packages specified in {c.MANIFEST_FILE} (from git)', add_help=False)
     module_parser.add_argument('-h', '--help', action="help", help="Show this help message and exit")
 
-    compile_parser = subparsers.add_parser(c.COMPILE_CMD, help='Create files for rendered sql queries in the "target/compile" folder', add_help=False)
+    compile_parser = subparsers.add_parser(c.COMPILE_CMD, help='Create rendered SQL files in the folder "./target/compile"', add_help=False)
+    compile_dataset_group = compile_parser.add_mutually_exclusive_group(required=True)
+    compile_test_set_group = compile_parser.add_mutually_exclusive_group(required=False)
     compile_parser.add_argument('-h', '--help', action="help", help="Show this help message and exit")
-    compile_parser.add_argument('-d', '--dataset', type=str, help="Select dataset to use for dataset traits. If not specified, all models for all datasets are compiled")
-    compile_parser.add_argument('-a', '--all-test-sets', action="store_true", help="Compile models for all selection test sets")
-    compile_parser.add_argument('-t', '--test-set', type=str, help="The selection test set to use. Default selections are used if not specified. Ignored if using --all-test-sets")
-    compile_parser.add_argument('-s', '--select', type=str, help="Select single model to compile. If not specified, all models for the dataset are compiled. Also, ignored if --dataset is not specified")
+    
+    compile_dataset_group.add_argument('-d', '--dataset', type=str, help="Select dataset to use for dataset traits. Is required, unless using --all-datasets")
+    compile_dataset_group.add_argument('-D', '--all-datasets', action="store_true", help="Compile models for all datasets. Only required if --dataset is not specified")
+    compile_test_set_group.add_argument('-t', '--test-set', type=str, help="The selection test set to use. If not specified, default selections are used, unless using --all-test-sets")
+    compile_test_set_group.add_argument('-T', '--all-test-sets', action="store_true", help="Compile models for all selection test sets")
+    
+    compile_parser.add_argument('-s', '--select', type=str, help="Select single model to compile. If not specified, all models for the dataset are compiled. Ignored if using --all-datasets")
     compile_parser.add_argument('-r', '--runquery', action='store_true', help='Runs all target models, and produce the results as csv files')
 
-    run_parser = subparsers.add_parser(c.RUN_CMD, help='Run the builtin API server', add_help=False)
+    run_parser = subparsers.add_parser(c.RUN_CMD, help='Run the API server', add_help=False)
     run_parser.add_argument('-h', '--help', action="help", help="Show this help message and exit")
     run_parser.add_argument('--no-cache', action='store_true', help='Do not cache any api results')
-    run_parser.add_argument('--debug', action='store_true', help='Show all "hidden parameters" in the parameters response')
     run_parser.add_argument('--host', type=str, default='127.0.0.1', help="The host to run on")
     run_parser.add_argument('--port', type=int, default=4465, help="The port to run on")
 
@@ -58,6 +62,7 @@ def main():
     from ._package_loader import PackageLoaderIO
     from ._connection_set import ConnectionSetIO
     from ._parameter_sets import ParameterConfigsSetIO
+    from ._seeds import SeedsIO
 
     if args.version:
         print(__version__)
@@ -68,18 +73,19 @@ def main():
         PackageLoaderIO.LoadPackages(reload=True)
     elif args.command in [c.RUN_CMD, c.COMPILE_CMD]:
         ManifestIO.LoadFromFile()
+        SeedsIO.LoadFiles()
         ConnectionSetIO.LoadFromFile()
         try:
             ParameterConfigsSetIO.LoadFromFile()
             ModelsIO.LoadFiles()
-            
             if args.command == c.RUN_CMD:
-                server = ApiServer(args.no_cache, args.debug)
+                server = ApiServer(args.no_cache)
                 server.run(args)
-                pass
             elif args.command == c.COMPILE_CMD:
-                task = ModelsIO.WriteOutputs(args.dataset, args.select, args.all_test_sets, args.test_set, args.runquery)
+                task = ModelsIO.WriteOutputs(args.dataset, args.all_datasets, args.select, args.test_set, args.all_test_sets, args.runquery)
                 asyncio.run(task)
+        except KeyboardInterrupt:
+            pass
         finally:
             ConnectionSetIO.Dispose()
     elif args.command is None:

squirrels/_connection_set.py

@@ -29,9 +29,9 @@ class ConnectionSet:
             raise u.ConfigurationError(f'Connection name "{conn_name}" was not configured') from e
         return connection_pool
     
-    def run_sql_query_from_conn_name(self, query: str, conn_name: str) -> pd.DataFrame:
+    def run_sql_query_from_conn_name(self, query: str, conn_name: str, placeholders: dict = {}) -> pd.DataFrame:
         engine = self._get_engine(conn_name)
-        df = pd.read_sql(query, engine)
+        df = pd.read_sql(query, engine, params=placeholders)
         return df
 
     def _dispose(self) -> None:

squirrels/_constants.py

@@ -39,6 +39,7 @@ PARAMETER_ARGS_KEY = 'arguments'
 TEST_SETS_KEY = 'selection_test_sets'
 TEST_SET_NAME_KEY = 'name'
 DEFAULT_TEST_SET_NAME = 'default'
+TEST_SET_DATASETS_KEY = 'datasets'
 TEST_SET_USER_ATTR_KEY = 'user_attributes'
 TEST_SET_PARAMETERS_KEY = 'parameters'
 
@@ -48,6 +49,7 @@ DATASET_LABEL_KEY = 'label'
 DATASET_MODEL_KEY = 'model'
 DATASET_PARAMETERS_KEY = 'parameters'
 DATASET_TRAITS_KEY = 'traits'
+DATASET_DEFAULT_TEST_SET_KEY = 'default_test_set'
 
 DATASET_SCOPE_KEY = 'scope'
 PUBLIC_SCOPE = 'public'
@@ -94,7 +96,7 @@ FEDERATES_FOLDER = 'federates'
 FEDERATE_SQL_NAME = 'dataset_example.sql'
 FEDERATE_PY_NAME = 'dataset_example.py'
 
-PYCONFIG_FOLDER = 'pyconfigs'
+PYCONFIGS_FOLDER = 'pyconfigs'
 AUTH_FILE = 'auth.py'
 CONNECTIONS_FILE = 'connections.py'
 CONTEXT_FILE = 'context.py'
@@ -103,9 +105,9 @@ PARAMETERS_FILE = 'parameters.py'
 TARGET_FOLDER = 'target'
 COMPILE_FOLDER = 'compile'
 
-OUTPUTS_FOLDER = 'outputs'
-PARAMETERS_OUTPUT = 'parameters.json'
-FINAL_VIEW_OUT_STEM = 'final_view'
+SEEDS_FOLDER = 'seeds'
+CATEGORY_SEED_FILE = 'seed_categories.csv'
+SUBCATEGORY_SEED_FILE = 'seed_subcategories.csv'
 
 # Dataset setting names
 AUTH_TOKEN_EXPIRE_SETTING = 'auth.token.expire_minutes'
@@ -116,6 +118,8 @@ RESULTS_CACHE_TTL_SETTING = 'results.cache.ttl_minutes'
 TEST_SET_DEFAULT_USED_SETTING = 'selection_test_sets.default_name_used'
 DB_CONN_DEFAULT_USED_SETTING = 'connections.default_name_used'
 DEFAULT_MATERIALIZE_SETTING = 'defaults.federates.materialized'
+SEEDS_INFER_SCHEMA_SETTING = 'seeds.infer_schema'
+SEEDS_NA_VALUES_SETTING = 'seeds.na_values'
 IN_MEMORY_DB_SETTING = 'in_memory_database'
 SQLITE = 'sqlite'
 DUCKDB = 'duckdb'
@@ -138,9 +142,13 @@ CONF_FORMAT_CHOICES2 = [(PYTHON_FORMAT2, PYTHON_FORMAT), YML_FORMAT]
 
 EXPENSES_DB_NAME = 'expenses'
 WEATHER_DB_NAME = 'weather'
-DATABASE_CHOICES = [EXPENSES_DB_NAME, WEATHER_DB_NAME]
+NO_DB = 'none'
+DATABASE_CHOICES = [EXPENSES_DB_NAME, WEATHER_DB_NAME, NO_DB]
 
 # Function names
 GET_USER_FUNC = "get_user_if_valid"
 DEP_FUNC = "dependencies"
 MAIN_FUNC = "main"
+
+# Regex
+date_regex = r'^\d{4}\-\d{2}\-\d{2}$'