squirrels 0.4.0__py3-none-any.whl → 0.5.0rc0__py3-none-any.whl

squirrels/_api_server.py

@@ -1,22 +1,26 @@
 from typing import Coroutine, Mapping, Callable, TypeVar, Annotated, Any
 from dataclasses import make_dataclass, asdict
-from fastapi import Depends, FastAPI, Request, HTTPException, Response, status
-from fastapi.responses import HTMLResponse, JSONResponse
-from fastapi.templating import Jinja2Templates
-from fastapi.staticfiles import StaticFiles
+from fastapi import Depends, FastAPI, Request, Response, status
+from fastapi.responses import HTMLResponse, JSONResponse, RedirectResponse
 from fastapi.security import OAuth2PasswordBearer, OAuth2PasswordRequestForm
 from fastapi.middleware.cors import CORSMiddleware
-from pydantic import create_model, BaseModel
+from pydantic import create_model, BaseModel, Field
+from contextlib import asynccontextmanager
 from cachetools import TTLCache
 from argparse import Namespace
-import os, io, time, mimetypes, traceback, uuid, pandas as pd
+from pathlib import Path
+import io, time, mimetypes, traceback, uuid, asyncio, urllib.parse
 
 from . import _constants as c, _utils as u, _api_response_models as arm
-from ._version import sq_major_version
-from ._authenticator import User
+from ._exceptions import InvalidInputError, ConfigurationError, FileExecutionError
+from ._version import __version__, sq_major_version
+from ._manifest import PermissionScope
+from ._auth import BaseUser, AccessToken, UserField
 from ._parameter_sets import ParameterSet
 from .dashboards import Dashboard
-from .project import SquirrelsProject
+from ._project import SquirrelsProject
+from .dataset_result import DatasetResult
+from ._parameter_configs import APIParamFieldInfo
 
 mimetypes.add_type('application/javascript', '.js')
 
@@ -32,20 +36,54 @@ class ApiServer:
         self.no_cache = no_cache
         self.project = project
         self.logger = project._logger
-
+        self.env_vars = project._env_vars
         self.j2_env = project._j2_env
-        self.env_cfg = project._env_cfg
         self.manifest_cfg = project._manifest_cfg
         self.seeds = project._seeds
         self.conn_args = project._conn_args
         self.conn_set = project._conn_set
-        self.authenticator = project._authenticator
+        self.authenticator = project._auth
         self.param_args = project._param_args
         self.param_cfg_set = project._param_cfg_set
         self.context_func = project._context_func
-        self.model_files = project._model_files
         self.dashboards = project._dashboards
     
+    
+    async def _monitor_for_staging_file(self) -> None:
+        """Background task that monitors for staging file and renames it when present"""
+        duckdb_venv_path = self.project._duckdb_venv_path
+        staging_file = Path(duckdb_venv_path + ".stg")
+        target_file = Path(duckdb_venv_path)
+                
+        while True:
+            try:
+                if staging_file.exists():
+                    try:
+                        staging_file.replace(target_file)
+                        self.logger.info("Successfully renamed staging database to virtual environment database")
+                    except OSError:
+                        # Silently continue if file cannot be renamed (will retry next iteration)
+                        pass
+                
+            except Exception as e:
+                # Log any unexpected errors but keep running
+                self.logger.error(f"Error in monitoring {c.DUCKDB_VENV_FILE + '.stg'}: {str(e)}")
+            
+            await asyncio.sleep(1)  # Check every second
+    
+    @asynccontextmanager
+    async def _run_background_tasks(self, app: FastAPI):
+        task = asyncio.create_task(self._monitor_for_staging_file())
+        yield
+        task.cancel()
+
+
+    def _validate_request_params(self, all_request_params: Mapping, params: Mapping) -> None:
+        invalid_params = [param for param in all_request_params if param not in params]
+        if params.get("x_verify_params", False) and invalid_params:
+            raise InvalidInputError(201, f"Invalid query parameters: {', '.join(invalid_params)}")
+        
+    
     def run(self, uvicorn_args: Namespace) -> None:
         """
         Runs the API server with uvicorn for CLI "squirrels run"
@@ -55,18 +93,29 @@ class ApiServer:
         """
         start = time.time()
         
+        squirrels_version_path = f'/api/squirrels-v{sq_major_version}'
+        project_name = u.normalize_name_for_api(self.manifest_cfg.project_variables.name)
+        project_version = f"v{self.manifest_cfg.project_variables.major_version}"
+        project_metadata_path = squirrels_version_path + f"/project/{project_name}/{project_version}"
+        
+        param_fields = self.param_cfg_set.get_all_api_field_info()
+
         tags_metadata = [
             {
-                "name": "Project Metadata",
-                "description": "Get information on project such as name, version, and other API endpoints",
+                "name": "Authentication",
+                "description": "Submit authentication credentials, and get token for authentication",
+            },
+            {
+                "name": "User Management",
+                "description": "Manage users and their attributes",
             },
             {
-                "name": "Login",
-                "description": "Submit username and password, and get token for authentication",
+                "name": "Project Metadata",
+                "description": "Get information on project such as name, version, and other API endpoints",
             },
             {
-                "name": "Catalogs",
-                "description": "Get catalog of datasets and dashboards with endpoints for their parameters and results",
+                "name": "Data Management",
+                "description": "Actions to update the data components of the project",
             }
         ]
 
@@ -76,7 +125,7 @@ class ApiServer:
                 "description": f"Get parameters or results for dataset '{dataset_name}'",
             })
         
-        for dashboard_name in self.manifest_cfg.dashboards:
+        for dashboard_name in self.dashboards:
             tags_metadata.append({
                 "name": f"Dashboard '{dashboard_name}'",
                 "description": f"Get parameters or results for dashboard '{dashboard_name}'",
@@ -84,7 +133,11 @@ class ApiServer:
         
         app = FastAPI(
             title=f"Squirrels APIs for '{self.manifest_cfg.project_variables.label}'", openapi_tags=tags_metadata,
-            description="For specifying parameter selections to dataset APIs, you can choose between using query parameters with the GET method or using request body with the POST method"
+            description="For specifying parameter selections to dataset APIs, you can choose between using query parameters with the GET method or using request body with the POST method",
+            lifespan=self._run_background_tasks,
+            openapi_url=project_metadata_path+"/openapi.json",
+            docs_url=project_metadata_path+"/docs",
+            redoc_url=project_metadata_path+"/redoc"
         )
 
         async def _log_request_run(request: Request) -> None:
@@ -114,18 +167,31 @@ class ApiServer:
             try:
                 await _log_request_run(request)
                 return await call_next(request)
-            except u.InvalidInputError as exc:
+            except InvalidInputError as exc:
                 traceback.print_exc(file=buffer)
+                message = str(exc)
+                if exc.error_code < 20:
+                    status_code = status.HTTP_401_UNAUTHORIZED
+                elif exc.error_code < 40:
+                    status_code = status.HTTP_403_FORBIDDEN
+                elif exc.error_code < 60:
+                    status_code = status.HTTP_404_NOT_FOUND
+                elif exc.error_code < 70:
+                    if exc.error_code == 61:
+                        message = "The dataset depends on static data models that cannot be found. You may need to build the virtual data environment first."
+                    status_code = status.HTTP_409_CONFLICT
+                else:
+                    status_code = status.HTTP_400_BAD_REQUEST
                 response = JSONResponse(
-                    status_code=status.HTTP_400_BAD_REQUEST, content={"message": str(exc), "blame": "API client"}
+                    status_code=status_code, content={"message": message, "blame": "API client", "error_code": exc.error_code}
                 )
-            except u.FileExecutionError as exc:
+            except FileExecutionError as exc:
                 traceback.print_exception(exc.error, file=buffer)
                 buffer.write(str(exc))
                 response = JSONResponse(
                     status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, content={"message": f"An unexpected error occurred", "blame": "Squirrels project"}
                 )
-            except u.ConfigurationError as exc:
+            except ConfigurationError as exc:
                 traceback.print_exc(file=buffer)
                 response = JSONResponse(
                     status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, content={"message": f"An unexpected error occurred", "blame": "Squirrels project"}
@@ -146,56 +212,22 @@ class ApiServer:
             expose_headers=["Applied-Username"]
         )
 
-        squirrels_version_path = f'/squirrels-v{sq_major_version}'
-        partial_base_path = f'/{self.manifest_cfg.project_variables.name}/v{self.manifest_cfg.project_variables.major_version}'
-        base_path = squirrels_version_path + u.normalize_name_for_api(partial_base_path)
-        
         # Helpers
         T = TypeVar('T')
         
-        def get_versioning_request_header(headers: Mapping, header_key: str):
-            header_value = headers.get(header_key)
-            if header_value is None:
-                return None
-            
-            try:
-                result = int(header_value)
-            except ValueError:
-                raise u.InvalidInputError(f"Request header '{header_key}' must be an integer. Got '{header_value}'")
-            
-            if result < 0 or result > int(sq_major_version):
-                raise u.InvalidInputError(f"Request header '{header_key}' not in valid range. Got '{result}'")
-            
-            return result
-
-        def get_request_version_header(headers: Mapping):
-            REQUEST_VERSION_REQUEST_HEADER = "squirrels-request-version"
-            return get_versioning_request_header(headers, REQUEST_VERSION_REQUEST_HEADER)
-        
-        def process_based_on_response_version_header(headers: Mapping, processes: dict[int, Callable[[], T]]) -> T:
-            RESPONSE_VERSION_REQUEST_HEADER = "squirrels-response-version"
-            response_version = get_versioning_request_header(headers, RESPONSE_VERSION_REQUEST_HEADER)
-            if response_version is None or response_version >= 0:
-                return processes[0]()
-            else:
-                raise u.InvalidInputError(f'Invalid value for "{RESPONSE_VERSION_REQUEST_HEADER}" header: {response_version}')
-        
-        def get_selections_and_request_version(params: Mapping, headers: Mapping) -> tuple[frozenset[tuple[str, Any]], int | None]:
-            # Changing selections into a cachable "frozenset" that will later be converted to dictionary
-            selections = set()
+        def get_selections_as_immutable(params: Mapping, uncached_keys: set[str]) -> tuple[tuple[str, Any], ...]:
+            # Changing selections into a cachable "tuple of pairs" that will later be converted to dictionary
+            selections = list()
             for key, val in params.items():
-                if val is None:
+                if key in uncached_keys or val is None:
                     continue
                 if isinstance(val, (list, tuple)):
                     if len(val) == 1: # for backward compatibility
                         val = val[0]
                     else:
                         val = tuple(val)
-                selections.add((u.normalize_name(key), val))
-            selections = frozenset(selections)
-            
-            request_version = get_request_version_header(headers)
-            return selections, request_version
+                selections.append((u.normalize_name(key), val))
+            return tuple(selections)
 
         async def do_cachable_action(cache: TTLCache, action: Callable[..., Coroutine[Any, Any, T]], *args) -> T:
             cache_key = tuple(args)
@@ -204,18 +236,55 @@ class ApiServer:
                 result = await action(*args)
                 cache[cache_key] = result
             return result
-        
-        def get_query_models_from_widget_params(parameters: list):
+
+        def _get_query_models_helper(widget_parameters: list[str] | None, predefined_params: list[APIParamFieldInfo]):
+            if widget_parameters is None:
+                widget_parameters = list(param_fields.keys())
+            
             QueryModelForGetRaw = make_dataclass("QueryParams", [
-                param_fields[param].as_query_info() for param in parameters
-            ])
+                param_fields[param].as_query_info() for param in widget_parameters
+            ] + [param.as_query_info() for param in predefined_params])
             QueryModelForGet = Annotated[QueryModelForGetRaw, Depends()]
 
-            QueryModelForPost = create_model("RequestBodyParams", **{
-                param: param_fields[param].as_body_info() for param in parameters
-            }) # type: ignore
+            field_definitions = {param: param_fields[param].as_body_info() for param in widget_parameters}
+            for param in predefined_params:
+                field_definitions[param.name] = param.as_body_info()
+            QueryModelForPost = create_model("RequestBodyParams", **field_definitions) # type: ignore
             return QueryModelForGet, QueryModelForPost
         
+        def get_query_models_for_parameters(widget_parameters: list[str] | None):
+            predefined_params = [
+                APIParamFieldInfo("x_verify_params", bool, default=False, description="If true, the query parameters are verified to be valid for the dataset"),
+                APIParamFieldInfo("x_parent_param", str, description="The parameter name used for parameter updates. If not provided, then all parameters are retrieved"),
+            ]
+            return _get_query_models_helper(widget_parameters, predefined_params)
+        
+        def get_query_models_for_dataset(widget_parameters: list[str] | None):
+            predefined_params = [
+                APIParamFieldInfo("x_verify_params", bool, default=False, description="If true, the query parameters are verified to be valid for the dataset"),
+                APIParamFieldInfo("x_orientation", str, default="records", description="The orientation of the data to return, one of: 'records', 'rows', or 'columns'"),
+                APIParamFieldInfo("x_select", list[str], examples=[[]], description="The columns to select from the dataset. All are returned if not specified"), 
+                APIParamFieldInfo("x_offset", int, default=0, description="The number of rows to skip before returning data (applied after data caching)"),
+                APIParamFieldInfo("x_limit", int, default=1000, description="The maximum number of rows to return (applied after data caching and offset)"),
+            ]
+            return _get_query_models_helper(widget_parameters, predefined_params)
+        
+        def get_query_models_for_dashboard(widget_parameters: list[str] | None):
+            predefined_params = [
+                APIParamFieldInfo("x_verify_params", bool, default=False, description="If true, the query parameters are verified to be valid for the dashboard"),
+            ]
+            return _get_query_models_helper(widget_parameters, predefined_params)
+        
+        def get_query_models_for_querying_models():
+            predefined_params = [
+                APIParamFieldInfo("x_verify_params", bool, default=False, description="If true, the query parameters are verified to be valid"),
+                APIParamFieldInfo("x_orientation", str, default="records", description="The orientation of the data to return, one of: 'records', 'rows', or 'columns'"),
+                APIParamFieldInfo("x_offset", int, default=0, description="The number of rows to skip before returning data (applied after data caching)"),
+                APIParamFieldInfo("x_limit", int, default=1000, description="The maximum number of rows to return (applied after data caching and offset)"),
+                APIParamFieldInfo("x_sql_query", str, description="The SQL query to execute on the data models"),
+            ]
+            return _get_query_models_helper(None, predefined_params)
+        
         def _get_section_from_request_path(request: Request, section: int) -> str:
             url_path: str = request.scope['route'].path
             return url_path.split('/')[section]
@@ -228,68 +297,245 @@ class ApiServer:
             dashboard_raw = _get_section_from_request_path(request, section)
             return u.normalize_name(dashboard_raw)
         
-        # Login & Authorization
-        token_path = base_path + '/token'
+        expiry_mins = self.env_vars.get(c.SQRL_AUTH_TOKEN_EXPIRE_MINUTES, 30)
+        try:
+            expiry_mins = int(expiry_mins)
+        except ValueError:
+            raise ConfigurationError(f"Value for environment variable {c.SQRL_AUTH_TOKEN_EXPIRE_MINUTES} is not an integer, got: {expiry_mins}")
+        
+        # Project Metadata API
+        
+        @app.get(project_metadata_path, tags=["Project Metadata"], response_class=JSONResponse)
+        async def get_project_metadata(request: Request) -> arm.ProjectModel:
+            return arm.ProjectModel(
+                name=project_name,
+                version=project_version,
+                label=self.manifest_cfg.project_variables.label,
+                description=self.manifest_cfg.project_variables.description,
+                squirrels_version=__version__
+            )
+        
+        # Authentication
+        login_path = project_metadata_path + '/login'
 
-        oauth2_scheme = OAuth2PasswordBearer(tokenUrl=token_path, auto_error=False)
+        oauth2_scheme = OAuth2PasswordBearer(tokenUrl=login_path, auto_error=False)
 
-        @app.post(token_path, tags=["Login"])
-        async def login_for_access_token(form_data: OAuth2PasswordRequestForm = Depends()) -> arm.LoginReponse:
-            user: User | None = self.authenticator.authenticate_user(form_data.username, form_data.password)
-            if not user:
-                raise HTTPException(
-                    status_code=status.HTTP_401_UNAUTHORIZED, detail="Incorrect username or password", headers={"WWW-Authenticate": "Bearer"}
-                )
-            access_token, expiry = self.authenticator.create_access_token(user)
-            return arm.LoginReponse(access_token=access_token, token_type="bearer", username=user.username, expiry_time=expiry)
-        
-        async def get_current_user(response: Response, token: str = Depends(oauth2_scheme)) -> User | None:
+        async def get_current_user(response: Response, token: str = Depends(oauth2_scheme)) -> BaseUser | None:
             user = self.authenticator.get_user_from_token(token)
             username = "" if user is None else user.username
             response.headers["Applied-Username"] = username
             return user
 
-        # Datasets / Dashboards Catalog API
-        data_catalog_path = base_path + '/data-catalog'
-        dataset_results_path = base_path + '/dataset/{dataset}'
+        ## Login API
+        @app.post(login_path, tags=["Authentication"])
+        async def login_for_access_token(form_data: OAuth2PasswordRequestForm = Depends()) -> arm.LoginReponse:
+            user = self.authenticator.get_user(form_data.username, form_data.password)
+            access_token, expiry = self.authenticator.create_access_token(user, expiry_minutes=expiry_mins)
+            return arm.LoginReponse(access_token=access_token, token_type="bearer", username=user.username, is_admin=user.is_admin, expiry_time=expiry)
+        
+        ## Change Password API
+        change_password_path = project_metadata_path + '/change-password'
+
+        class ChangePasswordRequest(BaseModel):
+            old_password: str
+            new_password: str
+
+        @app.put(change_password_path, description="Change the password for the current user", tags=["Authentication"])
+        async def change_password(request: ChangePasswordRequest, user: BaseUser | None = Depends(get_current_user)) -> None:
+            if user is None:
+                raise InvalidInputError(1, "Invalid authorization token")
+            self.authenticator.change_password(user.username, request.old_password, request.new_password)
+
+        ## Token API
+        tokens_path = project_metadata_path + '/tokens'
+
+        class TokenRequestBody(BaseModel):
+            title: str | None = Field(default=None, description=f"The title of the token. If not provided, a temporary token is created (expiring in {expiry_mins} minutes) and cannot be revoked")
+            expiry_minutes: int | None = Field(
+                default=None, 
+                description=f"The number of minutes the token is valid for (or indefinitely if not provided). Ignored and set to {expiry_mins} minutes if title is not provided."
+            )
+
+        @app.post(tokens_path, description="Create a new token for the user", tags=["Authentication"])
+        async def create_token(body: TokenRequestBody, user: BaseUser | None = Depends(get_current_user)) -> arm.LoginReponse:
+            if user is None:
+                raise InvalidInputError(1, "Invalid authorization token")
+            
+            if body.title is None:
+                expiry_minutes = expiry_mins
+            else:
+                expiry_minutes = body.expiry_minutes
+            
+            access_token, expiry = self.authenticator.create_access_token(user, expiry_minutes=expiry_minutes, title=body.title)
+            return arm.LoginReponse(access_token=access_token, token_type="bearer", username=user.username, is_admin=user.is_admin, expiry_time=expiry)
+        
+        ## Get All Tokens API
+        @app.get(tokens_path, description="Get all tokens with title for the current user", tags=["Authentication"])
+        async def get_all_tokens(user: BaseUser | None = Depends(get_current_user)) -> list[AccessToken]:
+            if user is None:
+                raise InvalidInputError(1, "Invalid authorization token")
+            return self.authenticator.get_all_tokens(user.username)
+        
+        ## Revoke Token API
+        revoke_token_path = project_metadata_path + '/tokens/{token_id}'
+
+        @app.delete(revoke_token_path, description="Revoke a token", tags=["Authentication"])
+        async def revoke_token(token_id: str, user: BaseUser | None = Depends(get_current_user)) -> None:
+            if user is None:
+                raise InvalidInputError(1, "Invalid authorization token")
+            self.authenticator.revoke_token(user.username, token_id)
+        
+        ## Get Authenticated User Fields From Token API
+        get_me_path = project_metadata_path + '/me'
+
+        fields_without_username = {
+            k: (v.annotation, v.default) 
+            for k, v in self.authenticator.User.model_fields.items() 
+            if k != "username"
+        }
+        UserModel = create_model("UserModel", __base__=BaseModel, **fields_without_username) # type: ignore
+
+        class UserWithoutUsername(UserModel):
+            pass
+
+        class UserWithUsername(UserModel):
+            username: str
+
+        class AddUserRequestBody(UserWithUsername):
+            password: str
+        
+        @app.get(get_me_path, description="Get the authenticated user's fields", tags=["Authentication"])
+        async def get_me(user: BaseUser | None = Depends(get_current_user)) -> UserWithUsername:
+            if user is None:
+                raise InvalidInputError(1, "Invalid authorization token")
+            return UserWithUsername(**user.model_dump(mode='json'))
+
+        # User Management
+
+        ## User Fields API
+        user_fields_path = project_metadata_path + '/user-fields'
+
+        @app.get(user_fields_path, description="Get details of the user fields", tags=["User Management"])
+        async def get_user_fields() -> list[UserField]:
+            return self.authenticator.user_fields
+        
+        ## Add User API
+        add_user_path = project_metadata_path + '/users'
+
+        @app.post(add_user_path, description="Add a new user by providing details for username, password, and user fields", tags=["User Management"])
+        async def add_user(
+            new_user: AddUserRequestBody, user: BaseUser | None = Depends(get_current_user)
+        ) -> None:
+            if user is None or not user.is_admin:
+                raise InvalidInputError(20, "Authorized user is forbidden to add new users")
+            self.authenticator.add_user(new_user.username, new_user.model_dump(mode='json', exclude={"username"}))
+
+        ## Update User API
+        update_user_path = project_metadata_path + '/users/{username}'
+
+        @app.put(update_user_path, description="Update the user of the given username given the new user details", tags=["User Management"])
+        async def update_user(
+            username: str, updated_user: UserWithoutUsername, user: BaseUser | None = Depends(get_current_user)
+        ) -> None:
+            if user is None or not user.is_admin:
+                raise InvalidInputError(20, "Authorized user is forbidden to update users")
+            self.authenticator.add_user(username, updated_user.model_dump(mode='json'), update_user=True)
+
+        ## List Users API
+        list_users_path = project_metadata_path + '/users'
+
+        @app.get(list_users_path, tags=["User Management"])
+        async def list_all_users() -> list[UserWithUsername]:
+            return self.authenticator.get_all_users()
+        
+        ## Delete User API
+        delete_user_path = project_metadata_path + '/users/{username}'
+
+        @app.delete(delete_user_path, tags=["User Management"])
+        async def delete_user(username: str, user: BaseUser | None = Depends(get_current_user)) -> None:
+            if user is None or not user.is_admin:
+                raise InvalidInputError(21, "Authorized user is forbidden to delete users")
+            if username == user.username:
+                raise InvalidInputError(22, "Cannot delete your own user")
+            self.authenticator.delete_user(username)
+        
+        # Data Catalog API
+        data_catalog_path = project_metadata_path + '/data-catalog'
+
+        dataset_results_path = project_metadata_path + '/dataset/{dataset}'
         dataset_parameters_path = dataset_results_path + '/parameters'
-        dashboard_results_path = base_path + '/dashboard/{dashboard}'
+
+        dashboard_results_path = project_metadata_path + '/dashboard/{dashboard}'
         dashboard_parameters_path = dashboard_results_path + '/parameters'
         
-        def get_data_catalog0(user: User | None) -> arm.CatalogModel:
+        async def get_data_catalog0(user: BaseUser | None) -> arm.CatalogModel:
+            parameters = self.param_cfg_set.apply_selections(None, {}, user)
+            parameters_model = parameters.to_api_response_model0()
+            full_parameters_list = [p.name for p in parameters_model.parameters]
+
             dataset_items: list[arm.DatasetItemModel] = []
             for name, config in self.manifest_cfg.datasets.items():
                 if self.authenticator.can_user_access_scope(user, config.scope):
                     name_normalized = u.normalize_name_for_api(name)
+                    metadata = self.project.dataset_metadata(name).to_json()
+                    parameters = config.parameters if config.parameters is not None else full_parameters_list
                     dataset_items.append(arm.DatasetItemModel(
-                        name=name, label=config.label, description=config.description,
+                        name=name_normalized, label=config.label, 
+                        description=config.description,
+                        schema=metadata["schema"], # type: ignore
+                        parameters=parameters,
                         parameters_path=dataset_parameters_path.format(dataset=name_normalized),
                         result_path=dataset_results_path.format(dataset=name_normalized)
                     ))
             
             dashboard_items: list[arm.DashboardItemModel] = []
-            for name, config in self.manifest_cfg.dashboards.items():
+            for name, dashboard in self.dashboards.items():
+                config = dashboard.config
                 if self.authenticator.can_user_access_scope(user, config.scope):
                     name_normalized = u.normalize_name_for_api(name)
 
                     try:
                         dashboard_format = self.dashboards[name].get_dashboard_format()
                     except KeyError:
-                        raise u.ConfigurationError(f"No dashboard file found for: {name}")
+                        raise ConfigurationError(f"No dashboard file found for: {name}")
                     
+                    parameters = config.parameters if config.parameters is not None else full_parameters_list
                     dashboard_items.append(arm.DashboardItemModel(
-                        name=name, label=config.label, description=config.description, result_format=dashboard_format,
+                        name=name, label=config.label, 
+                        description=config.description, 
+                        result_format=dashboard_format,
+                        parameters=parameters,
                         parameters_path=dashboard_parameters_path.format(dashboard=name_normalized),
                         result_path=dashboard_results_path.format(dashboard=name_normalized)
                     ))
             
-            return arm.CatalogModel(datasets=dataset_items, dashboards=dashboard_items)
+            if user and user.is_admin:
+                compiled_dag = await self.project._get_compiled_dag(user=user)
+                connections_items = self.project._get_all_connections()
+                data_models = self.project._get_all_data_models(compiled_dag)
+                lineage_items = self.project._get_all_data_lineage(compiled_dag)
+            else:
+                connections_items = []
+                data_models = []
+                lineage_items = []
+
+            return arm.CatalogModel(
+                parameters=parameters_model.parameters, 
+                datasets=dataset_items, 
+                dashboards=dashboard_items,
+                connections=connections_items,
+                models=data_models,
+                lineage=lineage_items,
+            )
         
-        @app.get(data_catalog_path, tags=["Catalogs"], summary="Get list of datasets and dashboards available for user")
-        def get_catalog_of_datasets_and_dashboards(request: Request, user: User | None = Depends(get_current_user)) -> arm.CatalogModel:
-            return process_based_on_response_version_header(request.headers, {
-                0: lambda: get_data_catalog0(user)
-            })
+        @app.get(data_catalog_path, tags=["Project Metadata"], summary="Get catalog of datasets and dashboards available for user")
+        async def get_data_catalog(request: Request, user: BaseUser | None = Depends(get_current_user)) -> arm.CatalogModel:
+            """
+            Get catalog of datasets and dashboards available for the authenticated user.
+            
+            For admin users, this endpoint will also return detailed information about all models and their lineage in the project.
+            """
+            return await get_data_catalog0(user)
         
         # Parameters API Helpers
         parameters_description = "Selections of one parameter may cascade the available options in another parameter. " \
@@ -298,99 +544,145 @@ class ApiServer:
                 "selection to this endpoint whenever it changes to refresh the parameter options of children parameters."
         
         async def get_parameters_helper(
-            parameters_tuple: tuple[str, ...], user: User | None, selections: frozenset[tuple[str, Any]], request_version: int | None
+            parameters_tuple: tuple[str, ...] | None, entity_type: str, entity_name: str, entity_scope: PermissionScope,
+            user: BaseUser | None, selections: tuple[tuple[str, Any], ...]
         ) -> ParameterSet:
-            if len(selections) > 1:
-                raise u.InvalidInputError(f"The /parameters endpoint takes at most 1 query parameter. Got {dict(selections)}")
+            selections_dict = dict(selections)
+            if "x_parent_param" not in selections_dict:
+                if len(selections_dict) > 1:
+                    raise InvalidInputError(202, f"The parameters endpoint takes at most 1 widget parameter selection (unless x_parent_param is provided). Got {selections_dict}")
+                elif len(selections_dict) == 1:
+                    parent_param = next(iter(selections_dict))
+                    selections_dict["x_parent_param"] = parent_param
             
-            param_set = self.param_cfg_set.apply_selections(
-                parameters_tuple, dict(selections), user, updates_only=True, request_version=request_version
-            )
+            parent_param = selections_dict.get("x_parent_param")
+            if parent_param is not None and parent_param not in selections_dict:
+                # this condition is possible for multi-select parameters with empty selection
+                selections_dict[parent_param] = list()
+            
+            if not self.authenticator.can_user_access_scope(user, entity_scope):
+                raise self.project._permission_error(user, entity_type, entity_name, entity_scope.name)
+            
+            param_set = self.param_cfg_set.apply_selections(parameters_tuple, selections_dict, user, parent_param=parent_param)
             return param_set
 
-        settings = self.manifest_cfg.settings
-        parameters_cache_size = settings.get(c.PARAMETERS_CACHE_SIZE_SETTING, 1024)
-        parameters_cache_ttl = settings.get(c.PARAMETERS_CACHE_TTL_SETTING, 60)
+        parameters_cache_size = int(self.env_vars.get(c.SQRL_PARAMETERS_CACHE_SIZE, 1024))
+        parameters_cache_ttl = int(self.env_vars.get(c.SQRL_PARAMETERS_CACHE_TTL_MINUTES, 60))
         params_cache = TTLCache(maxsize=parameters_cache_size, ttl=parameters_cache_ttl*60)
 
         async def get_parameters_cachable(
-            parameters_tuple: tuple[str, ...], user: User | None, selections: frozenset[tuple[str, Any]], request_version: int | None
+            parameters_tuple: tuple[str, ...] | None, entity_type: str, entity_name: str, entity_scope: PermissionScope,
+            user: BaseUser | None, selections: tuple[tuple[str, Any], ...]
         ) -> ParameterSet:
-            return await do_cachable_action(params_cache, get_parameters_helper, parameters_tuple, user, selections, request_version)
+            return await do_cachable_action(params_cache, get_parameters_helper, parameters_tuple, entity_type, entity_name, entity_scope, user, selections)
         
         async def get_parameters_definition(
-            parameters_list: list[str], user: User | None, headers: Mapping, params: Mapping
+            parameters_list: list[str] | None, entity_type: str, entity_name: str, entity_scope: PermissionScope,
+            user: BaseUser | None, all_request_params: dict, params: Mapping
         ) -> arm.ParametersModel:
-            get_parameters_function = get_parameters_helper if self.no_cache else get_parameters_cachable
-            selections, request_version = get_selections_and_request_version(params, headers)
-            result = await get_parameters_function(tuple(parameters_list), user, selections, request_version)
-            return process_based_on_response_version_header(headers, {
-                0: result.to_api_response_model0
-            })
-
-        param_fields = self.param_cfg_set.get_all_api_field_info()
+            self._validate_request_params(all_request_params, params)
 
-        def validate_parameters_list(parameters: list[str], entity_type: str) -> None:
+            get_parameters_function = get_parameters_helper if self.no_cache else get_parameters_cachable
+            selections = get_selections_as_immutable(params, uncached_keys={"x_verify_params"})
+            parameters_tuple = tuple(parameters_list) if parameters_list is not None else None
+            result = await get_parameters_function(parameters_tuple, entity_type, entity_name, entity_scope, user, selections)
+            return result.to_api_response_model0()
+
+        def validate_parameters_list(parameters: list[str] | None, entity_type: str) -> None:
+            if parameters is None:
+                return
             for param in parameters:
                 if param not in param_fields:
                     all_params = list(param_fields.keys())
-                    raise u.ConfigurationError(f"{entity_type} '{dataset_name}' use parameter '{param}' which doesn't exist. Available parameters are:"
-                                               f"\n  {all_params}")
+                    raise ConfigurationError(
+                        f"{entity_type} '{dataset_name}' use parameter '{param}' which doesn't exist. Available parameters are:"
+                        f"\n  {all_params}"
+                    )
+        
+        # Project-Level Parameters API
+        project_level_parameters_path = project_metadata_path + '/parameters'
+
+        QueryModelForGetProjectParams, QueryModelForPostProjectParams = get_query_models_for_parameters(None)
+
+        @app.get(project_level_parameters_path, tags=["Project Metadata"], description=parameters_description)
+        async def get_project_parameters(
+            request: Request, params: QueryModelForGetProjectParams, user: BaseUser | None = Depends(get_current_user) # type: ignore
+        ) -> arm.ParametersModel:
+            start = time.time()
+            result = await get_parameters_definition(
+                None, "project", "", PermissionScope.PUBLIC, user, dict(request.query_params), asdict(params)
+            )
+            self.logger.log_activity_time("GET REQUEST for PROJECT PARAMETERS", start, request_id=_get_request_id(request))
+            return result
+
+        @app.post(project_level_parameters_path, tags=["Project Metadata"], description=parameters_description)
+        async def get_project_parameters_with_post(
+            request: Request, params: QueryModelForPostProjectParams, user: BaseUser | None = Depends(get_current_user) # type: ignore
+        ) -> arm.ParametersModel:
+            start = time.time()
+            params_model: BaseModel = params
+            payload: dict = await request.json()
+            result = await get_parameters_definition(
+                None, "project", "", PermissionScope.PUBLIC, user, payload, params_model.model_dump()
+            )
+            self.logger.log_activity_time("POST REQUEST for PROJECT PARAMETERS", start, request_id=_get_request_id(request))
+            return result
         
         # Dataset Results API Helpers
         async def get_dataset_results_helper(
-            dataset: str, user: User | None, selections: frozenset[tuple[str, Any]], request_version: int | None
-        ) -> pd.DataFrame:
-            try:
-                return await self.project.dataset(dataset, selections=dict(selections), user=user)
-            except PermissionError as e:
-                raise HTTPException(status_code=status.HTTP_401_UNAUTHORIZED, detail=str(e), headers={"WWW-Authenticate": "Bearer"}) from e
+            dataset: str, user: BaseUser | None, selections: tuple[tuple[str, Any], ...]
+        ) -> DatasetResult:
+            return await self.project.dataset(dataset, selections=dict(selections), user=user)
 
-        settings = self.manifest_cfg.settings
-        dataset_results_cache_size = settings.get(c.DATASETS_CACHE_SIZE_SETTING, settings.get(c.RESULTS_CACHE_SIZE_SETTING, 128))
-        dataset_results_cache_ttl = settings.get(c.DATASETS_CACHE_TTL_SETTING, settings.get(c.RESULTS_CACHE_TTL_SETTING, 60))
+        dataset_results_cache_size = int(self.env_vars.get(c.SQRL_DATASETS_CACHE_SIZE, 128))
+        dataset_results_cache_ttl = int(self.env_vars.get(c.SQRL_DATASETS_CACHE_TTL_MINUTES, 60))
         dataset_results_cache = TTLCache(maxsize=dataset_results_cache_size, ttl=dataset_results_cache_ttl*60)
 
         async def get_dataset_results_cachable(
-            dataset: str, user: User | None, selections: frozenset[tuple[str, Any]], request_version: int | None
-        ) -> pd.DataFrame:
-            return await do_cachable_action(dataset_results_cache, get_dataset_results_helper, dataset, user, selections, request_version)
+            dataset: str, user: BaseUser | None, selections: tuple[tuple[str, Any], ...]
+        ) -> DatasetResult:
+            return await do_cachable_action(dataset_results_cache, get_dataset_results_helper, dataset, user, selections)
         
         async def get_dataset_results_definition(
-            dataset_name: str, user: User | None, headers: Mapping, params: Mapping
+            dataset_name: str, user: BaseUser | None, all_request_params: dict, params: Mapping
         ) -> arm.DatasetResultModel:
+            self._validate_request_params(all_request_params, params)
+
             get_dataset_function = get_dataset_results_helper if self.no_cache else get_dataset_results_cachable
-            selections, request_version = get_selections_and_request_version(params, headers)
-            result = await get_dataset_function(dataset_name, user, selections, request_version)
-            return process_based_on_response_version_header(headers, {
-                0: lambda: arm.DatasetResultModel(**u.df_to_json0(result))
-            })
+            uncached_keys = {"x_verify_params", "x_orientation", "x_select", "x_limit", "x_offset"}
+            selections = get_selections_as_immutable(params, uncached_keys)
+            result = await get_dataset_function(dataset_name, user, selections)
+            
+            orientation = params.get("x_orientation", "records")
+            raw_select = params.get("x_select")
+            select = tuple(raw_select) if raw_select is not None else tuple()
+            limit = params.get("x_limit", 1000)
+            offset = params.get("x_offset", 0)
+            return arm.DatasetResultModel(**result.to_json(orientation, select, limit, offset))
         
         # Dashboard Results API Helpers
         async def get_dashboard_results_helper(
-            dashboard: str, user: User | None, selections: frozenset[tuple[str, Any]], request_version: int | None
+            dashboard: str, user: BaseUser | None, selections: tuple[tuple[str, Any], ...]
         ) -> Dashboard:
-            try:
-                return await self.project.dashboard(dashboard, selections=dict(selections), user=user)
-            except PermissionError as e:
-                raise HTTPException(status_code=status.HTTP_401_UNAUTHORIZED, detail=str(e), headers={"WWW-Authenticate": "Bearer"}) from e
-
-        settings = self.manifest_cfg.settings
-        dashboard_results_cache_size = settings.get(c.DASHBOARDS_CACHE_SIZE_SETTING, 128)
-        dashboard_results_cache_ttl = settings.get(c.DASHBOARDS_CACHE_TTL_SETTING, 60)
+            return await self.project.dashboard(dashboard, selections=dict(selections), user=user)
+        
+        dashboard_results_cache_size = int(self.env_vars.get(c.SQRL_DASHBOARDS_CACHE_SIZE, 128))
+        dashboard_results_cache_ttl = int(self.env_vars.get(c.SQRL_DASHBOARDS_CACHE_TTL_MINUTES, 60))
         dashboard_results_cache = TTLCache(maxsize=dashboard_results_cache_size, ttl=dashboard_results_cache_ttl*60)
 
         async def get_dashboard_results_cachable(
-            dashboard: str, user: User | None, selections: frozenset[tuple[str, Any]], request_version: int | None
+            dashboard: str, user: BaseUser | None, selections: tuple[tuple[str, Any], ...]
         ) -> Dashboard:
-            return await do_cachable_action(dashboard_results_cache, get_dashboard_results_helper, dashboard, user, selections, request_version)
+            return await do_cachable_action(dashboard_results_cache, get_dashboard_results_helper, dashboard, user, selections)
         
         async def get_dashboard_results_definition(
-            dashboard_name: str, user: User | None, headers: Mapping, params: Mapping
+            dashboard_name: str, user: BaseUser | None, all_request_params: dict, params: Mapping
         ) -> Response:
+            self._validate_request_params(all_request_params, params)
+            
             get_dashboard_function = get_dashboard_results_helper if self.no_cache else get_dashboard_results_cachable
-            selections, request_version = get_selections_and_request_version(params, headers)
-            dashboard_obj = await get_dashboard_function(dashboard_name, user, selections, request_version)
+            selections = get_selections_as_immutable(params, uncached_keys={"x_verify_params"})
+            dashboard_obj = await get_dashboard_function(dashboard_name, user, selections)
             if dashboard_obj._format == c.PNG:
                 assert isinstance(dashboard_obj._content, bytes)
                 result = Response(dashboard_obj._content, media_type="image/png")
@@ -408,145 +700,205 @@ class ApiServer:
 
             validate_parameters_list(dataset_config.parameters, "Dataset")
 
-            QueryModelForGet, QueryModelForPost = get_query_models_from_widget_params(dataset_config.parameters)
+            QueryModelForGetParams, QueryModelForPostParams = get_query_models_for_parameters(dataset_config.parameters)
+            QueryModelForGetDataset, QueryModelForPostDataset = get_query_models_for_dataset(dataset_config.parameters)
 
-            @app.get(
-                curr_parameters_path, tags=[f"Dataset '{dataset_name}'"], openapi_extra={"dataset": dataset_name},
-                description=parameters_description, response_class=JSONResponse
-            )
+            @app.get(curr_parameters_path, tags=[f"Dataset '{dataset_name}'"], description=parameters_description, response_class=JSONResponse)
             async def get_dataset_parameters(
-                request: Request, params: QueryModelForGet, user: User | None = Depends(get_current_user) # type: ignore
+                request: Request, params: QueryModelForGetParams, user: BaseUser | None = Depends(get_current_user) # type: ignore
             ) -> arm.ParametersModel:
                 start = time.time()
                 curr_dataset_name = get_dataset_name(request, -2)
                 parameters_list = self.manifest_cfg.datasets[curr_dataset_name].parameters
-                result = await get_parameters_definition(parameters_list, user, request.headers, asdict(params))
+                scope = self.manifest_cfg.datasets[curr_dataset_name].scope
+                result = await get_parameters_definition(
+                    parameters_list, "dataset", curr_dataset_name, scope, user, dict(request.query_params), asdict(params)
+                )
                 self.logger.log_activity_time("GET REQUEST for PARAMETERS", start, request_id=_get_request_id(request))
                 return result
 
-            @app.post(
-                curr_parameters_path, tags=[f"Dataset '{dataset_name}'"], openapi_extra={"dataset": dataset_name},
-                description=parameters_description, response_class=JSONResponse
-            )
+            @app.post(curr_parameters_path, tags=[f"Dataset '{dataset_name}'"], description=parameters_description, response_class=JSONResponse)
             async def get_dataset_parameters_with_post(
-                request: Request, params: QueryModelForPost, user: User | None = Depends(get_current_user) # type: ignore
+                request: Request, params: QueryModelForPostParams, user: BaseUser | None = Depends(get_current_user) # type: ignore
             ) -> arm.ParametersModel:
                 start = time.time()
                 curr_dataset_name = get_dataset_name(request, -2)
                 parameters_list = self.manifest_cfg.datasets[curr_dataset_name].parameters
+                scope = self.manifest_cfg.datasets[curr_dataset_name].scope
                 params: BaseModel = params
-                result = await get_parameters_definition(parameters_list, user, request.headers, params.model_dump())
+                payload: dict = await request.json()
+                result = await get_parameters_definition(
+                    parameters_list, "dataset", curr_dataset_name, scope, user, payload, params.model_dump()
+                )
                 self.logger.log_activity_time("POST REQUEST for PARAMETERS", start, request_id=_get_request_id(request))
                 return result
             
             @app.get(curr_results_path, tags=[f"Dataset '{dataset_name}'"], description=dataset_config.description, response_class=JSONResponse)
             async def get_dataset_results(
-                request: Request, params: QueryModelForGet, user: User | None = Depends(get_current_user) # type: ignore
+                request: Request, params: QueryModelForGetDataset, user: BaseUser | None = Depends(get_current_user) # type: ignore
             ) -> arm.DatasetResultModel:
                 start = time.time()
                 curr_dataset_name = get_dataset_name(request, -1)
-                result = await get_dataset_results_definition(curr_dataset_name, user, request.headers, asdict(params))
+                result = await get_dataset_results_definition(curr_dataset_name, user, dict(request.query_params), asdict(params))
                 self.logger.log_activity_time("GET REQUEST for DATASET RESULTS", start, request_id=_get_request_id(request))
                 return result
             
             @app.post(curr_results_path, tags=[f"Dataset '{dataset_name}'"], description=dataset_config.description, response_class=JSONResponse)
             async def get_dataset_results_with_post(
-                request: Request, params: QueryModelForPost, user: User | None = Depends(get_current_user) # type: ignore
+                request: Request, params: QueryModelForPostDataset, user: BaseUser | None = Depends(get_current_user) # type: ignore
             ) -> arm.DatasetResultModel:
                 start = time.time()
                 curr_dataset_name = get_dataset_name(request, -1)
                 params: BaseModel = params
-                result = await get_dataset_results_definition(curr_dataset_name, user, request.headers, params.model_dump())
+                payload: dict = await request.json()
+                result = await get_dataset_results_definition(curr_dataset_name, user, payload, params.model_dump())
                 self.logger.log_activity_time("POST REQUEST for DATASET RESULTS", start, request_id=_get_request_id(request))
                 return result
         
         # Dashboard Parameters and Results APIs
-        for dashboard_name, dashboard_config in self.manifest_cfg.dashboards.items():
+        for dashboard_name, dashboard in self.dashboards.items():
             dashboard_normalized = u.normalize_name_for_api(dashboard_name)
             curr_parameters_path = dashboard_parameters_path.format(dashboard=dashboard_normalized)
             curr_results_path = dashboard_results_path.format(dashboard=dashboard_normalized)
 
-            validate_parameters_list(dashboard_config.parameters, "Dashboard")
+            validate_parameters_list(dashboard.config.parameters, "Dashboard")
             
-            QueryModelForGet, QueryModelForPost = get_query_models_from_widget_params(dashboard_config.parameters)
+            QueryModelForGetParams, QueryModelForPostParams = get_query_models_for_parameters(dashboard.config.parameters)
+            QueryModelForGetDash, QueryModelForPostDash = get_query_models_for_dashboard(dashboard.config.parameters)
 
             @app.get(curr_parameters_path, tags=[f"Dashboard '{dashboard_name}'"], description=parameters_description, response_class=JSONResponse)
             async def get_dashboard_parameters(
-                request: Request, params: QueryModelForGet, user: User | None = Depends(get_current_user) # type: ignore
+                request: Request, params: QueryModelForGetParams, user: BaseUser | None = Depends(get_current_user) # type: ignore
             ) -> arm.ParametersModel:
                 start = time.time()
                 curr_dashboard_name = get_dashboard_name(request, -2)
-                parameters_list = self.manifest_cfg.dashboards[curr_dashboard_name].parameters
-                result = await get_parameters_definition(parameters_list, user, request.headers, asdict(params))
+                parameters_list = self.dashboards[curr_dashboard_name].config.parameters    
+                scope = self.dashboards[curr_dashboard_name].config.scope
+                result = await get_parameters_definition(
+                    parameters_list, "dashboard", curr_dashboard_name, scope, user, dict(request.query_params), asdict(params)
+                )
                 self.logger.log_activity_time("GET REQUEST for PARAMETERS", start, request_id=_get_request_id(request))
                 return result
 
             @app.post(curr_parameters_path, tags=[f"Dashboard '{dashboard_name}'"], description=parameters_description, response_class=JSONResponse)
             async def get_dashboard_parameters_with_post(
-                request: Request, params: QueryModelForPost, user: User | None = Depends(get_current_user) # type: ignore
+                request: Request, params: QueryModelForPostParams, user: BaseUser | None = Depends(get_current_user) # type: ignore
             ) -> arm.ParametersModel:
                 start = time.time()
                 curr_dashboard_name = get_dashboard_name(request, -2)
-                parameters_list = self.manifest_cfg.dashboards[curr_dashboard_name].parameters
+                parameters_list = self.dashboards[curr_dashboard_name].config.parameters
+                scope = self.dashboards[curr_dashboard_name].config.scope
                 params: BaseModel = params
-                result = await get_parameters_definition(parameters_list, user, request.headers, params.model_dump())
+                payload: dict = await request.json()
+                result = await get_parameters_definition(
+                    parameters_list, "dashboard", curr_dashboard_name, scope, user, payload, params.model_dump()
+                )
                 self.logger.log_activity_time("POST REQUEST for PARAMETERS", start, request_id=_get_request_id(request))
                 return result
             
-            @app.get(curr_results_path, tags=[f"Dashboard '{dashboard_name}'"], description=dashboard_config.description, response_class=Response)
+            @app.get(curr_results_path, tags=[f"Dashboard '{dashboard_name}'"], description=dashboard.config.description, response_class=Response)
             async def get_dashboard_results(
-                request: Request, params: QueryModelForGet, user: User | None = Depends(get_current_user) # type: ignore
+                request: Request, params: QueryModelForGetDash, user: BaseUser | None = Depends(get_current_user) # type: ignore
             ) -> Response:
                 start = time.time()
                 curr_dashboard_name = get_dashboard_name(request, -1)
-                result = await get_dashboard_results_definition(curr_dashboard_name, user, request.headers, asdict(params))
+                result = await get_dashboard_results_definition(curr_dashboard_name, user, dict(request.query_params), asdict(params))
                 self.logger.log_activity_time("GET REQUEST for DASHBOARD RESULTS", start, request_id=_get_request_id(request))
                 return result
 
-            @app.post(curr_results_path, tags=[f"Dashboard '{dashboard_name}'"], description=dashboard_config.description, response_class=Response)
+            @app.post(curr_results_path, tags=[f"Dashboard '{dashboard_name}'"], description=dashboard.config.description, response_class=Response)
             async def get_dashboard_results_with_post(
-                request: Request, params: QueryModelForPost, user: User | None = Depends(get_current_user) # type: ignore
+                request: Request, params: QueryModelForPostDash, user: BaseUser | None = Depends(get_current_user) # type: ignore
             ) -> Response:
                 start = time.time()
                 curr_dashboard_name = get_dashboard_name(request, -1)
                 params: BaseModel = params
-                result = await get_dashboard_results_definition(curr_dashboard_name, user, request.headers, params.model_dump())
+                payload: dict = await request.json()
+                result = await get_dashboard_results_definition(curr_dashboard_name, user, payload, params.model_dump())
                 self.logger.log_activity_time("POST REQUEST for DASHBOARD RESULTS", start, request_id=_get_request_id(request))
                 return result
 
-        # Project Metadata API
-        def get_project_metadata0() -> arm.ProjectModel:
-            return arm.ProjectModel(
-                name=self.manifest_cfg.project_variables.name,
-                label=self.manifest_cfg.project_variables.label,
-                versions=[arm.ProjectVersionModel(
-                    major_version=self.manifest_cfg.project_variables.major_version,
-                    minor_versions=[0],
-                    token_path=token_path,
-                    data_catalog_path=data_catalog_path
-                )]
-            )
+        # Build Project API
+        @app.post(project_metadata_path + '/build', tags=["Data Management"], summary="Build or update the virtual data environment for the project")
+        async def build(user: BaseUser | None = Depends(get_current_user)): # type: ignore
+            if not self.authenticator.can_user_access_scope(user, PermissionScope.PRIVATE):
+                raise InvalidInputError(26, f"User '{user}' does not have permission to build the virtual data environment")
+            await self.project.build(stage_file=True)
+            return Response(status_code=status.HTTP_200_OK)
         
-        @app.get(squirrels_version_path, tags=["Project Metadata"], response_class=JSONResponse)
-        async def get_project_metadata(request: Request) -> arm.ProjectModel:
-            return process_based_on_response_version_header(request.headers, {
-                0: lambda: get_project_metadata0()
-            })
-        
-        # Squirrels Testing UI
-        static_dir = u.Path(os.path.dirname(__file__), c.PACKAGE_DATA_FOLDER, c.ASSETS_FOLDER)
-        app.mount('/'+c.ASSETS_FOLDER, StaticFiles(directory=static_dir), name=c.ASSETS_FOLDER)
-
-        templates_dir = u.Path(os.path.dirname(__file__), c.PACKAGE_DATA_FOLDER, c.TEMPLATES_FOLDER)
-        templates = Jinja2Templates(directory=templates_dir)
+        # Query Models API
+        query_models_path = project_metadata_path + '/query-models'
+        QueryModelForQueryModels, QueryModelForPostQueryModels = get_query_models_for_querying_models()
+
+        async def query_models_helper(
+            sql_query: str, user: BaseUser | None, selections: tuple[tuple[str, Any], ...]
+        ) -> DatasetResult:
+            return await self.project.query_models(sql_query, selections=dict(selections), user=user)
+
+        async def query_models_cachable(
+            sql_query: str, user: BaseUser | None, selections: tuple[tuple[str, Any], ...]
+        ) -> DatasetResult:
+            # Share the same cache for dataset results
+            return await do_cachable_action(dataset_results_cache, query_models_helper, sql_query, user, selections)
+
+        async def query_models_definition(
+            user: BaseUser | None, all_request_params: dict, params: Mapping
+        ) -> arm.DatasetResultModel:
+            self._validate_request_params(all_request_params, params)
 
-        @app.get('/', summary="Get the Squirrels Testing UI", response_class=HTMLResponse)
-        async def get_testing_ui(request: Request):
-            return templates.TemplateResponse('index.html', {
-                'request': request, 'project_metadata_path': squirrels_version_path, 'token_path': token_path
-            })
+            if not self.authenticator.can_user_access_scope(user, PermissionScope.PRIVATE):
+                raise InvalidInputError(27, f"User '{user}' does not have permission to query data models")
+            sql_query = params.get("x_sql_query")
+            if sql_query is None:
+                raise InvalidInputError(203, "SQL query must be provided")
+            
+            query_models_function = query_models_helper if self.no_cache else query_models_cachable
+            uncached_keys = {"x_verify_params", "x_sql_query", "x_orientation", "x_limit", "x_offset"}
+            selections = get_selections_as_immutable(params, uncached_keys)
+            result = await query_models_function(sql_query, user, selections)
+            
+            orientation = params.get("x_orientation", "records")
+            limit = params.get("x_limit", 1000)
+            offset = params.get("x_offset", 0)
+            return arm.DatasetResultModel(**result.to_json(orientation, tuple(), limit, offset))
+
+        @app.get(query_models_path, tags=["Data Management"], response_class=JSONResponse)
+        async def query_models(
+            request: Request, params: QueryModelForQueryModels, user: BaseUser | None = Depends(get_current_user)  # type: ignore
+        ) -> arm.DatasetResultModel:
+            start = time.time()
+            result = await query_models_definition(user, dict(request.query_params), asdict(params))
+            self.logger.log_activity_time("GET REQUEST for QUERY MODELS", start, request_id=_get_request_id(request))
+            return result
+        
+        @app.post(query_models_path, tags=["Data Management"], response_class=JSONResponse)
+        async def query_models_with_post(
+            request: Request, params: QueryModelForPostQueryModels, user: BaseUser | None = Depends(get_current_user)  # type: ignore
+        ) -> arm.DatasetResultModel:
+            start = time.time()
+            params: BaseModel = params
+            payload: dict = await request.json()
+            result = await query_models_definition(user, payload, params.model_dump())
+            self.logger.log_activity_time("POST REQUEST for QUERY MODELS", start, request_id=_get_request_id(request))
+            return result
+        
+        # Add Root Path Redirection to Squirrels Studio
+        full_hostname = f"http://{uvicorn_args.host}:{uvicorn_args.port}"
+        encoded_hostname = urllib.parse.quote(full_hostname, safe="")
+        squirrels_studio_url = f"https://squirrels-analytics.github.io/squirrels-studio/#/login?host={encoded_hostname}&projectName={project_name}&projectVersion={project_version}"
+
+        @app.get("/", include_in_schema=False)
+        async def redirect_to_studio():
+            return RedirectResponse(url=squirrels_studio_url)
         
-        # Run API server
+        # Run the API Server
         import uvicorn
+        
+        print("\nWelcome to the Squirrels Data Application!\n")
+        print(f"- Application UI: {squirrels_studio_url}")
+        print(f"- API Docs (with ReDoc): {full_hostname}{project_metadata_path}/redoc")
+        print(f"- API Docs (with Swagger UI): {full_hostname}{project_metadata_path}/docs")
+        print()
+        
         self.logger.log_activity_time("creating app server", start)
-        uvicorn.run(app, host=uvicorn_args.host, port=uvicorn_args.port)
+        uvicorn.run(app, host=uvicorn_args.host, port=uvicorn_args.port)