squirrels 0.1.0__py3-none-any.whl → 0.6.0.post0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- dateutils/__init__.py +6 -0
- dateutils/_enums.py +25 -0
- squirrels/dateutils.py → dateutils/_implementation.py +409 -380
- dateutils/types.py +6 -0
- squirrels/__init__.py +21 -18
- squirrels/_api_routes/__init__.py +5 -0
- squirrels/_api_routes/auth.py +337 -0
- squirrels/_api_routes/base.py +196 -0
- squirrels/_api_routes/dashboards.py +156 -0
- squirrels/_api_routes/data_management.py +148 -0
- squirrels/_api_routes/datasets.py +220 -0
- squirrels/_api_routes/project.py +289 -0
- squirrels/_api_server.py +552 -134
- squirrels/_arguments/__init__.py +0 -0
- squirrels/_arguments/init_time_args.py +83 -0
- squirrels/_arguments/run_time_args.py +111 -0
- squirrels/_auth.py +777 -0
- squirrels/_command_line.py +239 -107
- squirrels/_compile_prompts.py +147 -0
- squirrels/_connection_set.py +94 -0
- squirrels/_constants.py +141 -64
- squirrels/_dashboards.py +179 -0
- squirrels/_data_sources.py +570 -0
- squirrels/_dataset_types.py +91 -0
- squirrels/_env_vars.py +209 -0
- squirrels/_exceptions.py +29 -0
- squirrels/_http_error_responses.py +52 -0
- squirrels/_initializer.py +319 -110
- squirrels/_logging.py +121 -0
- squirrels/_manifest.py +357 -187
- squirrels/_mcp_server.py +578 -0
- squirrels/_model_builder.py +69 -0
- squirrels/_model_configs.py +74 -0
- squirrels/_model_queries.py +52 -0
- squirrels/_models.py +1201 -0
- squirrels/_package_data/base_project/.env +7 -0
- squirrels/_package_data/base_project/.env.example +44 -0
- squirrels/_package_data/base_project/connections.yml +16 -0
- squirrels/_package_data/base_project/dashboards/dashboard_example.py +40 -0
- squirrels/_package_data/base_project/dashboards/dashboard_example.yml +22 -0
- squirrels/_package_data/base_project/docker/.dockerignore +16 -0
- squirrels/_package_data/base_project/docker/Dockerfile +16 -0
- squirrels/_package_data/base_project/docker/compose.yml +7 -0
- squirrels/_package_data/base_project/duckdb_init.sql +10 -0
- squirrels/_package_data/base_project/gitignore +13 -0
- squirrels/_package_data/base_project/macros/macros_example.sql +17 -0
- squirrels/_package_data/base_project/models/builds/build_example.py +26 -0
- squirrels/_package_data/base_project/models/builds/build_example.sql +16 -0
- squirrels/_package_data/base_project/models/builds/build_example.yml +57 -0
- squirrels/_package_data/base_project/models/dbviews/dbview_example.sql +17 -0
- squirrels/_package_data/base_project/models/dbviews/dbview_example.yml +32 -0
- squirrels/_package_data/base_project/models/federates/federate_example.py +51 -0
- squirrels/_package_data/base_project/models/federates/federate_example.sql +21 -0
- squirrels/_package_data/base_project/models/federates/federate_example.yml +65 -0
- squirrels/_package_data/base_project/models/sources.yml +38 -0
- squirrels/_package_data/base_project/parameters.yml +142 -0
- squirrels/_package_data/base_project/pyconfigs/connections.py +19 -0
- squirrels/_package_data/base_project/pyconfigs/context.py +96 -0
- squirrels/_package_data/base_project/pyconfigs/parameters.py +141 -0
- squirrels/_package_data/base_project/pyconfigs/user.py +56 -0
- squirrels/_package_data/base_project/resources/expenses.db +0 -0
- squirrels/_package_data/base_project/resources/public/.gitkeep +0 -0
- squirrels/_package_data/base_project/resources/weather.db +0 -0
- squirrels/_package_data/base_project/seeds/seed_categories.csv +6 -0
- squirrels/_package_data/base_project/seeds/seed_categories.yml +15 -0
- squirrels/_package_data/base_project/seeds/seed_subcategories.csv +15 -0
- squirrels/_package_data/base_project/seeds/seed_subcategories.yml +21 -0
- squirrels/_package_data/base_project/squirrels.yml.j2 +61 -0
- squirrels/_package_data/base_project/tmp/.gitignore +2 -0
- squirrels/_package_data/templates/login_successful.html +53 -0
- squirrels/_package_data/templates/squirrels_studio.html +22 -0
- squirrels/_package_loader.py +29 -0
- squirrels/_parameter_configs.py +592 -0
- squirrels/_parameter_options.py +348 -0
- squirrels/_parameter_sets.py +207 -0
- squirrels/_parameters.py +1703 -0
- squirrels/_project.py +796 -0
- squirrels/_py_module.py +122 -0
- squirrels/_request_context.py +33 -0
- squirrels/_schemas/__init__.py +0 -0
- squirrels/_schemas/auth_models.py +83 -0
- squirrels/_schemas/query_param_models.py +70 -0
- squirrels/_schemas/request_models.py +26 -0
- squirrels/_schemas/response_models.py +286 -0
- squirrels/_seeds.py +97 -0
- squirrels/_sources.py +112 -0
- squirrels/_utils.py +540 -149
- squirrels/_version.py +1 -3
- squirrels/arguments.py +7 -0
- squirrels/auth.py +4 -0
- squirrels/connections.py +3 -0
- squirrels/dashboards.py +3 -0
- squirrels/data_sources.py +14 -282
- squirrels/parameter_options.py +13 -189
- squirrels/parameters.py +14 -801
- squirrels/types.py +18 -0
- squirrels-0.6.0.post0.dist-info/METADATA +148 -0
- squirrels-0.6.0.post0.dist-info/RECORD +101 -0
- {squirrels-0.1.0.dist-info → squirrels-0.6.0.post0.dist-info}/WHEEL +1 -2
- {squirrels-0.1.0.dist-info → squirrels-0.6.0.post0.dist-info}/entry_points.txt +1 -0
- squirrels-0.6.0.post0.dist-info/licenses/LICENSE +201 -0
- squirrels/_credentials_manager.py +0 -87
- squirrels/_module_loader.py +0 -37
- squirrels/_parameter_set.py +0 -151
- squirrels/_renderer.py +0 -286
- squirrels/_timed_imports.py +0 -37
- squirrels/connection_set.py +0 -126
- squirrels/package_data/base_project/.gitignore +0 -4
- squirrels/package_data/base_project/connections.py +0 -21
- squirrels/package_data/base_project/database/sample_database.db +0 -0
- squirrels/package_data/base_project/database/seattle_weather.db +0 -0
- squirrels/package_data/base_project/datasets/sample_dataset/context.py +0 -8
- squirrels/package_data/base_project/datasets/sample_dataset/database_view1.py +0 -23
- squirrels/package_data/base_project/datasets/sample_dataset/database_view1.sql.j2 +0 -7
- squirrels/package_data/base_project/datasets/sample_dataset/final_view.py +0 -10
- squirrels/package_data/base_project/datasets/sample_dataset/final_view.sql.j2 +0 -2
- squirrels/package_data/base_project/datasets/sample_dataset/parameters.py +0 -30
- squirrels/package_data/base_project/datasets/sample_dataset/selections.cfg +0 -6
- squirrels/package_data/base_project/squirrels.yaml +0 -26
- squirrels/package_data/static/favicon.ico +0 -0
- squirrels/package_data/static/script.js +0 -234
- squirrels/package_data/static/style.css +0 -110
- squirrels/package_data/templates/index.html +0 -32
- squirrels-0.1.0.dist-info/LICENSE +0 -22
- squirrels-0.1.0.dist-info/METADATA +0 -67
- squirrels-0.1.0.dist-info/RECORD +0 -40
- squirrels-0.1.0.dist-info/top_level.txt +0 -1
squirrels/_api_server.py
CHANGED
|
@@ -1,134 +1,552 @@
|
|
|
1
|
-
from typing import
|
|
2
|
-
from
|
|
3
|
-
from fastapi
|
|
4
|
-
from fastapi.responses import HTMLResponse,
|
|
5
|
-
from fastapi.
|
|
6
|
-
from fastapi.staticfiles import StaticFiles
|
|
7
|
-
from
|
|
8
|
-
import
|
|
9
|
-
|
|
10
|
-
from
|
|
11
|
-
from
|
|
12
|
-
from
|
|
13
|
-
from
|
|
14
|
-
from
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
#
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
1
|
+
from typing import TYPE_CHECKING
|
|
2
|
+
from dataclasses import dataclass
|
|
3
|
+
from fastapi import FastAPI, Request, status
|
|
4
|
+
from fastapi.responses import JSONResponse, HTMLResponse, RedirectResponse, PlainTextResponse
|
|
5
|
+
from fastapi.security import HTTPBearer
|
|
6
|
+
from fastapi.staticfiles import StaticFiles
|
|
7
|
+
from fastapi.templating import Jinja2Templates
|
|
8
|
+
from starlette.middleware.base import BaseHTTPMiddleware
|
|
9
|
+
from starlette.responses import Response as StarletteResponse
|
|
10
|
+
from starlette.types import ASGIApp
|
|
11
|
+
from contextlib import asynccontextmanager
|
|
12
|
+
from argparse import Namespace
|
|
13
|
+
from pathlib import Path
|
|
14
|
+
from starlette.middleware.sessions import SessionMiddleware
|
|
15
|
+
import io, time, mimetypes, traceback, asyncio
|
|
16
|
+
|
|
17
|
+
from . import _constants as c, _utils as u, _parameter_sets as ps
|
|
18
|
+
from ._schemas import response_models as rm
|
|
19
|
+
from ._exceptions import InvalidInputError, ConfigurationError, FileExecutionError
|
|
20
|
+
from ._http_error_responses import invalid_input_error_to_json_response
|
|
21
|
+
from ._manifest import AuthStrategy, AuthType
|
|
22
|
+
from ._request_context import set_request_id
|
|
23
|
+
from ._mcp_server import McpServerBuilder
|
|
24
|
+
|
|
25
|
+
# Import route modules
|
|
26
|
+
from ._api_routes.base import RouteBase
|
|
27
|
+
from ._api_routes.auth import AuthRoutes
|
|
28
|
+
from ._api_routes.project import ProjectRoutes
|
|
29
|
+
from ._api_routes.datasets import DatasetRoutes
|
|
30
|
+
from ._api_routes.dashboards import DashboardRoutes
|
|
31
|
+
from ._api_routes.data_management import DataManagementRoutes
|
|
32
|
+
|
|
33
|
+
if TYPE_CHECKING:
|
|
34
|
+
from contextlib import _AsyncGeneratorContextManager
|
|
35
|
+
from ._project import SquirrelsProject
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
mimetypes.add_type('application/javascript', '.js')
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
class SmartCORSMiddleware(BaseHTTPMiddleware):
|
|
42
|
+
"""
|
|
43
|
+
Custom CORS middleware that allows specific origins to use credentials
|
|
44
|
+
while still allowing all other origins without credentials.
|
|
45
|
+
"""
|
|
46
|
+
|
|
47
|
+
def __init__(self, app, allowed_credential_origins: list[str], configurables_as_headers: list[str]):
|
|
48
|
+
super().__init__(app)
|
|
49
|
+
|
|
50
|
+
allowed_predefined_headers = ["Authorization", "Content-Type", "x-api-key"]
|
|
51
|
+
|
|
52
|
+
self.allowed_credential_origins = allowed_credential_origins
|
|
53
|
+
self.allowed_request_headers = ",".join(allowed_predefined_headers + configurables_as_headers)
|
|
54
|
+
|
|
55
|
+
async def dispatch(self, request: Request, call_next):
|
|
56
|
+
origin = request.headers.get("origin")
|
|
57
|
+
|
|
58
|
+
# Handle preflight requests
|
|
59
|
+
if request.method == "OPTIONS":
|
|
60
|
+
response = StarletteResponse(status_code=200)
|
|
61
|
+
response.headers["Access-Control-Allow-Methods"] = "GET, POST, PUT, DELETE, OPTIONS"
|
|
62
|
+
response.headers["Access-Control-Allow-Headers"] = self.allowed_request_headers
|
|
63
|
+
|
|
64
|
+
else:
|
|
65
|
+
# Call the next middleware/route
|
|
66
|
+
response: StarletteResponse = await call_next(request)
|
|
67
|
+
|
|
68
|
+
# Always expose the Applied-Username header
|
|
69
|
+
response.headers["Access-Control-Expose-Headers"] = "Applied-Username"
|
|
70
|
+
|
|
71
|
+
if origin:
|
|
72
|
+
request_origin = f"{request.url.scheme}://{request.url.netloc}"
|
|
73
|
+
# Check if this origin is in the whitelist or if origin matches the host origin
|
|
74
|
+
if origin == request_origin or origin in self.allowed_credential_origins:
|
|
75
|
+
response.headers["Access-Control-Allow-Origin"] = origin
|
|
76
|
+
response.headers["Access-Control-Allow-Credentials"] = "true"
|
|
77
|
+
else:
|
|
78
|
+
# Allow all other origins but without credentials / cookies
|
|
79
|
+
response.headers["Access-Control-Allow-Origin"] = "*"
|
|
80
|
+
else:
|
|
81
|
+
# No origin header (probably a non-browser request)
|
|
82
|
+
response.headers["Access-Control-Allow-Origin"] = "*"
|
|
83
|
+
|
|
84
|
+
return response
|
|
85
|
+
|
|
86
|
+
|
|
87
|
+
@dataclass
|
|
88
|
+
class FastAPIComponents:
|
|
89
|
+
"""
|
|
90
|
+
HTTP server components to mount the Squirrels project into an existing FastAPI application.
|
|
91
|
+
|
|
92
|
+
Properties:
|
|
93
|
+
mount_path: The mount path for the Squirrels project.
|
|
94
|
+
lifespan: The lifespan context manager for the Squirrels project.
|
|
95
|
+
fastapi_app: The FastAPI app for the Squirrels project.
|
|
96
|
+
"""
|
|
97
|
+
mount_path: str
|
|
98
|
+
lifespan: "_AsyncGeneratorContextManager"
|
|
99
|
+
fastapi_app: "FastAPI"
|
|
100
|
+
|
|
101
|
+
|
|
102
|
+
class ApiServer:
|
|
103
|
+
def __init__(self, no_cache: bool, project: "SquirrelsProject") -> None:
|
|
104
|
+
"""
|
|
105
|
+
Constructor for ApiServer
|
|
106
|
+
|
|
107
|
+
Arguments:
|
|
108
|
+
no_cache (bool): Whether to disable caching
|
|
109
|
+
"""
|
|
110
|
+
self.project = project
|
|
111
|
+
self.logger = project._logger
|
|
112
|
+
self.env_vars = project._env_vars
|
|
113
|
+
self.manifest_cfg = project._manifest_cfg
|
|
114
|
+
self.seeds = project._seeds
|
|
115
|
+
self.conn_set = project._conn_set
|
|
116
|
+
self.param_cfg_set = project._param_cfg_set
|
|
117
|
+
self.dashboards = project._dashboards
|
|
118
|
+
|
|
119
|
+
# Initialize route modules
|
|
120
|
+
get_bearer_token = HTTPBearer(auto_error=False)
|
|
121
|
+
# self.oauth2_routes = OAuth2Routes(get_bearer_token, project, no_cache)
|
|
122
|
+
self.auth_routes = AuthRoutes(get_bearer_token, project, no_cache)
|
|
123
|
+
self.project_routes = ProjectRoutes(get_bearer_token, project, no_cache)
|
|
124
|
+
self.dataset_routes = DatasetRoutes(get_bearer_token, project, no_cache)
|
|
125
|
+
self.dashboard_routes = DashboardRoutes(get_bearer_token, project, no_cache)
|
|
126
|
+
self.data_management_routes = DataManagementRoutes(get_bearer_token, project, no_cache)
|
|
127
|
+
|
|
128
|
+
self._mcp_builder: McpServerBuilder | None = None
|
|
129
|
+
self._mcp_app: ASGIApp | None = None
|
|
130
|
+
|
|
131
|
+
|
|
132
|
+
async def _refresh_datasource_params(self) -> None:
|
|
133
|
+
"""
|
|
134
|
+
Background task to periodically refresh datasource parameter options.
|
|
135
|
+
Runs every N minutes as configured by SQRL_PARAMETERS__DATASOURCE_REFRESH_MINUTES (default: 60).
|
|
136
|
+
"""
|
|
137
|
+
refresh_minutes = self.env_vars.parameters_datasource_refresh_minutes
|
|
138
|
+
if refresh_minutes <= 0:
|
|
139
|
+
self.logger.info(f"The value of {c.SQRL_PARAMETERS_DATASOURCE_REFRESH_MINUTES} is: {refresh_minutes} minutes")
|
|
140
|
+
self.logger.info(f"Datasource parameter refresh is disabled since the refresh interval is not positive.")
|
|
141
|
+
return
|
|
142
|
+
|
|
143
|
+
refresh_seconds = refresh_minutes * 60
|
|
144
|
+
self.logger.info(f"Starting datasource parameter refresh background task (every {refresh_minutes} minutes)")
|
|
145
|
+
|
|
146
|
+
default_conn_name = self.env_vars.connections_default_name_used
|
|
147
|
+
while True:
|
|
148
|
+
try:
|
|
149
|
+
await asyncio.sleep(refresh_seconds)
|
|
150
|
+
self.logger.info("Refreshing datasource parameter options...")
|
|
151
|
+
|
|
152
|
+
# Fetch fresh dataframes from datasources in a thread pool to avoid blocking
|
|
153
|
+
loop = asyncio.get_running_loop()
|
|
154
|
+
df_dict = await loop.run_in_executor(
|
|
155
|
+
None,
|
|
156
|
+
ps.ParameterConfigsSetIO._get_df_dict_from_data_sources,
|
|
157
|
+
self.param_cfg_set,
|
|
158
|
+
default_conn_name,
|
|
159
|
+
self.seeds,
|
|
160
|
+
self.conn_set,
|
|
161
|
+
self.project._vdl_catalog_db_path
|
|
162
|
+
)
|
|
163
|
+
|
|
164
|
+
# Re-convert datasource parameters with fresh data
|
|
165
|
+
self.param_cfg_set._post_process_params(df_dict)
|
|
166
|
+
|
|
167
|
+
self.logger.info("Successfully refreshed datasource parameter options")
|
|
168
|
+
except asyncio.CancelledError:
|
|
169
|
+
self.logger.info("Datasource parameter refresh task cancelled")
|
|
170
|
+
break
|
|
171
|
+
except Exception as e:
|
|
172
|
+
self.logger.error(f"Error refreshing datasource parameter options: {e}", exc_info=True)
|
|
173
|
+
# Continue the loop even if there's an error
|
|
174
|
+
|
|
175
|
+
|
|
176
|
+
def _get_tags_metadata(self) -> list[dict]:
|
|
177
|
+
tags_metadata = [
|
|
178
|
+
{
|
|
179
|
+
"name": "Project Metadata",
|
|
180
|
+
"description": "Get information on project such as name, version, and other API endpoints",
|
|
181
|
+
},
|
|
182
|
+
{
|
|
183
|
+
"name": "Data Management",
|
|
184
|
+
"description": "Actions to update the data components of the project",
|
|
185
|
+
}
|
|
186
|
+
]
|
|
187
|
+
|
|
188
|
+
for dataset_name in self.manifest_cfg.datasets:
|
|
189
|
+
tags_metadata.append({
|
|
190
|
+
"name": f"Dataset '{dataset_name}'",
|
|
191
|
+
"description": f"Get parameters or results for dataset '{dataset_name}'",
|
|
192
|
+
})
|
|
193
|
+
|
|
194
|
+
for dashboard_name in self.dashboards:
|
|
195
|
+
tags_metadata.append({
|
|
196
|
+
"name": f"Dashboard '{dashboard_name}'",
|
|
197
|
+
"description": f"Get parameters or results for dashboard '{dashboard_name}'",
|
|
198
|
+
})
|
|
199
|
+
|
|
200
|
+
tags_metadata.extend([
|
|
201
|
+
{
|
|
202
|
+
"name": "Authentication",
|
|
203
|
+
"description": "Submit authentication credentials and authorize with a session cookie",
|
|
204
|
+
},
|
|
205
|
+
{
|
|
206
|
+
"name": "User Management",
|
|
207
|
+
"description": "Manage users and their attributes",
|
|
208
|
+
}
|
|
209
|
+
])
|
|
210
|
+
return tags_metadata
|
|
211
|
+
|
|
212
|
+
|
|
213
|
+
def _print_banner(self, mount_path: str, host: str | None, port: int | None, is_standalone_mode: bool) -> None:
|
|
214
|
+
"""
|
|
215
|
+
Print the welcome banner with information about the running server.
|
|
216
|
+
"""
|
|
217
|
+
full_hostname = f"http://{host}:{port}" if host and port else ""
|
|
218
|
+
mount_path_stripped = mount_path.rstrip("/")
|
|
219
|
+
show_multiple_options = is_standalone_mode and mount_path_stripped != ""
|
|
220
|
+
|
|
221
|
+
banner_width = 80
|
|
222
|
+
|
|
223
|
+
print()
|
|
224
|
+
print("═" * banner_width)
|
|
225
|
+
print("👋 WELCOME TO SQUIRRELS!".center(banner_width))
|
|
226
|
+
print("═" * banner_width)
|
|
227
|
+
print()
|
|
228
|
+
print(" 🖥️ Application UI")
|
|
229
|
+
print(f" └─ Squirrels Studio: {full_hostname}{mount_path_stripped}/studio")
|
|
230
|
+
if show_multiple_options:
|
|
231
|
+
print(f" ├─ The root path also redirects to Squirrels Studio: {full_hostname}/")
|
|
232
|
+
print( " ├─ This requires an internet connection to load the JS and CSS files")
|
|
233
|
+
print( f" └─ Automatically uses mount path: {mount_path_stripped}")
|
|
234
|
+
print()
|
|
235
|
+
print(" 🔌 MCP Server URLs")
|
|
236
|
+
if show_multiple_options:
|
|
237
|
+
print(f" ├─ Option 1: {full_hostname}{mount_path_stripped}/mcp")
|
|
238
|
+
print(f" └─ Option 2: {full_hostname}/mcp")
|
|
239
|
+
else:
|
|
240
|
+
print(f" └─ Project MCP: {full_hostname}{mount_path_stripped}/mcp")
|
|
241
|
+
print()
|
|
242
|
+
print(" 📖 API Documentation (for the latest version of API contract)")
|
|
243
|
+
print(f" ├─ Swagger UI: {full_hostname}{mount_path_stripped}{c.LATEST_API_VERSION_MOUNT_PATH}/docs")
|
|
244
|
+
print(f" ├─ ReDoc UI: {full_hostname}{mount_path_stripped}{c.LATEST_API_VERSION_MOUNT_PATH}/redoc")
|
|
245
|
+
print(f" └─ OpenAPI Spec: {full_hostname}{mount_path_stripped}{c.LATEST_API_VERSION_MOUNT_PATH}/openapi.json")
|
|
246
|
+
print()
|
|
247
|
+
print(f" To explore all HTTP endpoints, see: {full_hostname}{mount_path_stripped}/docs")
|
|
248
|
+
print()
|
|
249
|
+
print("─" * banner_width)
|
|
250
|
+
print("✨ Server is running! Press CTRL+C to stop.".center(banner_width))
|
|
251
|
+
print("─" * banner_width)
|
|
252
|
+
print()
|
|
253
|
+
|
|
254
|
+
|
|
255
|
+
def get_lifespan(
|
|
256
|
+
self, mount_path: str, host: str | None, port: int | None, is_standalone_mode: bool
|
|
257
|
+
) -> "_AsyncGeneratorContextManager":
|
|
258
|
+
"""
|
|
259
|
+
Get the lifespan context manager for the Squirrels project.
|
|
260
|
+
"""
|
|
261
|
+
@asynccontextmanager
|
|
262
|
+
async def lifespan(app: FastAPI | None = None):
|
|
263
|
+
"""App lifespan that includes MCP server lifecycle and background tasks."""
|
|
264
|
+
self._print_banner(mount_path, host, port, is_standalone_mode)
|
|
265
|
+
|
|
266
|
+
refresh_datasource_task = asyncio.create_task(self._refresh_datasource_params())
|
|
267
|
+
|
|
268
|
+
if self._mcp_builder:
|
|
269
|
+
async with self._mcp_builder.lifespan():
|
|
270
|
+
yield
|
|
271
|
+
else:
|
|
272
|
+
yield
|
|
273
|
+
|
|
274
|
+
refresh_datasource_task.cancel()
|
|
275
|
+
|
|
276
|
+
return lifespan
|
|
277
|
+
|
|
278
|
+
|
|
279
|
+
def create_app(
|
|
280
|
+
self,
|
|
281
|
+
lifespan: "_AsyncGeneratorContextManager",
|
|
282
|
+
*,
|
|
283
|
+
mount_path: str = ""
|
|
284
|
+
) -> FastAPI:
|
|
285
|
+
"""
|
|
286
|
+
Create the FastAPI app for the Squirrels project.
|
|
287
|
+
"""
|
|
288
|
+
start = time.time()
|
|
289
|
+
|
|
290
|
+
project_name = self.manifest_cfg.project_variables.name
|
|
291
|
+
project_label = self.manifest_cfg.project_variables.label
|
|
292
|
+
|
|
293
|
+
param_fields = self.param_cfg_set.get_all_api_field_info()
|
|
294
|
+
tags_metadata = self._get_tags_metadata()
|
|
295
|
+
|
|
296
|
+
mount_path_stripped = mount_path.rstrip("/")
|
|
297
|
+
api_v0_mount_path = "/api/0"
|
|
298
|
+
|
|
299
|
+
app = FastAPI(
|
|
300
|
+
title=f"Squirrels for '{project_label}'",
|
|
301
|
+
lifespan=lifespan
|
|
302
|
+
)
|
|
303
|
+
|
|
304
|
+
api_v0_app = FastAPI(
|
|
305
|
+
title=f"Squirrels APIs for '{project_label}'", openapi_tags=tags_metadata,
|
|
306
|
+
description="For specifying parameter selections to dataset APIs, you can choose between using query parameters with the GET method or using request body with the POST method",
|
|
307
|
+
openapi_url="/openapi.json",
|
|
308
|
+
docs_url="/docs",
|
|
309
|
+
redoc_url="/redoc"
|
|
310
|
+
)
|
|
311
|
+
|
|
312
|
+
api_v0_app.add_middleware(SessionMiddleware, secret_key=self.env_vars.secret_key, max_age=None, same_site="none", https_only=True)
|
|
313
|
+
|
|
314
|
+
async def _log_request_run(request: Request) -> None:
|
|
315
|
+
try:
|
|
316
|
+
body = await request.json()
|
|
317
|
+
except Exception:
|
|
318
|
+
body = None # Non-JSON payloads may contain sensitive information, so we don't log them
|
|
319
|
+
|
|
320
|
+
partial_headers: dict[str, str] = {}
|
|
321
|
+
for header in request.headers.keys():
|
|
322
|
+
if header.startswith("x-") and header not in ["x-api-key"]:
|
|
323
|
+
partial_headers[header] = request.headers[header]
|
|
324
|
+
|
|
325
|
+
path, params = request.url.path, dict(request.query_params)
|
|
326
|
+
path_with_params = f"{path}?{request.query_params}" if len(params) > 0 else path
|
|
327
|
+
data = {"request_method": request.method, "request_path": path, "request_params": params, "request_body": body, "partial_headers": partial_headers}
|
|
328
|
+
self.logger.info(f'Running request: {request.method} {path_with_params}', data=data)
|
|
329
|
+
|
|
330
|
+
@api_v0_app.middleware("http")
|
|
331
|
+
async def catch_exceptions_middleware(request: Request, call_next):
|
|
332
|
+
# Generate and set request ID for this request
|
|
333
|
+
request_id = set_request_id()
|
|
334
|
+
|
|
335
|
+
buffer = io.StringIO()
|
|
336
|
+
try:
|
|
337
|
+
await _log_request_run(request)
|
|
338
|
+
response = await call_next(request)
|
|
339
|
+
except InvalidInputError as exc:
|
|
340
|
+
message = str(exc)
|
|
341
|
+
self.logger.error(message)
|
|
342
|
+
strip_path_suffix = f"{mount_path_stripped}{api_v0_mount_path}"
|
|
343
|
+
response = invalid_input_error_to_json_response(
|
|
344
|
+
request,
|
|
345
|
+
exc,
|
|
346
|
+
oauth_resource_metadata_path="/.well-known/oauth-protected-resource",
|
|
347
|
+
strip_path_suffix=strip_path_suffix,
|
|
348
|
+
)
|
|
349
|
+
except FileExecutionError as exc:
|
|
350
|
+
traceback.print_exception(exc.error, file=buffer)
|
|
351
|
+
buffer.write(str(exc))
|
|
352
|
+
response = JSONResponse(
|
|
353
|
+
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, content={"message": f"An unexpected server error occurred", "blame": "Squirrels project"}
|
|
354
|
+
)
|
|
355
|
+
except ConfigurationError as exc:
|
|
356
|
+
traceback.print_exc(file=buffer)
|
|
357
|
+
response = JSONResponse(
|
|
358
|
+
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, content={"message": f"An unexpected server error occurred", "blame": "Squirrels project"}
|
|
359
|
+
)
|
|
360
|
+
except Exception as exc:
|
|
361
|
+
traceback.print_exc(file=buffer)
|
|
362
|
+
response = JSONResponse(
|
|
363
|
+
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, content={"message": f"An unexpected server error occurred", "blame": "Squirrels framework"}
|
|
364
|
+
)
|
|
365
|
+
|
|
366
|
+
err_msg = buffer.getvalue()
|
|
367
|
+
if err_msg:
|
|
368
|
+
self.logger.error(err_msg)
|
|
369
|
+
|
|
370
|
+
# Add request ID to response header
|
|
371
|
+
response.headers["X-Request-ID"] = request_id
|
|
372
|
+
|
|
373
|
+
return response
|
|
374
|
+
|
|
375
|
+
# Configure CORS with smart credential handling
|
|
376
|
+
allowed_credential_origins = self.env_vars.auth_credential_origins
|
|
377
|
+
|
|
378
|
+
configurables_as_headers = []
|
|
379
|
+
for name in self.manifest_cfg.configurables.keys():
|
|
380
|
+
configurables_as_headers.append(f"x-config-{name}") # underscore version
|
|
381
|
+
configurables_as_headers.append(f"x-config-{u.normalize_name_for_api(name)}") # dash version
|
|
382
|
+
|
|
383
|
+
api_v0_app.add_middleware(SmartCORSMiddleware, allowed_credential_origins=allowed_credential_origins, configurables_as_headers=configurables_as_headers)
|
|
384
|
+
|
|
385
|
+
# Setup route modules for the v0 API
|
|
386
|
+
get_parameters_definition = self.project_routes.setup_routes(api_v0_app, param_fields)
|
|
387
|
+
self.data_management_routes.setup_routes(api_v0_app, param_fields)
|
|
388
|
+
self.dataset_routes.setup_routes(api_v0_app, param_fields, get_parameters_definition)
|
|
389
|
+
self.dashboard_routes.setup_routes(api_v0_app, param_fields, get_parameters_definition)
|
|
390
|
+
# self.oauth2_routes.setup_routes(api_v0_app)
|
|
391
|
+
self.auth_routes.setup_routes(api_v0_app)
|
|
392
|
+
|
|
393
|
+
app.mount(api_v0_mount_path, api_v0_app)
|
|
394
|
+
|
|
395
|
+
@app.get("/health", summary="Health check endpoint")
|
|
396
|
+
async def health() -> PlainTextResponse:
|
|
397
|
+
return PlainTextResponse(status_code=200, content="OK")
|
|
398
|
+
|
|
399
|
+
# Mount static files from the public directories if they exist
|
|
400
|
+
# This allows users to serve public-facing static assets (images, CSS, JS, etc.) with HTTP requests
|
|
401
|
+
public_dirs = ["public"]
|
|
402
|
+
for public_dir in public_dirs:
|
|
403
|
+
static_dir = Path(self.project._project_path) / "resources" / public_dir
|
|
404
|
+
if static_dir.exists() and static_dir.is_dir():
|
|
405
|
+
app.mount(f"/{public_dir}", StaticFiles(directory=str(static_dir)), name=public_dir)
|
|
406
|
+
self.logger.info(f"Mounted static files from: {str(static_dir)}")
|
|
407
|
+
|
|
408
|
+
# Build the MCP server after routes are set up
|
|
409
|
+
enforce_mcp_oauth = (
|
|
410
|
+
self.manifest_cfg.project_variables.auth_strategy == AuthStrategy.EXTERNAL
|
|
411
|
+
and self.manifest_cfg.project_variables.auth_type == AuthType.REQUIRED
|
|
412
|
+
)
|
|
413
|
+
self._mcp_builder = McpServerBuilder(
|
|
414
|
+
project_name=project_name,
|
|
415
|
+
project_label=project_label,
|
|
416
|
+
max_rows_for_ai=self.env_vars.datasets_max_rows_for_ai,
|
|
417
|
+
get_user_from_headers=self.project_routes.get_user_from_headers,
|
|
418
|
+
get_data_catalog_for_mcp=self.project_routes._get_data_catalog_for_mcp,
|
|
419
|
+
get_dataset_parameters_for_mcp=self.dataset_routes._get_dataset_parameters_for_mcp,
|
|
420
|
+
get_dataset_results_for_mcp=self.dataset_routes._get_dataset_results_for_mcp,
|
|
421
|
+
enforce_oauth_bearer=enforce_mcp_oauth,
|
|
422
|
+
oauth_resource_metadata_path="/.well-known/oauth-protected-resource",
|
|
423
|
+
www_authenticate_strip_path_suffix=f"{mount_path_stripped}/mcp",
|
|
424
|
+
)
|
|
425
|
+
self._mcp_app = self._mcp_builder.get_asgi_app()
|
|
426
|
+
|
|
427
|
+
# Mount MCP server
|
|
428
|
+
app.add_route("/mcp", self._mcp_app, methods=["GET", "POST"])
|
|
429
|
+
|
|
430
|
+
# Get API versions and other endpoints
|
|
431
|
+
@app.get("/", summary="Explore all HTTP endpoints")
|
|
432
|
+
async def explore_http_endpoints(request: Request) -> rm.ExploreEndpointsModel:
|
|
433
|
+
_, root_path = RouteBase._get_base_url_for_current_app(request)
|
|
434
|
+
return rm.ExploreEndpointsModel(
|
|
435
|
+
health_url=root_path + "/health",
|
|
436
|
+
api_versions={
|
|
437
|
+
"0": rm.APIVersionMetadataModel(
|
|
438
|
+
project_metadata_url=root_path + api_v0_mount_path + "/",
|
|
439
|
+
documentation_routes=rm.DocumentationRoutesModel(
|
|
440
|
+
swagger_url=root_path + api_v0_mount_path + "/docs",
|
|
441
|
+
redoc_url=root_path + api_v0_mount_path + "/redoc",
|
|
442
|
+
openapi_url=root_path + api_v0_mount_path + "/openapi.json"
|
|
443
|
+
)
|
|
444
|
+
)
|
|
445
|
+
},
|
|
446
|
+
documentation_routes=rm.DocumentationRoutesModel(
|
|
447
|
+
swagger_url=root_path + "/docs",
|
|
448
|
+
redoc_url=root_path + "/redoc",
|
|
449
|
+
openapi_url=root_path + "/openapi.json"
|
|
450
|
+
),
|
|
451
|
+
mcp_server_url=root_path + "/mcp",
|
|
452
|
+
studio_url=root_path + "/studio",
|
|
453
|
+
)
|
|
454
|
+
|
|
455
|
+
# Add Squirrels Studio
|
|
456
|
+
templates = Jinja2Templates(directory=str(Path(__file__).parent / "_package_data" / "templates"))
|
|
457
|
+
|
|
458
|
+
@app.get("/studio", include_in_schema=False)
|
|
459
|
+
async def squirrels_studio(request: Request):
|
|
460
|
+
sqrl_studio_base_url = self.env_vars.studio_base_url
|
|
461
|
+
|
|
462
|
+
# IMPORTANT: avoid `request.url_for("explore_http_endpoints")` here.
|
|
463
|
+
# When multiple Squirrels FastAPI apps are mounted into a root app, that route name
|
|
464
|
+
# can become ambiguous and resolve to the wrong mounted app. `request.base_url`
|
|
465
|
+
# is derived from the current request scope (including `root_path`), so it always
|
|
466
|
+
# points at the correct mounted Squirrels server instance.
|
|
467
|
+
_, mount_path = RouteBase._get_base_url_for_current_app(request)
|
|
468
|
+
|
|
469
|
+
context = {
|
|
470
|
+
"sqrl_studio_base_url": sqrl_studio_base_url,
|
|
471
|
+
"mount_path": mount_path,
|
|
472
|
+
}
|
|
473
|
+
template = templates.get_template("squirrels_studio.html")
|
|
474
|
+
return HTMLResponse(content=template.render(context))
|
|
475
|
+
|
|
476
|
+
self.logger.log_activity_time("creating app server", start)
|
|
477
|
+
return app
|
|
478
|
+
|
|
479
|
+
def get_fastapi_components(
|
|
480
|
+
self, host: str, port: int, *,
|
|
481
|
+
mount_path_format: str = "/analytics/{project_name}/v{project_version}",
|
|
482
|
+
is_standalone_mode: bool = False
|
|
483
|
+
) -> FastAPIComponents:
|
|
484
|
+
"""
|
|
485
|
+
Get the FastAPI components for the Squirrels project including mount path, lifespan, and FastAPI app.
|
|
486
|
+
"""
|
|
487
|
+
project_name = u.normalize_name_for_api(self.manifest_cfg.project_variables.name)
|
|
488
|
+
project_version = self.manifest_cfg.project_variables.major_version
|
|
489
|
+
mount_path = mount_path_format.format(project_name=project_name, project_version=project_version)
|
|
490
|
+
|
|
491
|
+
lifespan = self.get_lifespan(mount_path, host, port, is_standalone_mode)
|
|
492
|
+
fastapi_app = self.create_app(lifespan, mount_path=mount_path)
|
|
493
|
+
return FastAPIComponents(mount_path=mount_path, lifespan=lifespan, fastapi_app=fastapi_app)
|
|
494
|
+
|
|
495
|
+
def run(self, uvicorn_args: Namespace) -> None:
|
|
496
|
+
"""
|
|
497
|
+
Runs the API server with uvicorn for CLI "squirrels run"
|
|
498
|
+
|
|
499
|
+
Arguments:
|
|
500
|
+
uvicorn_args: List of arguments to pass to uvicorn.run. Supports "host", "port", and "forwarded_allow_ips"
|
|
501
|
+
"""
|
|
502
|
+
host = uvicorn_args.host
|
|
503
|
+
port = uvicorn_args.port
|
|
504
|
+
forwarded_allow_ips = uvicorn_args.forwarded_allow_ips
|
|
505
|
+
|
|
506
|
+
server = self.get_fastapi_components(host=host, port=port, is_standalone_mode=True)
|
|
507
|
+
|
|
508
|
+
root_app = FastAPI(lifespan=server.lifespan)
|
|
509
|
+
root_app.mount(server.mount_path, server.fastapi_app)
|
|
510
|
+
|
|
511
|
+
# Enable CORS handling on the root app so preflight requests (OPTIONS)
|
|
512
|
+
# to top-level endpoints like `/.well-known/oauth-protected-resource` do not 405.
|
|
513
|
+
allowed_credential_origins = self.env_vars.auth_credential_origins
|
|
514
|
+
configurables_as_headers: list[str] = []
|
|
515
|
+
for name in self.manifest_cfg.configurables.keys():
|
|
516
|
+
configurables_as_headers.append(f"x-config-{name}") # underscore version
|
|
517
|
+
configurables_as_headers.append(f"x-config-{u.normalize_name_for_api(name)}") # dash version
|
|
518
|
+
|
|
519
|
+
root_app.add_middleware(
|
|
520
|
+
SmartCORSMiddleware,
|
|
521
|
+
allowed_credential_origins=allowed_credential_origins,
|
|
522
|
+
configurables_as_headers=configurables_as_headers,
|
|
523
|
+
)
|
|
524
|
+
|
|
525
|
+
@root_app.get("/.well-known/oauth-protected-resource", tags=["Authentication"])
|
|
526
|
+
async def oauth_protected_resource(request: Request) -> rm.OAuthProtectedResourceMetadata:
|
|
527
|
+
resource = str(request.base_url).rstrip("/")
|
|
528
|
+
|
|
529
|
+
auth_servers: list[str] = []
|
|
530
|
+
for provider in self.project._auth.auth_providers:
|
|
531
|
+
auth_servers.append(provider.provider_configs.server_url)
|
|
532
|
+
|
|
533
|
+
return rm.OAuthProtectedResourceMetadata(
|
|
534
|
+
resource=resource,
|
|
535
|
+
authorization_servers=list(set(auth_servers)),
|
|
536
|
+
scopes_supported=["email", "profile"],
|
|
537
|
+
)
|
|
538
|
+
|
|
539
|
+
mount_path_stripped = server.mount_path.rstrip("/")
|
|
540
|
+
if mount_path_stripped != "":
|
|
541
|
+
root_app.add_route("/mcp", self._mcp_app, methods=["GET", "POST"])
|
|
542
|
+
|
|
543
|
+
@root_app.get("/", include_in_schema=False)
|
|
544
|
+
async def redirect_to_studio():
|
|
545
|
+
return RedirectResponse(url=f"{mount_path_stripped}/studio")
|
|
546
|
+
|
|
547
|
+
# Run the API Server
|
|
548
|
+
import uvicorn
|
|
549
|
+
uvicorn.run(
|
|
550
|
+
root_app, host=host, port=port, proxy_headers=True, forwarded_allow_ips=forwarded_allow_ips
|
|
551
|
+
)
|
|
552
|
+
|