squirrels 0.3.0__tar.gz → 0.3.2__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of squirrels might be problematic. Click here for more details.
- {squirrels-0.3.0 → squirrels-0.3.2}/PKG-INFO +3 -3
- {squirrels-0.3.0 → squirrels-0.3.2}/pyproject.toml +2 -2
- {squirrels-0.3.0 → squirrels-0.3.2}/squirrels/__init__.py +1 -1
- {squirrels-0.3.0 → squirrels-0.3.2}/squirrels/_api_response_models.py +1 -2
- {squirrels-0.3.0 → squirrels-0.3.2}/squirrels/_api_server.py +9 -7
- {squirrels-0.3.0 → squirrels-0.3.2}/squirrels/_authenticator.py +2 -2
- {squirrels-0.3.0 → squirrels-0.3.2}/squirrels/_connection_set.py +7 -3
- {squirrels-0.3.0 → squirrels-0.3.2}/squirrels/_constants.py +2 -1
- {squirrels-0.3.0 → squirrels-0.3.2}/squirrels/_environcfg.py +15 -11
- {squirrels-0.3.0 → squirrels-0.3.2}/squirrels/_initializer.py +2 -2
- {squirrels-0.3.0 → squirrels-0.3.2}/squirrels/_manifest.py +2 -2
- {squirrels-0.3.0 → squirrels-0.3.2}/squirrels/_models.py +33 -24
- {squirrels-0.3.0 → squirrels-0.3.2}/squirrels/_parameter_configs.py +49 -34
- {squirrels-0.3.0 → squirrels-0.3.2}/squirrels/_py_module.py +1 -1
- {squirrels-0.3.0 → squirrels-0.3.2}/squirrels/arguments/run_time_args.py +4 -3
- {squirrels-0.3.0 → squirrels-0.3.2}/squirrels/data_sources.py +23 -15
- squirrels-0.3.2/squirrels/package_data/assets/index.css +1 -0
- {squirrels-0.3.0 → squirrels-0.3.2}/squirrels/package_data/assets/index.js +13 -13
- {squirrels-0.3.0 → squirrels-0.3.2}/squirrels/package_data/base_project/.gitignore +1 -0
- squirrels-0.3.2/squirrels/package_data/base_project/assets/weather.db +0 -0
- squirrels-0.3.0/squirrels/package_data/base_project/environcfg.yml → squirrels-0.3.2/squirrels/package_data/base_project/env.yml +1 -1
- {squirrels-0.3.0 → squirrels-0.3.2}/squirrels/package_data/base_project/models/dbviews/database_view1.sql +4 -5
- {squirrels-0.3.0 → squirrels-0.3.2}/squirrels/package_data/base_project/parameters.yml +16 -16
- {squirrels-0.3.0 → squirrels-0.3.2}/squirrels/package_data/base_project/pyconfigs/connections.py +1 -1
- {squirrels-0.3.0 → squirrels-0.3.2}/squirrels/package_data/base_project/pyconfigs/parameters.py +16 -16
- {squirrels-0.3.0 → squirrels-0.3.2}/squirrels/package_data/base_project/squirrels.yml.j2 +1 -1
- {squirrels-0.3.0 → squirrels-0.3.2}/squirrels/parameter_options.py +5 -4
- {squirrels-0.3.0 → squirrels-0.3.2}/squirrels/parameters.py +179 -62
- squirrels-0.3.0/squirrels/package_data/assets/index.css +0 -1
- squirrels-0.3.0/squirrels/package_data/base_project/database/weather.db +0 -0
- {squirrels-0.3.0 → squirrels-0.3.2}/LICENSE +0 -0
- {squirrels-0.3.0 → squirrels-0.3.2}/README.md +0 -0
- {squirrels-0.3.0 → squirrels-0.3.2}/squirrels/_command_line.py +0 -0
- {squirrels-0.3.0 → squirrels-0.3.2}/squirrels/_package_loader.py +0 -0
- {squirrels-0.3.0 → squirrels-0.3.2}/squirrels/_parameter_sets.py +0 -0
- {squirrels-0.3.0 → squirrels-0.3.2}/squirrels/_seeds.py +0 -0
- {squirrels-0.3.0 → squirrels-0.3.2}/squirrels/_timer.py +0 -0
- {squirrels-0.3.0 → squirrels-0.3.2}/squirrels/_utils.py +0 -0
- {squirrels-0.3.0 → squirrels-0.3.2}/squirrels/_version.py +0 -0
- {squirrels-0.3.0 → squirrels-0.3.2}/squirrels/arguments/init_time_args.py +0 -0
- {squirrels-0.3.0 → squirrels-0.3.2}/squirrels/dateutils.py +0 -0
- {squirrels-0.3.0 → squirrels-0.3.2}/squirrels/package_data/assets/favicon.ico +0 -0
- {squirrels-0.3.0/squirrels/package_data/base_project/database → squirrels-0.3.2/squirrels/package_data/base_project/assets}/expenses.db +0 -0
- {squirrels-0.3.0 → squirrels-0.3.2}/squirrels/package_data/base_project/connections.yml +0 -0
- {squirrels-0.3.0 → squirrels-0.3.2}/squirrels/package_data/base_project/docker/.dockerignore +0 -0
- {squirrels-0.3.0 → squirrels-0.3.2}/squirrels/package_data/base_project/docker/Dockerfile +0 -0
- {squirrels-0.3.0 → squirrels-0.3.2}/squirrels/package_data/base_project/docker/compose.yml +0 -0
- {squirrels-0.3.0 → squirrels-0.3.2}/squirrels/package_data/base_project/models/dbviews/database_view1.py +0 -0
- {squirrels-0.3.0 → squirrels-0.3.2}/squirrels/package_data/base_project/models/federates/dataset_example.py +0 -0
- {squirrels-0.3.0 → squirrels-0.3.2}/squirrels/package_data/base_project/models/federates/dataset_example.sql +0 -0
- {squirrels-0.3.0 → squirrels-0.3.2}/squirrels/package_data/base_project/pyconfigs/auth.py +0 -0
- {squirrels-0.3.0 → squirrels-0.3.2}/squirrels/package_data/base_project/pyconfigs/context.py +0 -0
- {squirrels-0.3.0 → squirrels-0.3.2}/squirrels/package_data/base_project/seeds/seed_categories.csv +0 -0
- {squirrels-0.3.0 → squirrels-0.3.2}/squirrels/package_data/base_project/seeds/seed_subcategories.csv +0 -0
- {squirrels-0.3.0 → squirrels-0.3.2}/squirrels/package_data/base_project/tmp/.gitignore +0 -0
- {squirrels-0.3.0 → squirrels-0.3.2}/squirrels/package_data/templates/index.html +0 -0
- {squirrels-0.3.0 → squirrels-0.3.2}/squirrels/user_base.py +0 -0
|
@@ -1,14 +1,14 @@
|
|
|
1
1
|
Metadata-Version: 2.1
|
|
2
2
|
Name: squirrels
|
|
3
|
-
Version: 0.3.
|
|
3
|
+
Version: 0.3.2
|
|
4
4
|
Summary: Squirrels - API Framework for Data Analytics
|
|
5
5
|
Home-page: https://squirrels-analytics.github.io
|
|
6
|
-
License:
|
|
6
|
+
License: Apache-2.0
|
|
7
7
|
Author: Tim Huang
|
|
8
8
|
Author-email: tim.yuting@hotmail.com
|
|
9
9
|
Requires-Python: >=3.9,<4.0
|
|
10
10
|
Classifier: Intended Audience :: Developers
|
|
11
|
-
Classifier: License :: OSI Approved ::
|
|
11
|
+
Classifier: License :: OSI Approved :: Apache Software License
|
|
12
12
|
Classifier: Programming Language :: Python :: 3
|
|
13
13
|
Classifier: Programming Language :: Python :: 3.9
|
|
14
14
|
Classifier: Programming Language :: Python :: 3.10
|
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
[tool.poetry]
|
|
2
2
|
name = "squirrels"
|
|
3
|
-
version = "0.3.
|
|
3
|
+
version = "0.3.2"
|
|
4
4
|
description = "Squirrels - API Framework for Data Analytics"
|
|
5
|
-
license = "
|
|
5
|
+
license = "Apache-2.0"
|
|
6
6
|
authors = ["Tim Huang <tim.yuting@hotmail.com>"]
|
|
7
7
|
readme = "README.md"
|
|
8
8
|
homepage = "https://squirrels-analytics.github.io"
|
|
@@ -33,7 +33,6 @@ class SingleSelectParameterModel(SelectParameterModel):
|
|
|
33
33
|
class MultiSelectParameterModel(SelectParameterModel):
|
|
34
34
|
widget_type: Annotated[str, Field(examples=["multi_select"])]
|
|
35
35
|
show_select_all: bool
|
|
36
|
-
is_dropdown: bool
|
|
37
36
|
order_matters: bool
|
|
38
37
|
selected_ids: list[str]
|
|
39
38
|
|
|
@@ -63,7 +62,7 @@ class NumberRangeParameterModel(NumericParameterModel):
|
|
|
63
62
|
class TextParameterModel(ParameterModelBase):
|
|
64
63
|
widget_type: Annotated[str, Field(examples=["text"])]
|
|
65
64
|
entered_text: str
|
|
66
|
-
|
|
65
|
+
input_type: str
|
|
67
66
|
|
|
68
67
|
class ParametersModel(BaseModel):
|
|
69
68
|
parameters: list[
|
|
@@ -81,19 +81,15 @@ class ApiServer:
|
|
|
81
81
|
except u.InvalidInputError as exc:
|
|
82
82
|
traceback.print_exc()
|
|
83
83
|
return JSONResponse(status_code=status.HTTP_400_BAD_REQUEST,
|
|
84
|
-
content={"message":
|
|
84
|
+
content={"message": str(exc), "blame": "API client"})
|
|
85
85
|
except u.ConfigurationError as exc:
|
|
86
86
|
traceback.print_exc()
|
|
87
87
|
return JSONResponse(status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
|
|
88
|
-
content={"message": f"
|
|
89
|
-
except NotImplementedError as exc:
|
|
90
|
-
traceback.print_exc()
|
|
91
|
-
return JSONResponse(status_code=status.HTTP_501_NOT_IMPLEMENTED,
|
|
92
|
-
content={"message": f"Not implemented error: {str(exc)}"})
|
|
88
|
+
content={"message": f"An unexpected error occurred", "blame": "Squirrels project"})
|
|
93
89
|
except Exception as exc:
|
|
94
90
|
traceback.print_exc()
|
|
95
91
|
return JSONResponse(status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
|
|
96
|
-
content={"message": f"
|
|
92
|
+
content={"message": f"An unexpected error occurred", "blame": "Squirrels framework"})
|
|
97
93
|
|
|
98
94
|
app.add_middleware(
|
|
99
95
|
CORSMiddleware, allow_origins=["*"], allow_credentials=True, allow_methods=["*"], allow_headers=["*"],
|
|
@@ -266,6 +262,12 @@ class ApiServer:
|
|
|
266
262
|
curr_parameters_path = parameters_path.format(dataset=dataset_normalized)
|
|
267
263
|
curr_results_path = results_path.format(dataset=dataset_normalized)
|
|
268
264
|
|
|
265
|
+
for param in dataset_cfg.parameters:
|
|
266
|
+
if param not in param_fields:
|
|
267
|
+
all_params = list(param_fields.keys())
|
|
268
|
+
raise u.ConfigurationError(f"Dataset '{dataset_name}' use parameter '{param}' which doesn't exist. Available parameters are:"
|
|
269
|
+
f"\n {all_params}")
|
|
270
|
+
|
|
269
271
|
QueryModelGet = make_dataclass("QueryParams", [
|
|
270
272
|
param_fields[param].as_query_info() for param in dataset_cfg.parameters
|
|
271
273
|
])
|
|
@@ -39,7 +39,7 @@ class Authenticator:
|
|
|
39
39
|
try:
|
|
40
40
|
real_user = get_user(self._get_auth_args(username, password)) if get_user is not None else None
|
|
41
41
|
except Exception as e:
|
|
42
|
-
raise u.FileExecutionError(f'Failed to run "{c.GET_USER_FUNC}" in {c.AUTH_FILE}', e)
|
|
42
|
+
raise u.FileExecutionError(f'Failed to run "{c.GET_USER_FUNC}" in {c.AUTH_FILE}', e) from e
|
|
43
43
|
|
|
44
44
|
if isinstance(real_user, User):
|
|
45
45
|
return real_user
|
|
@@ -53,7 +53,7 @@ class Authenticator:
|
|
|
53
53
|
try:
|
|
54
54
|
return user_cls.Create(username, is_internal=is_internal, **fake_user)
|
|
55
55
|
except Exception as e:
|
|
56
|
-
raise u.FileExecutionError(f'Failed to create user from User model in {c.AUTH_FILE}', e)
|
|
56
|
+
raise u.FileExecutionError(f'Failed to create user from User model in {c.AUTH_FILE}', e) from e
|
|
57
57
|
|
|
58
58
|
return None
|
|
59
59
|
|
|
@@ -31,15 +31,19 @@ class ConnectionSet:
|
|
|
31
31
|
|
|
32
32
|
def run_sql_query_from_conn_name(self, query: str, conn_name: str, placeholders: dict = {}) -> pd.DataFrame:
|
|
33
33
|
engine = self._get_engine(conn_name)
|
|
34
|
-
|
|
35
|
-
|
|
34
|
+
try:
|
|
35
|
+
df = pd.read_sql(query, engine, params=placeholders)
|
|
36
|
+
return df
|
|
37
|
+
except Exception as e:
|
|
38
|
+
raise RuntimeError(e) from e
|
|
36
39
|
|
|
37
40
|
def _dispose(self) -> None:
|
|
38
41
|
"""
|
|
39
42
|
Disposes of all the engines in this ConnectionSet
|
|
40
43
|
"""
|
|
41
44
|
for pool in self._engines.values():
|
|
42
|
-
pool
|
|
45
|
+
if isinstance(pool, Engine):
|
|
46
|
+
pool.dispose()
|
|
43
47
|
|
|
44
48
|
|
|
45
49
|
class ConnectionSetIO:
|
|
@@ -79,13 +79,14 @@ ASSETS_FOLDER = 'assets'
|
|
|
79
79
|
TEMPLATES_FOLDER = 'templates'
|
|
80
80
|
|
|
81
81
|
ENVIRON_CONFIG_FILE = 'environcfg.yml'
|
|
82
|
+
ENV_CONFIG_FILE = 'env.yml'
|
|
82
83
|
MANIFEST_JINJA_FILE = 'squirrels.yml.j2'
|
|
83
84
|
CONNECTIONS_YML_FILE = 'connections.yml'
|
|
84
85
|
PARAMETERS_YML_FILE = 'parameters.yml'
|
|
85
86
|
MANIFEST_FILE = 'squirrels.yml'
|
|
86
87
|
LU_DATA_FILE = 'lu_data.xlsx'
|
|
87
88
|
|
|
88
|
-
DATABASE_FOLDER = '
|
|
89
|
+
DATABASE_FOLDER = 'assets'
|
|
89
90
|
PACKAGES_FOLDER = 'sqrl_packages'
|
|
90
91
|
|
|
91
92
|
MODELS_FOLDER = 'models'
|
|
@@ -5,7 +5,8 @@ import os, yaml
|
|
|
5
5
|
from . import _constants as c, _utils as u
|
|
6
6
|
from ._timer import timer, time
|
|
7
7
|
|
|
8
|
-
|
|
8
|
+
_GLOBAL_SQUIRRELS_CFG_FILE1 = u.join_paths(os.path.expanduser('~'), '.squirrels', c.ENVIRON_CONFIG_FILE)
|
|
9
|
+
_GLOBAL_SQUIRRELS_CFG_FILE2 = u.join_paths(os.path.expanduser('~'), '.squirrels', c.ENV_CONFIG_FILE)
|
|
9
10
|
|
|
10
11
|
|
|
11
12
|
@dataclass
|
|
@@ -61,17 +62,20 @@ class EnvironConfigIO:
|
|
|
61
62
|
except FileNotFoundError:
|
|
62
63
|
return {}
|
|
63
64
|
|
|
64
|
-
|
|
65
|
-
|
|
65
|
+
master_env_config1 = load_yaml(_GLOBAL_SQUIRRELS_CFG_FILE1)
|
|
66
|
+
master_env_config2 = load_yaml(_GLOBAL_SQUIRRELS_CFG_FILE2)
|
|
67
|
+
proj_env_config1 = load_yaml(c.ENVIRON_CONFIG_FILE)
|
|
68
|
+
proj_env_config2 = load_yaml(c.ENV_CONFIG_FILE)
|
|
66
69
|
|
|
67
|
-
for
|
|
68
|
-
|
|
69
|
-
|
|
70
|
+
for project_config in [master_env_config2, proj_env_config1, proj_env_config2]:
|
|
71
|
+
for key in project_config:
|
|
72
|
+
master_env_config1.setdefault(key, {})
|
|
73
|
+
master_env_config1[key].update(project_config[key])
|
|
70
74
|
|
|
71
|
-
users =
|
|
72
|
-
env_vars =
|
|
73
|
-
credentials =
|
|
74
|
-
secrets =
|
|
75
|
+
users = master_env_config1.get(c.USERS_KEY, {})
|
|
76
|
+
env_vars = master_env_config1.get(c.ENV_VARS_KEY, {})
|
|
77
|
+
credentials = master_env_config1.get(c.CREDENTIALS_KEY, {})
|
|
78
|
+
secrets = master_env_config1.get(c.SECRETS_KEY, {})
|
|
75
79
|
|
|
76
80
|
cls.obj = _EnvironConfig(users, env_vars, credentials, secrets)
|
|
77
|
-
timer.add_activity_time(f"loading {c.
|
|
81
|
+
timer.add_activity_time(f"loading {c.ENV_CONFIG_FILE} file", start)
|
|
@@ -72,7 +72,7 @@ class Initializer:
|
|
|
72
72
|
|
|
73
73
|
remaining_questions = [
|
|
74
74
|
inquirer.Confirm(AUTH,
|
|
75
|
-
message=f"Do you want to add the '{c.AUTH_FILE}' file?" ,
|
|
75
|
+
message=f"Do you want to add the '{c.AUTH_FILE}' file to enable custom API authentication?" ,
|
|
76
76
|
default=False),
|
|
77
77
|
inquirer.List(SAMPLE_DB,
|
|
78
78
|
message="What sample sqlite database do you wish to use (if any)?",
|
|
@@ -154,7 +154,7 @@ class Initializer:
|
|
|
154
154
|
raise NotImplementedError(f"Format '{parameters_format}' not supported for configuring widget parameters")
|
|
155
155
|
|
|
156
156
|
self._copy_pyconfig_file(c.CONTEXT_FILE)
|
|
157
|
-
self._copy_file(c.
|
|
157
|
+
self._copy_file(c.ENV_CONFIG_FILE)
|
|
158
158
|
self._copy_seed_file(c.CATEGORY_SEED_FILE)
|
|
159
159
|
self._copy_seed_file(c.SUBCATEGORY_SEED_FILE)
|
|
160
160
|
|
|
@@ -149,7 +149,7 @@ class DatasetsConfig(ManifestComponentConfig):
|
|
|
149
149
|
label: str
|
|
150
150
|
model: str
|
|
151
151
|
scope: DatasetScope
|
|
152
|
-
parameters:
|
|
152
|
+
parameters: list[str]
|
|
153
153
|
traits: dict
|
|
154
154
|
default_test_set: Optional[str]
|
|
155
155
|
|
|
@@ -166,7 +166,7 @@ class DatasetsConfig(ManifestComponentConfig):
|
|
|
166
166
|
scope_list = [scope.name.lower() for scope in DatasetScope]
|
|
167
167
|
raise u.ConfigurationError(f'Scope not found for dataset "{name}". Scope must be one of {scope_list}') from e
|
|
168
168
|
|
|
169
|
-
parameters = kwargs.get(c.DATASET_PARAMETERS_KEY)
|
|
169
|
+
parameters = kwargs.get(c.DATASET_PARAMETERS_KEY, [])
|
|
170
170
|
traits = kwargs.get(c.DATASET_TRAITS_KEY, {})
|
|
171
171
|
default_test_set = kwargs.get(c.DATASET_DEFAULT_TEST_SET_KEY)
|
|
172
172
|
return cls(name, label, model, scope, parameters, traits, default_test_set)
|
|
@@ -38,6 +38,24 @@ class _SqlModelConfig:
|
|
|
38
38
|
|
|
39
39
|
## Applicable for federated models
|
|
40
40
|
materialized: Materialization
|
|
41
|
+
|
|
42
|
+
def set_attribute(self, **kwargs) -> str:
|
|
43
|
+
connection_name = kwargs.get(c.DBVIEW_CONN_KEY)
|
|
44
|
+
if connection_name is not None:
|
|
45
|
+
if not isinstance(connection_name, str):
|
|
46
|
+
raise u.ConfigurationError("The 'connection_name' argument of 'config' macro must be a string")
|
|
47
|
+
self.connection_name = connection_name
|
|
48
|
+
|
|
49
|
+
materialized: str = kwargs.get(c.MATERIALIZED_KEY)
|
|
50
|
+
if materialized is not None:
|
|
51
|
+
if not isinstance(materialized, str):
|
|
52
|
+
raise u.ConfigurationError("The 'materialized' argument of 'config' macro must be a string")
|
|
53
|
+
try:
|
|
54
|
+
self.materialized = Materialization[materialized.upper()]
|
|
55
|
+
except KeyError as e:
|
|
56
|
+
valid_options = [x.name for x in Materialization]
|
|
57
|
+
raise u.ConfigurationError(f"The 'materialized' argument value '{materialized}' is not valid. Must be one of: {valid_options}") from e
|
|
58
|
+
return ""
|
|
41
59
|
|
|
42
60
|
def get_sql_for_create(self, model_name: str, select_query: str) -> str:
|
|
43
61
|
if self.materialized == Materialization.TABLE:
|
|
@@ -45,18 +63,9 @@ class _SqlModelConfig:
|
|
|
45
63
|
elif self.materialized == Materialization.VIEW:
|
|
46
64
|
create_prefix = f"CREATE VIEW {model_name} AS\n"
|
|
47
65
|
else:
|
|
48
|
-
raise
|
|
66
|
+
raise u.ConfigurationError(f"Materialization option not supported: {self.materialized}")
|
|
49
67
|
|
|
50
68
|
return create_prefix + select_query
|
|
51
|
-
|
|
52
|
-
def set_attribute(self, **kwargs) -> str:
|
|
53
|
-
connection_name = kwargs.get(c.DBVIEW_CONN_KEY)
|
|
54
|
-
materialized = kwargs.get(c.MATERIALIZED_KEY)
|
|
55
|
-
if isinstance(connection_name, str):
|
|
56
|
-
self.connection_name = connection_name
|
|
57
|
-
if isinstance(materialized, str):
|
|
58
|
-
self.materialized = Materialization[materialized.upper()]
|
|
59
|
-
return ""
|
|
60
69
|
|
|
61
70
|
|
|
62
71
|
ContextFunc = Callable[[dict[str, Any], ContextArgs], None]
|
|
@@ -220,11 +229,11 @@ class _Model(_Referable):
|
|
|
220
229
|
connection_name = self._get_dbview_conn_name()
|
|
221
230
|
materialized = self._get_materialized()
|
|
222
231
|
configuration = _SqlModelConfig(connection_name, materialized)
|
|
223
|
-
is_placeholder = lambda
|
|
232
|
+
is_placeholder = lambda placeholder: placeholder in placeholders
|
|
224
233
|
kwargs = {
|
|
225
234
|
"proj_vars": ctx_args.proj_vars, "env_vars": ctx_args.env_vars, "user": ctx_args.user, "prms": ctx_args.prms,
|
|
226
235
|
"traits": ctx_args.traits, "ctx": ctx, "is_placeholder": is_placeholder, "set_placeholder": ctx_args.set_placeholder,
|
|
227
|
-
"config": configuration.set_attribute, "
|
|
236
|
+
"config": configuration.set_attribute, "param_exists": ctx_args.param_exists
|
|
228
237
|
}
|
|
229
238
|
dependencies = set()
|
|
230
239
|
if self.query_file.model_type == ModelType.FEDERATE:
|
|
@@ -236,7 +245,7 @@ class _Model(_Referable):
|
|
|
236
245
|
try:
|
|
237
246
|
query = await asyncio.to_thread(u.render_string, raw_query, **kwargs)
|
|
238
247
|
except Exception as e:
|
|
239
|
-
raise u.FileExecutionError(f'Failed to compile sql model "{self.name}"', e)
|
|
248
|
+
raise u.FileExecutionError(f'Failed to compile sql model "{self.name}"', e) from e
|
|
240
249
|
|
|
241
250
|
compiled_query = _SqlModelQuery(query, configuration)
|
|
242
251
|
return compiled_query, dependencies
|
|
@@ -252,11 +261,11 @@ class _Model(_Referable):
|
|
|
252
261
|
try:
|
|
253
262
|
dependencies = await asyncio.to_thread(self.query_file.raw_query.dependencies_func, sqrl_args)
|
|
254
263
|
except Exception as e:
|
|
255
|
-
raise u.FileExecutionError(f'Failed to run "{c.DEP_FUNC}" function for python model "{self.name}"', e)
|
|
264
|
+
raise u.FileExecutionError(f'Failed to run "{c.DEP_FUNC}" function for python model "{self.name}"', e) from e
|
|
256
265
|
|
|
257
266
|
dbview_conn_name = self._get_dbview_conn_name()
|
|
258
267
|
connections = ConnectionSetIO.obj.get_engines_as_dict()
|
|
259
|
-
ref = lambda
|
|
268
|
+
ref = lambda model: self.upstreams[model].result
|
|
260
269
|
sqrl_args = ModelArgs(
|
|
261
270
|
ctx_args.proj_vars, ctx_args.env_vars, ctx_args.user, ctx_args.prms, ctx_args.traits, placeholders, ctx,
|
|
262
271
|
dbview_conn_name, connections, dependencies, ref
|
|
@@ -267,7 +276,7 @@ class _Model(_Referable):
|
|
|
267
276
|
raw_query: _RawPyQuery = self.query_file.raw_query
|
|
268
277
|
return raw_query.query(sqrl=sqrl_args)
|
|
269
278
|
except Exception as e:
|
|
270
|
-
raise u.FileExecutionError(f'Failed to run "{c.MAIN_FUNC}" function for python model "{self.name}"', e)
|
|
279
|
+
raise u.FileExecutionError(f'Failed to run "{c.MAIN_FUNC}" function for python model "{self.name}"', e) from e
|
|
271
280
|
|
|
272
281
|
return _PyModelQuery(compiled_query), dependencies
|
|
273
282
|
|
|
@@ -286,7 +295,7 @@ class _Model(_Referable):
|
|
|
286
295
|
elif self.query_file.query_type == QueryType.PYTHON:
|
|
287
296
|
compiled_query, dependencies = await self._compile_python_model(ctx, ctx_args, placeholders)
|
|
288
297
|
else:
|
|
289
|
-
raise
|
|
298
|
+
raise u.ConfigurationError(f"Query type not supported: {self.query_file.query_type}")
|
|
290
299
|
|
|
291
300
|
self.compiled_query = compiled_query
|
|
292
301
|
self.wait_count = len(dependencies)
|
|
@@ -335,7 +344,7 @@ class _Model(_Referable):
|
|
|
335
344
|
try:
|
|
336
345
|
return ConnectionSetIO.obj.run_sql_query_from_conn_name(query, config.connection_name, placeholders)
|
|
337
346
|
except RuntimeError as e:
|
|
338
|
-
raise u.FileExecutionError(f'Failed to run dbview sql model "{self.name}"', e)
|
|
347
|
+
raise u.FileExecutionError(f'Failed to run dbview sql model "{self.name}"', e) from e
|
|
339
348
|
|
|
340
349
|
df = await asyncio.to_thread(run_sql_query)
|
|
341
350
|
await asyncio.to_thread(self._load_pandas_to_table, df, conn)
|
|
@@ -347,7 +356,7 @@ class _Model(_Referable):
|
|
|
347
356
|
try:
|
|
348
357
|
return conn.execute(text(create_query), placeholders)
|
|
349
358
|
except Exception as e:
|
|
350
|
-
raise u.FileExecutionError(f'Failed to run federate sql model "{self.name}"', e)
|
|
359
|
+
raise u.FileExecutionError(f'Failed to run federate sql model "{self.name}"', e) from e
|
|
351
360
|
|
|
352
361
|
await asyncio.to_thread(create_table)
|
|
353
362
|
if self.needs_pandas or self.is_target:
|
|
@@ -410,7 +419,7 @@ class _DAG:
|
|
|
410
419
|
try:
|
|
411
420
|
context_func(ctx=context, sqrl=args)
|
|
412
421
|
except Exception as e:
|
|
413
|
-
raise u.FileExecutionError(f'Failed to run {c.CONTEXT_FILE} for dataset "{self.dataset.name}"', e)
|
|
422
|
+
raise u.FileExecutionError(f'Failed to run {c.CONTEXT_FILE} for dataset "{self.dataset.name}"', e) from e
|
|
414
423
|
timer.add_activity_time(f"running context.py for dataset '{self.dataset.name}'", start)
|
|
415
424
|
return context, args
|
|
416
425
|
|
|
@@ -496,7 +505,7 @@ class ModelsIO:
|
|
|
496
505
|
if extension == '.py':
|
|
497
506
|
query_type = QueryType.PYTHON
|
|
498
507
|
module = pm.PyModule(filepath)
|
|
499
|
-
dependencies_func = module.get_func_or_class(c.DEP_FUNC, default_attr=lambda
|
|
508
|
+
dependencies_func = module.get_func_or_class(c.DEP_FUNC, default_attr=lambda sqrl: [])
|
|
500
509
|
raw_query = _RawPyQuery(module.get_func_or_class(c.MAIN_FUNC), dependencies_func)
|
|
501
510
|
elif extension == '.sql':
|
|
502
511
|
query_type = QueryType.SQL
|
|
@@ -520,7 +529,7 @@ class ModelsIO:
|
|
|
520
529
|
populate_raw_queries_for_type(federates_path, ModelType.FEDERATE)
|
|
521
530
|
|
|
522
531
|
context_path = u.join_paths(c.PYCONFIGS_FOLDER, c.CONTEXT_FILE)
|
|
523
|
-
cls.context_func = pm.PyModule(context_path).get_func_or_class(c.MAIN_FUNC, default_attr=lambda
|
|
532
|
+
cls.context_func = pm.PyModule(context_path).get_func_or_class(c.MAIN_FUNC, default_attr=lambda ctx, sqrl: None)
|
|
524
533
|
|
|
525
534
|
timer.add_activity_time("loading files for models and context.py", start)
|
|
526
535
|
|
|
@@ -572,11 +581,11 @@ class ModelsIO:
|
|
|
572
581
|
elif test_set in ManifestIO.obj.selection_test_sets:
|
|
573
582
|
test_set_conf = ManifestIO.obj.selection_test_sets[test_set]
|
|
574
583
|
else:
|
|
575
|
-
raise u.
|
|
584
|
+
raise u.ConfigurationError(f"No test set named '{test_set}' was found when compiling dataset '{dataset}'. The test set must be defined if not default for dataset.")
|
|
576
585
|
|
|
577
586
|
error_msg_intro = f"Cannot compile dataset '{dataset}' with test set '{test_set}'."
|
|
578
587
|
if test_set_conf.datasets is not None and dataset not in test_set_conf.datasets:
|
|
579
|
-
raise u.
|
|
588
|
+
raise u.ConfigurationError(f"{error_msg_intro}\n Applicable datasets for test set '{test_set}' does not include dataset '{dataset}'.")
|
|
580
589
|
|
|
581
590
|
user_attributes = test_set_conf.user_attributes.copy()
|
|
582
591
|
selections = test_set_conf.parameters.copy()
|
|
@@ -1,11 +1,12 @@
|
|
|
1
1
|
from __future__ import annotations
|
|
2
2
|
from typing import Annotated, Type, Optional, Union, Sequence, Iterator, Any
|
|
3
|
+
from datetime import datetime
|
|
3
4
|
from dataclasses import dataclass, field
|
|
4
5
|
from abc import ABCMeta, abstractmethod
|
|
5
6
|
from copy import copy
|
|
6
7
|
from fastapi import Query
|
|
7
8
|
from pydantic.fields import Field, FieldInfo
|
|
8
|
-
import pandas as pd
|
|
9
|
+
import pandas as pd, re
|
|
9
10
|
|
|
10
11
|
from . import parameter_options as po, parameters as p, data_sources as d, _api_response_models as arm, _utils as u, _constants as c
|
|
11
12
|
from .user_base import User
|
|
@@ -75,11 +76,6 @@ class ParameterConfigBase(metaclass=ABCMeta):
|
|
|
75
76
|
"""
|
|
76
77
|
return copy(self)
|
|
77
78
|
|
|
78
|
-
def to_json_dict0(self) -> arm.ParameterModelBase:
|
|
79
|
-
return {
|
|
80
|
-
"widget_type": self.widget_type, "name": self.name, "label": self.label, "description": self.description
|
|
81
|
-
}
|
|
82
|
-
|
|
83
79
|
|
|
84
80
|
@dataclass
|
|
85
81
|
class ParameterConfig(ParameterConfigBase):
|
|
@@ -171,11 +167,6 @@ class SelectionParameterConfig(ParameterConfig):
|
|
|
171
167
|
other.children = self.children.copy()
|
|
172
168
|
return other
|
|
173
169
|
|
|
174
|
-
def to_json_dict0(self) -> dict:
|
|
175
|
-
output = super().to_json_dict0()
|
|
176
|
-
output['trigger_refresh'] = self.trigger_refresh
|
|
177
|
-
return output
|
|
178
|
-
|
|
179
170
|
|
|
180
171
|
@dataclass
|
|
181
172
|
class SingleSelectParameterConfig(SelectionParameterConfig):
|
|
@@ -220,19 +211,17 @@ class MultiSelectParameterConfig(SelectionParameterConfig):
|
|
|
220
211
|
Class to define configurations for multi-select parameter widgets.
|
|
221
212
|
"""
|
|
222
213
|
show_select_all: bool # = field(default=True, kw_only=True)
|
|
223
|
-
is_dropdown: bool # = field(default=True, kw_only=True)
|
|
224
214
|
order_matters: bool # = field(default=False, kw_only=True)
|
|
225
215
|
none_is_all: bool # = field(default=True, kw_only=True)
|
|
226
216
|
|
|
227
217
|
def __init__(
|
|
228
218
|
self, name: str, label: str, all_options: Sequence[Union[po.SelectParameterOption, dict]], *, description: str = "",
|
|
229
|
-
show_select_all: bool = True,
|
|
219
|
+
show_select_all: bool = True, order_matters: bool = False, none_is_all: bool = True,
|
|
230
220
|
user_attribute: Optional[str] = None, parent_name: Optional[str] = None
|
|
231
221
|
) -> None:
|
|
232
222
|
super().__init__("multi_select", name, label, all_options, description=description, user_attribute=user_attribute,
|
|
233
223
|
parent_name=parent_name)
|
|
234
224
|
self.show_select_all = show_select_all
|
|
235
|
-
self.is_dropdown = is_dropdown
|
|
236
225
|
self.order_matters = order_matters
|
|
237
226
|
self.none_is_all = none_is_all
|
|
238
227
|
|
|
@@ -250,13 +239,6 @@ class MultiSelectParameterConfig(SelectionParameterConfig):
|
|
|
250
239
|
else:
|
|
251
240
|
selected_ids = u.load_json_or_comma_delimited_str_as_list(selection)
|
|
252
241
|
return p.MultiSelectParameter(self, options, selected_ids)
|
|
253
|
-
|
|
254
|
-
def to_json_dict0(self) -> dict:
|
|
255
|
-
output = super().to_json_dict0()
|
|
256
|
-
output['show_select_all'] = self.show_select_all
|
|
257
|
-
output['is_dropdown'] = self.is_dropdown
|
|
258
|
-
output['order_matters'] = self.order_matters
|
|
259
|
-
return output
|
|
260
242
|
|
|
261
243
|
def get_api_field_info(self) -> APIParamFieldInfo:
|
|
262
244
|
identifiers = [x._identifier for x in self.all_options]
|
|
@@ -354,7 +336,7 @@ class DateRangeParameterConfig(_DateTypeParameterConfig):
|
|
|
354
336
|
try:
|
|
355
337
|
selected_start_date, selected_end_date = u.load_json_or_comma_delimited_str_as_list(selection)
|
|
356
338
|
except ValueError as e:
|
|
357
|
-
self._raise_invalid_input_error(selection, "Date range parameter selection must be two dates
|
|
339
|
+
self._raise_invalid_input_error(selection, "Date range parameter selection must be two dates.", e)
|
|
358
340
|
return p.DateRangeParameter(self, curr_option, selected_start_date, selected_end_date)
|
|
359
341
|
|
|
360
342
|
def get_api_field_info(self) -> APIParamFieldInfo:
|
|
@@ -442,7 +424,7 @@ class NumberRangeParameterConfig(_NumericParameterConfig):
|
|
|
442
424
|
self, selection: Optional[str], user: Optional[User], parent_param: Optional[p._SelectionParameter],
|
|
443
425
|
*, request_version: Optional[int] = None
|
|
444
426
|
) -> p.NumberRangeParameter:
|
|
445
|
-
curr_option: po.NumberRangeParameterOption = next(self._get_options_iterator(user, parent_param), None)
|
|
427
|
+
curr_option: Optional[po.NumberRangeParameterOption] = next(self._get_options_iterator(user, parent_param), None)
|
|
446
428
|
if selection is None:
|
|
447
429
|
if curr_option is not None:
|
|
448
430
|
selected_lower_value = curr_option._default_lower_value
|
|
@@ -453,7 +435,7 @@ class NumberRangeParameterConfig(_NumericParameterConfig):
|
|
|
453
435
|
try:
|
|
454
436
|
selected_lower_value, selected_upper_value = u.load_json_or_comma_delimited_str_as_list(selection)
|
|
455
437
|
except ValueError as e:
|
|
456
|
-
self._raise_invalid_input_error(selection, "Number range parameter selection must be two numbers
|
|
438
|
+
self._raise_invalid_input_error(selection, "Number range parameter selection must be two numbers.", e)
|
|
457
439
|
return p.NumberRangeParameter(self, curr_option, selected_lower_value, selected_upper_value)
|
|
458
440
|
|
|
459
441
|
def get_api_field_info(self) -> APIParamFieldInfo:
|
|
@@ -469,16 +451,54 @@ class TextParameterConfig(ParameterConfig):
|
|
|
469
451
|
Class to define configurations for text parameter widgets.
|
|
470
452
|
"""
|
|
471
453
|
all_options: Sequence[po.TextParameterOption] = field(repr=False)
|
|
472
|
-
|
|
454
|
+
input_type: str
|
|
473
455
|
|
|
474
456
|
def __init__(
|
|
475
|
-
self, name: str, label: str, all_options: Sequence[Union[po.TextParameterOption, dict]], *,
|
|
476
|
-
|
|
477
|
-
parent_name: Optional[str] = None
|
|
457
|
+
self, name: str, label: str, all_options: Sequence[Union[po.TextParameterOption, dict]], *, description: str = "",
|
|
458
|
+
input_type: str = "text", user_attribute: Optional[str] = None, parent_name: Optional[str] = None
|
|
478
459
|
) -> None:
|
|
479
460
|
super().__init__("text", name, label, all_options, description=description, user_attribute=user_attribute,
|
|
480
461
|
parent_name=parent_name)
|
|
481
|
-
|
|
462
|
+
|
|
463
|
+
allowed_input_types = ["text", "textarea", "number", "date", "datetime-local", "month", "time", "color", "password"]
|
|
464
|
+
if input_type not in allowed_input_types:
|
|
465
|
+
raise u.ConfigurationError(f"Invalid input type '{input_type}' for text parameter '{name}'. Must be one of {allowed_input_types}.")
|
|
466
|
+
|
|
467
|
+
self.input_type = input_type
|
|
468
|
+
for option in self.all_options:
|
|
469
|
+
self.validate_entered_text(option._default_text)
|
|
470
|
+
|
|
471
|
+
def validate_entered_text(self, entered_text: str) -> str:
|
|
472
|
+
if self.input_type == "number":
|
|
473
|
+
try:
|
|
474
|
+
int(entered_text)
|
|
475
|
+
except ValueError as e:
|
|
476
|
+
raise self._raise_invalid_input_error(entered_text, "Must be an integer (without decimals).", e)
|
|
477
|
+
elif self.input_type == "date":
|
|
478
|
+
try:
|
|
479
|
+
datetime.strptime(entered_text, "%Y-%m-%d")
|
|
480
|
+
except ValueError as e:
|
|
481
|
+
raise self._raise_invalid_input_error(entered_text, "Must be a date in YYYY-MM-DD format.", e)
|
|
482
|
+
elif self.input_type == "datetime-local":
|
|
483
|
+
try:
|
|
484
|
+
datetime.strptime(entered_text, "%Y-%m-%dT%H:%M")
|
|
485
|
+
except ValueError as e:
|
|
486
|
+
raise self._raise_invalid_input_error(entered_text, "Must be a date in YYYY-MM-DDThh:mm format (e.g. 2020-01-01T07:00).", e)
|
|
487
|
+
elif self.input_type == "month":
|
|
488
|
+
try:
|
|
489
|
+
datetime.strptime(entered_text, "%Y-%m")
|
|
490
|
+
except ValueError as e:
|
|
491
|
+
raise self._raise_invalid_input_error(entered_text, "Must be a date in YYYY-MM format.", e)
|
|
492
|
+
elif self.input_type == "time":
|
|
493
|
+
try:
|
|
494
|
+
datetime.strptime(entered_text, "%H:%M")
|
|
495
|
+
except ValueError as e:
|
|
496
|
+
raise self._raise_invalid_input_error(entered_text, "Must be a time in hh:mm format.", e)
|
|
497
|
+
elif self.input_type == "color":
|
|
498
|
+
if not re.match(r"^#[0-9a-fA-F]{6}$", entered_text):
|
|
499
|
+
raise self._raise_invalid_input_error(entered_text, "Must be a valid color hex code (e.g. #000000).")
|
|
500
|
+
|
|
501
|
+
return entered_text
|
|
482
502
|
|
|
483
503
|
@staticmethod
|
|
484
504
|
def ParameterOption(*args, **kwargs):
|
|
@@ -495,11 +515,6 @@ class TextParameterConfig(ParameterConfig):
|
|
|
495
515
|
curr_option: po.TextParameterOption = next(self._get_options_iterator(user, parent_param), None)
|
|
496
516
|
entered_text = curr_option._default_text if selection is None and curr_option is not None else selection
|
|
497
517
|
return p.TextParameter(self, curr_option, entered_text)
|
|
498
|
-
|
|
499
|
-
def to_json_dict0(self) -> dict:
|
|
500
|
-
output = super().to_json_dict0()
|
|
501
|
-
output['is_textarea'] = self.is_textarea
|
|
502
|
-
return output
|
|
503
518
|
|
|
504
519
|
def get_api_field_info(self) -> APIParamFieldInfo:
|
|
505
520
|
examples = [x._default_text for x in self.all_options]
|
|
@@ -57,4 +57,4 @@ def run_pyconfig_main(filename: str, kwargs: dict[str, Any] = {}) -> None:
|
|
|
57
57
|
try:
|
|
58
58
|
main_function(**kwargs)
|
|
59
59
|
except Exception as e:
|
|
60
|
-
raise u.FileExecutionError(f'Failed to run python file "{filepath}"', e)
|
|
60
|
+
raise u.FileExecutionError(f'Failed to run python file "{filepath}"', e) from e
|
|
@@ -5,7 +5,7 @@ import pandas as pd
|
|
|
5
5
|
|
|
6
6
|
from .init_time_args import ConnectionsArgs, ParametersArgs
|
|
7
7
|
from ..user_base import User
|
|
8
|
-
from ..parameters import Parameter,
|
|
8
|
+
from ..parameters import Parameter, TextValue
|
|
9
9
|
from .._connection_set import ConnectionSetIO
|
|
10
10
|
from .. import _utils as u
|
|
11
11
|
|
|
@@ -24,7 +24,7 @@ class ContextArgs(ParametersArgs):
|
|
|
24
24
|
traits: dict[str, Any]
|
|
25
25
|
_placeholders: dict[str, Any]
|
|
26
26
|
|
|
27
|
-
def set_placeholder(self, placeholder: str, value: Union[
|
|
27
|
+
def set_placeholder(self, placeholder: str, value: Union[TextValue, Any]) -> str:
|
|
28
28
|
"""
|
|
29
29
|
Method to set a placeholder value.
|
|
30
30
|
|
|
@@ -32,9 +32,10 @@ class ContextArgs(ParametersArgs):
|
|
|
32
32
|
placeholder: A string for the name of the placeholder
|
|
33
33
|
value: The value of the placeholder. Can be of any type
|
|
34
34
|
"""
|
|
35
|
-
if isinstance(value,
|
|
35
|
+
if isinstance(value, TextValue):
|
|
36
36
|
value = value._value_do_not_touch
|
|
37
37
|
self._placeholders[placeholder] = value
|
|
38
|
+
return ""
|
|
38
39
|
|
|
39
40
|
def param_exists(self, param_name: str) -> bool:
|
|
40
41
|
"""
|