squirrels 0.3.2__py3-none-any.whl → 0.4.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of squirrels might be problematic. Click here for more details.
- squirrels/__init__.py +7 -3
- squirrels/_api_response_models.py +96 -72
- squirrels/_api_server.py +375 -201
- squirrels/_authenticator.py +23 -22
- squirrels/_command_line.py +70 -46
- squirrels/_connection_set.py +23 -25
- squirrels/_constants.py +29 -78
- squirrels/_dashboards_io.py +61 -0
- squirrels/_environcfg.py +53 -50
- squirrels/_initializer.py +184 -141
- squirrels/_manifest.py +168 -195
- squirrels/_models.py +159 -292
- squirrels/_package_loader.py +7 -8
- squirrels/_parameter_configs.py +173 -141
- squirrels/_parameter_sets.py +49 -38
- squirrels/_py_module.py +7 -7
- squirrels/_seeds.py +13 -12
- squirrels/_utils.py +114 -54
- squirrels/_version.py +1 -1
- squirrels/arguments/init_time_args.py +16 -10
- squirrels/arguments/run_time_args.py +89 -24
- squirrels/dashboards.py +82 -0
- squirrels/data_sources.py +212 -232
- squirrels/dateutils.py +29 -26
- squirrels/package_data/assets/index.css +1 -1
- squirrels/package_data/assets/index.js +27 -18
- squirrels/package_data/base_project/.gitignore +2 -2
- squirrels/package_data/base_project/connections.yml +1 -1
- squirrels/package_data/base_project/dashboards/dashboard_example.py +32 -0
- squirrels/package_data/base_project/dashboards.yml +10 -0
- squirrels/package_data/base_project/docker/.dockerignore +9 -4
- squirrels/package_data/base_project/docker/Dockerfile +7 -6
- squirrels/package_data/base_project/docker/compose.yml +1 -1
- squirrels/package_data/base_project/env.yml +2 -2
- squirrels/package_data/base_project/models/dbviews/{database_view1.py → dbview_example.py} +2 -1
- squirrels/package_data/base_project/models/dbviews/{database_view1.sql → dbview_example.sql} +3 -2
- squirrels/package_data/base_project/models/federates/{dataset_example.py → federate_example.py} +6 -6
- squirrels/package_data/base_project/models/federates/{dataset_example.sql → federate_example.sql} +1 -1
- squirrels/package_data/base_project/parameters.yml +6 -4
- squirrels/package_data/base_project/pyconfigs/auth.py +1 -1
- squirrels/package_data/base_project/pyconfigs/connections.py +1 -1
- squirrels/package_data/base_project/pyconfigs/context.py +38 -10
- squirrels/package_data/base_project/pyconfigs/parameters.py +15 -7
- squirrels/package_data/base_project/squirrels.yml.j2 +14 -7
- squirrels/package_data/templates/index.html +3 -3
- squirrels/parameter_options.py +103 -106
- squirrels/parameters.py +347 -195
- squirrels/project.py +378 -0
- squirrels/user_base.py +14 -6
- {squirrels-0.3.2.dist-info → squirrels-0.4.0.dist-info}/METADATA +12 -23
- squirrels-0.4.0.dist-info/RECORD +60 -0
- squirrels/_timer.py +0 -23
- squirrels-0.3.2.dist-info/RECORD +0 -56
- {squirrels-0.3.2.dist-info → squirrels-0.4.0.dist-info}/LICENSE +0 -0
- {squirrels-0.3.2.dist-info → squirrels-0.4.0.dist-info}/WHEEL +0 -0
- {squirrels-0.3.2.dist-info → squirrels-0.4.0.dist-info}/entry_points.txt +0 -0
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
from typing import
|
|
1
|
+
from typing import Iterable, Callable, Any, Coroutine
|
|
2
2
|
from dataclasses import dataclass
|
|
3
3
|
from sqlalchemy import Engine
|
|
4
4
|
import pandas as pd
|
|
@@ -6,29 +6,58 @@ import pandas as pd
|
|
|
6
6
|
from .init_time_args import ConnectionsArgs, ParametersArgs
|
|
7
7
|
from ..user_base import User
|
|
8
8
|
from ..parameters import Parameter, TextValue
|
|
9
|
-
from ..
|
|
10
|
-
from .. import _utils as u
|
|
9
|
+
from .. import _utils as _u
|
|
11
10
|
|
|
12
11
|
|
|
13
12
|
@dataclass
|
|
14
13
|
class AuthArgs(ConnectionsArgs):
|
|
15
|
-
|
|
14
|
+
_connections: dict[str, Engine]
|
|
16
15
|
username: str
|
|
17
16
|
password: str
|
|
18
17
|
|
|
18
|
+
@property
|
|
19
|
+
def connections(self) -> dict[str, Engine]:
|
|
20
|
+
"""
|
|
21
|
+
A dictionary of connection keys to SQLAlchemy Engines for database connections.
|
|
22
|
+
|
|
23
|
+
Can also be used to store other in-memory objects in advance such as ML models.
|
|
24
|
+
"""
|
|
25
|
+
return self._connections.copy()
|
|
26
|
+
|
|
19
27
|
|
|
20
28
|
@dataclass
|
|
21
29
|
class ContextArgs(ParametersArgs):
|
|
22
|
-
user: User
|
|
23
|
-
|
|
24
|
-
|
|
30
|
+
user: User | None
|
|
31
|
+
_prms: dict[str, Parameter]
|
|
32
|
+
_traits: dict[str, Any]
|
|
25
33
|
_placeholders: dict[str, Any]
|
|
26
34
|
|
|
27
|
-
|
|
35
|
+
@property
|
|
36
|
+
def prms(self) -> dict[str, Parameter]:
|
|
37
|
+
"""
|
|
38
|
+
A dictionary of parameter names to parameter
|
|
39
|
+
"""
|
|
40
|
+
return self._prms.copy()
|
|
41
|
+
|
|
42
|
+
@property
|
|
43
|
+
def traits(self) -> dict[str, Any]:
|
|
44
|
+
"""
|
|
45
|
+
A dictionary of dataset trait name to value
|
|
46
|
+
"""
|
|
47
|
+
return self._traits.copy()
|
|
48
|
+
|
|
49
|
+
@property
|
|
50
|
+
def placeholders(self) -> dict[str, Any]:
|
|
51
|
+
"""
|
|
52
|
+
A dictionary of placeholder name to placeholder value
|
|
53
|
+
"""
|
|
54
|
+
return self._placeholders.copy()
|
|
55
|
+
|
|
56
|
+
def set_placeholder(self, placeholder: str, value: TextValue | Any) -> str:
|
|
28
57
|
"""
|
|
29
58
|
Method to set a placeholder value.
|
|
30
59
|
|
|
31
|
-
|
|
60
|
+
Arguments:
|
|
32
61
|
placeholder: A string for the name of the placeholder
|
|
33
62
|
value: The value of the placeholder. Can be of any type
|
|
34
63
|
"""
|
|
@@ -42,7 +71,7 @@ class ContextArgs(ParametersArgs):
|
|
|
42
71
|
Method to check whether a given parameter exists and is enabled (i.e., not hidden based on other parameter selections) for the current
|
|
43
72
|
dataset at runtime.
|
|
44
73
|
|
|
45
|
-
|
|
74
|
+
Arguments:
|
|
46
75
|
param_name: A string for the name of the parameter
|
|
47
76
|
|
|
48
77
|
Returns:
|
|
@@ -53,29 +82,46 @@ class ContextArgs(ParametersArgs):
|
|
|
53
82
|
|
|
54
83
|
@dataclass
|
|
55
84
|
class ModelDepsArgs(ContextArgs):
|
|
56
|
-
|
|
85
|
+
_ctx: dict[str, Any]
|
|
86
|
+
|
|
87
|
+
@property
|
|
88
|
+
def ctx(self) -> dict[str, Any]:
|
|
89
|
+
"""
|
|
90
|
+
Dictionary of context variables
|
|
91
|
+
"""
|
|
92
|
+
return self._ctx.copy()
|
|
57
93
|
|
|
58
94
|
|
|
59
95
|
@dataclass
|
|
60
96
|
class ModelArgs(ModelDepsArgs):
|
|
61
97
|
connection_name: str
|
|
62
98
|
_connections: dict[str, Engine]
|
|
63
|
-
_dependencies:
|
|
99
|
+
_dependencies: Iterable[str]
|
|
64
100
|
_ref: Callable[[str], pd.DataFrame]
|
|
101
|
+
_run_external_sql: Callable[[str, str | None], pd.DataFrame]
|
|
102
|
+
_use_duckdb: bool
|
|
65
103
|
|
|
66
104
|
@property
|
|
67
105
|
def connections(self) -> dict[str, Engine]:
|
|
106
|
+
"""
|
|
107
|
+
A dictionary of connection keys to SQLAlchemy Engines for database connections.
|
|
108
|
+
|
|
109
|
+
Can also be used to store other in-memory objects in advance such as ML models.
|
|
110
|
+
"""
|
|
68
111
|
return self._connections.copy()
|
|
69
112
|
|
|
70
113
|
@property
|
|
71
114
|
def dependencies(self) -> set[str]:
|
|
72
|
-
|
|
115
|
+
"""
|
|
116
|
+
The set of dependent data model names
|
|
117
|
+
"""
|
|
118
|
+
return set(self._dependencies)
|
|
73
119
|
|
|
74
120
|
def is_placeholder(self, placeholder: str) -> bool:
|
|
75
121
|
"""
|
|
76
122
|
Checks whether a name is a valid placeholder
|
|
77
123
|
|
|
78
|
-
|
|
124
|
+
Arguments:
|
|
79
125
|
placeholder: A string for the name of the placeholder
|
|
80
126
|
|
|
81
127
|
Returns:
|
|
@@ -83,13 +129,13 @@ class ModelArgs(ModelDepsArgs):
|
|
|
83
129
|
"""
|
|
84
130
|
return placeholder in self._placeholders
|
|
85
131
|
|
|
86
|
-
def get_placeholder_value(self, placeholder: str) ->
|
|
132
|
+
def get_placeholder_value(self, placeholder: str) -> Any | None:
|
|
87
133
|
"""
|
|
88
134
|
Gets the value of a placeholder.
|
|
89
135
|
|
|
90
136
|
USE WITH CAUTION. Do not use the return value directly in a SQL query since that could be prone to SQL injection
|
|
91
137
|
|
|
92
|
-
|
|
138
|
+
Arguments:
|
|
93
139
|
placeholder: A string for the name of the placeholder
|
|
94
140
|
|
|
95
141
|
Returns:
|
|
@@ -104,7 +150,7 @@ class ModelArgs(ModelDepsArgs):
|
|
|
104
150
|
Note: This is different behaviour than the "ref" function for SQL models, which figures out the dependent models for you,
|
|
105
151
|
and returns a string for the table/view name in SQLite instead of a pandas DataFrame.
|
|
106
152
|
|
|
107
|
-
|
|
153
|
+
Arguments:
|
|
108
154
|
model: The model name
|
|
109
155
|
|
|
110
156
|
Returns:
|
|
@@ -112,25 +158,24 @@ class ModelArgs(ModelDepsArgs):
|
|
|
112
158
|
"""
|
|
113
159
|
return self._ref(model)
|
|
114
160
|
|
|
115
|
-
def run_external_sql(self, sql_query: str, *, connection_name: str = None, **kwargs) -> pd.DataFrame:
|
|
161
|
+
def run_external_sql(self, sql_query: str, *, connection_name: str | None = None, **kwargs) -> pd.DataFrame:
|
|
116
162
|
"""
|
|
117
163
|
Runs a SQL query against an external database, with option to specify the connection name. Placeholder values are provided automatically
|
|
118
164
|
|
|
119
|
-
|
|
165
|
+
Arguments:
|
|
120
166
|
sql_query: The SQL query. Can be parameterized with placeholders
|
|
121
167
|
connection_name: The connection name for the database. If None, uses the one configured for the model
|
|
122
168
|
|
|
123
169
|
Returns:
|
|
124
170
|
The query result as a pandas DataFrame
|
|
125
171
|
"""
|
|
126
|
-
|
|
127
|
-
return ConnectionSetIO.obj.run_sql_query_from_conn_name(sql_query, connection_name, self._placeholders)
|
|
172
|
+
return self._run_external_sql(sql_query, connection_name)
|
|
128
173
|
|
|
129
|
-
def run_sql_on_dataframes(self, sql_query: str, *, dataframes: dict[str, pd.DataFrame] = None, **kwargs) -> pd.DataFrame:
|
|
174
|
+
def run_sql_on_dataframes(self, sql_query: str, *, dataframes: dict[str, pd.DataFrame] | None = None, **kwargs) -> pd.DataFrame:
|
|
130
175
|
"""
|
|
131
176
|
Uses a dictionary of dataframes to execute a SQL query in an embedded in-memory database (sqlite or duckdb based on setting)
|
|
132
177
|
|
|
133
|
-
|
|
178
|
+
Arguments:
|
|
134
179
|
sql_query: The SQL query to run
|
|
135
180
|
dataframes: A dictionary of table names to their pandas Dataframe. If None, uses results of dependent models
|
|
136
181
|
|
|
@@ -140,4 +185,24 @@ class ModelArgs(ModelDepsArgs):
|
|
|
140
185
|
if dataframes is None:
|
|
141
186
|
dataframes = {x: self.ref(x) for x in self._dependencies}
|
|
142
187
|
|
|
143
|
-
return
|
|
188
|
+
return _u.run_sql_on_dataframes(sql_query, dataframes, self._use_duckdb)
|
|
189
|
+
|
|
190
|
+
|
|
191
|
+
@dataclass
|
|
192
|
+
class DashboardArgs(ParametersArgs):
|
|
193
|
+
_get_dataset: Callable[[str, dict[str, Any]], Coroutine[Any, Any, pd.DataFrame]]
|
|
194
|
+
|
|
195
|
+
async def dataset(self, name: str, *, fixed_parameters: dict[str, Any] = {}) -> pd.DataFrame:
|
|
196
|
+
"""
|
|
197
|
+
Get dataset as DataFrame given dataset name. Can use this to access protected/private datasets regardless of user authenticated to the dashboard.
|
|
198
|
+
|
|
199
|
+
The parameters used for the dataset include the parameter selections coming from the REST API and the fixed_parameters argument. The fixed_parameters takes precedence.
|
|
200
|
+
|
|
201
|
+
Arguments:
|
|
202
|
+
name: A string for the dataset name
|
|
203
|
+
fixed_parameters: Parameters to set for this dataset (in addition to the ones set through real-time selections)
|
|
204
|
+
|
|
205
|
+
Returns:
|
|
206
|
+
A DataFrame for the result of the dataset
|
|
207
|
+
"""
|
|
208
|
+
return await self._get_dataset(name, fixed_parameters)
|
squirrels/dashboards.py
ADDED
|
@@ -0,0 +1,82 @@
|
|
|
1
|
+
import matplotlib.figure as _figure, io as _io, abc as _abc, typing as _t
|
|
2
|
+
|
|
3
|
+
from . import _constants as _c
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
class Dashboard(metaclass=_abc.ABCMeta):
|
|
7
|
+
"""
|
|
8
|
+
Abstract parent class for all Dashboard classes.
|
|
9
|
+
"""
|
|
10
|
+
|
|
11
|
+
@property
|
|
12
|
+
@_abc.abstractmethod
|
|
13
|
+
def _content(self) -> bytes | str:
|
|
14
|
+
pass
|
|
15
|
+
|
|
16
|
+
@property
|
|
17
|
+
@_abc.abstractmethod
|
|
18
|
+
def _format(self) -> str:
|
|
19
|
+
pass
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
class PngDashboard(Dashboard):
|
|
23
|
+
"""
|
|
24
|
+
Instantiate a Dashboard in PNG format from a matplotlib figure or bytes
|
|
25
|
+
"""
|
|
26
|
+
|
|
27
|
+
def __init__(self, content: _figure.Figure | _io.BytesIO | bytes) -> None:
|
|
28
|
+
"""
|
|
29
|
+
Constructor for PngDashboard
|
|
30
|
+
|
|
31
|
+
Arguments:
|
|
32
|
+
content: The content of the dashboard as a matplotlib.figure.Figure or bytes
|
|
33
|
+
"""
|
|
34
|
+
if isinstance(content, _figure.Figure):
|
|
35
|
+
buffer = _io.BytesIO()
|
|
36
|
+
content.savefig(buffer, format=_c.PNG)
|
|
37
|
+
content = buffer.getvalue()
|
|
38
|
+
|
|
39
|
+
if isinstance(content, _io.BytesIO):
|
|
40
|
+
content = content.getvalue()
|
|
41
|
+
|
|
42
|
+
self.__content = content
|
|
43
|
+
|
|
44
|
+
@property
|
|
45
|
+
def _content(self) -> bytes:
|
|
46
|
+
return self.__content
|
|
47
|
+
|
|
48
|
+
@property
|
|
49
|
+
def _format(self) -> _t.Literal['png']:
|
|
50
|
+
return _c.PNG
|
|
51
|
+
|
|
52
|
+
def _repr_png_(self):
|
|
53
|
+
return self._content
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
class HtmlDashboard(Dashboard):
|
|
57
|
+
"""
|
|
58
|
+
Instantiate a Dashboard from an HTML string
|
|
59
|
+
"""
|
|
60
|
+
|
|
61
|
+
def __init__(self, content: _io.StringIO | str) -> None:
|
|
62
|
+
"""
|
|
63
|
+
Constructor for HtmlDashboard
|
|
64
|
+
|
|
65
|
+
Arguments:
|
|
66
|
+
content: The content of the dashboard as HTML string
|
|
67
|
+
"""
|
|
68
|
+
if isinstance(content, _io.StringIO):
|
|
69
|
+
content = content.getvalue()
|
|
70
|
+
|
|
71
|
+
self.__content = content
|
|
72
|
+
|
|
73
|
+
@property
|
|
74
|
+
def _content(self) -> str:
|
|
75
|
+
return self.__content
|
|
76
|
+
|
|
77
|
+
@property
|
|
78
|
+
def _format(self) -> _t.Literal['html']:
|
|
79
|
+
return _c.HTML
|
|
80
|
+
|
|
81
|
+
def _repr_html_(self):
|
|
82
|
+
return self._content
|