squirrels 0.1.1.post1__py3-none-any.whl → 0.2.0.dev0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of squirrels might be problematic. Click here for more details.
- squirrels/__init__.py +10 -16
- squirrels/_api_server.py +234 -80
- squirrels/_authenticator.py +84 -0
- squirrels/_command_line.py +60 -72
- squirrels/_connection_set.py +96 -0
- squirrels/_constants.py +114 -33
- squirrels/_environcfg.py +77 -0
- squirrels/_initializer.py +126 -67
- squirrels/_manifest.py +195 -168
- squirrels/_models.py +495 -0
- squirrels/_package_loader.py +26 -0
- squirrels/_parameter_configs.py +401 -0
- squirrels/_parameter_sets.py +188 -0
- squirrels/_py_module.py +60 -0
- squirrels/_timer.py +36 -0
- squirrels/_utils.py +81 -49
- squirrels/_version.py +2 -2
- squirrels/arguments/init_time_args.py +32 -0
- squirrels/arguments/run_time_args.py +82 -0
- squirrels/data_sources.py +380 -155
- squirrels/dateutils.py +86 -57
- squirrels/package_data/base_project/Dockerfile +15 -0
- squirrels/package_data/base_project/connections.yml +7 -0
- squirrels/package_data/base_project/database/{sample_database.db → expenses.db} +0 -0
- squirrels/package_data/base_project/environcfg.yml +29 -0
- squirrels/package_data/base_project/ignores/.dockerignore +8 -0
- squirrels/package_data/base_project/ignores/.gitignore +7 -0
- squirrels/package_data/base_project/models/dbviews/database_view1.py +36 -0
- squirrels/package_data/base_project/models/dbviews/database_view1.sql +15 -0
- squirrels/package_data/base_project/models/federates/dataset_example.py +20 -0
- squirrels/package_data/base_project/models/federates/dataset_example.sql +3 -0
- squirrels/package_data/base_project/parameters.yml +109 -0
- squirrels/package_data/base_project/pyconfigs/auth.py +47 -0
- squirrels/package_data/base_project/pyconfigs/connections.py +28 -0
- squirrels/package_data/base_project/pyconfigs/context.py +45 -0
- squirrels/package_data/base_project/pyconfigs/parameters.py +55 -0
- squirrels/package_data/base_project/seeds/mocks/category.csv +3 -0
- squirrels/package_data/base_project/seeds/mocks/max_filter.csv +2 -0
- squirrels/package_data/base_project/seeds/mocks/subcategory.csv +6 -0
- squirrels/package_data/base_project/squirrels.yml.j2 +57 -0
- squirrels/package_data/base_project/tmp/.gitignore +2 -0
- squirrels/package_data/static/script.js +159 -63
- squirrels/package_data/static/style.css +79 -15
- squirrels/package_data/static/widgets.js +133 -0
- squirrels/package_data/templates/index.html +65 -23
- squirrels/package_data/templates/index2.html +22 -0
- squirrels/parameter_options.py +216 -119
- squirrels/parameters.py +407 -478
- squirrels/user_base.py +58 -0
- squirrels-0.2.0.dev0.dist-info/METADATA +126 -0
- squirrels-0.2.0.dev0.dist-info/RECORD +56 -0
- {squirrels-0.1.1.post1.dist-info → squirrels-0.2.0.dev0.dist-info}/WHEEL +1 -2
- squirrels-0.2.0.dev0.dist-info/entry_points.txt +3 -0
- squirrels/_credentials_manager.py +0 -87
- squirrels/_module_loader.py +0 -37
- squirrels/_parameter_set.py +0 -151
- squirrels/_renderer.py +0 -286
- squirrels/_timed_imports.py +0 -37
- squirrels/connection_set.py +0 -126
- squirrels/package_data/base_project/.gitignore +0 -4
- squirrels/package_data/base_project/connections.py +0 -20
- squirrels/package_data/base_project/datasets/sample_dataset/context.py +0 -22
- squirrels/package_data/base_project/datasets/sample_dataset/database_view1.py +0 -29
- squirrels/package_data/base_project/datasets/sample_dataset/database_view1.sql.j2 +0 -12
- squirrels/package_data/base_project/datasets/sample_dataset/final_view.py +0 -11
- squirrels/package_data/base_project/datasets/sample_dataset/final_view.sql.j2 +0 -3
- squirrels/package_data/base_project/datasets/sample_dataset/parameters.py +0 -47
- squirrels/package_data/base_project/datasets/sample_dataset/selections.cfg +0 -9
- squirrels/package_data/base_project/squirrels.yaml +0 -22
- squirrels-0.1.1.post1.dist-info/METADATA +0 -67
- squirrels-0.1.1.post1.dist-info/RECORD +0 -40
- squirrels-0.1.1.post1.dist-info/entry_points.txt +0 -2
- squirrels-0.1.1.post1.dist-info/top_level.txt +0 -1
- {squirrels-0.1.1.post1.dist-info → squirrels-0.2.0.dev0.dist-info}/LICENSE +0 -0
squirrels/dateutils.py
CHANGED
|
@@ -1,10 +1,11 @@
|
|
|
1
|
-
from typing import Sequence
|
|
1
|
+
from typing import Sequence, Type
|
|
2
2
|
from dataclasses import dataclass
|
|
3
|
-
from datetime import datetime
|
|
3
|
+
from datetime import date as Date, datetime
|
|
4
4
|
from dateutil.relativedelta import relativedelta
|
|
5
|
+
from abc import ABCMeta, abstractmethod
|
|
5
6
|
from enum import Enum
|
|
6
7
|
|
|
7
|
-
from
|
|
8
|
+
from . import _utils as u
|
|
8
9
|
|
|
9
10
|
|
|
10
11
|
class DayOfWeek(Enum):
|
|
@@ -31,13 +32,13 @@ class Month(Enum):
|
|
|
31
32
|
December = 12
|
|
32
33
|
|
|
33
34
|
|
|
34
|
-
|
|
35
|
-
class DateModifier:
|
|
35
|
+
class DateModifier(metaclass=ABCMeta):
|
|
36
36
|
"""
|
|
37
37
|
Interface for all Date modification classes, and declares a "modify" method
|
|
38
38
|
"""
|
|
39
39
|
|
|
40
|
-
|
|
40
|
+
@abstractmethod
|
|
41
|
+
def modify(self, date: Date) -> Date:
|
|
41
42
|
"""
|
|
42
43
|
Method to be overwritten, modifies the input date
|
|
43
44
|
|
|
@@ -47,19 +48,21 @@ class DateModifier:
|
|
|
47
48
|
Returns:
|
|
48
49
|
The modified date.
|
|
49
50
|
"""
|
|
50
|
-
|
|
51
|
+
pass
|
|
52
|
+
|
|
53
|
+
def _get_date(self, datetype: Type, year: int, month: int, day: int) -> Date:
|
|
54
|
+
return datetype(year, month, day)
|
|
51
55
|
|
|
52
56
|
|
|
53
|
-
@dataclass
|
|
54
57
|
class _DayIdxOfCalendarUnit(DateModifier):
|
|
55
58
|
"""
|
|
56
59
|
Interface for adjusting a date to some day of calendar unit
|
|
57
60
|
"""
|
|
58
|
-
idx: int
|
|
59
|
-
|
|
60
|
-
|
|
61
|
+
def __init__(self, idx: int) -> None:
|
|
62
|
+
super().__init__()
|
|
63
|
+
self.idx = idx
|
|
61
64
|
if self.idx == 0:
|
|
62
|
-
raise
|
|
65
|
+
raise u.ConfigurationError(f"For constructors of class names that start with DayIdxOf_, idx cannot be zero")
|
|
63
66
|
self.incr = self.idx - 1 if self.idx > 0 else self.idx
|
|
64
67
|
|
|
65
68
|
|
|
@@ -73,21 +76,23 @@ class DayIdxOfMonthsCycle(_DayIdxOfCalendarUnit):
|
|
|
73
76
|
num_months_in_cycle: 2 for one 6th of year, 3 for Quarter, 4 for one 3rd of year, 6 for half year, 12 for full year. Must fit evenly in 12
|
|
74
77
|
first_month_of_cycle: The first month of months cycle of year. Default is January
|
|
75
78
|
"""
|
|
76
|
-
|
|
77
|
-
|
|
79
|
+
_num_months_in_cycle: int
|
|
80
|
+
_first_month_of_cycle: Month
|
|
78
81
|
|
|
79
|
-
def
|
|
80
|
-
super().
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
82
|
+
def __init__(self, idx: int, num_months_in_cycle: int, first_month_of_cycle: Month = Month.January) -> None:
|
|
83
|
+
super().__init__(idx)
|
|
84
|
+
self._num_months_in_cycle = num_months_in_cycle
|
|
85
|
+
self._first_month_of_cycle = first_month_of_cycle
|
|
86
|
+
if 12 % self._num_months_in_cycle != 0:
|
|
87
|
+
raise u.ConfigurationError(f"Value X must fit evenly in 12")
|
|
88
|
+
self.first_month_of_first_cycle = (self._first_month_of_cycle.value - 1) % self._num_months_in_cycle + 1
|
|
84
89
|
|
|
85
|
-
def modify(self, date:
|
|
86
|
-
current_cycle = (date.month - self.first_month_of_first_cycle) % 12 // self.
|
|
87
|
-
first_month_of_curr_cycle = current_cycle * self.
|
|
90
|
+
def modify(self, date: Date) -> Date:
|
|
91
|
+
current_cycle = (date.month - self.first_month_of_first_cycle) % 12 // self._num_months_in_cycle
|
|
92
|
+
first_month_of_curr_cycle = current_cycle * self._num_months_in_cycle + self.first_month_of_first_cycle
|
|
88
93
|
year = date.year if date.month >= first_month_of_curr_cycle else date.year - 1
|
|
89
|
-
first_day =
|
|
90
|
-
ref_date = first_day if self.idx > 0 else first_day + relativedelta(months=self.
|
|
94
|
+
first_day = self._get_date(type(date), year, first_month_of_curr_cycle, 1)
|
|
95
|
+
ref_date = first_day if self.idx > 0 else first_day + relativedelta(months=self._num_months_in_cycle)
|
|
91
96
|
return ref_date + relativedelta(days=self.incr)
|
|
92
97
|
|
|
93
98
|
|
|
@@ -128,8 +133,11 @@ class DayIdxOfMonth(_DayIdxOfCalendarUnit):
|
|
|
128
133
|
idx: 1 for first, 2 for second, etc. Or, -1 for last, -2 for second last, etc. Must not be 0
|
|
129
134
|
"""
|
|
130
135
|
|
|
131
|
-
def
|
|
132
|
-
|
|
136
|
+
def __init__(self, idx: int) -> None:
|
|
137
|
+
super().__init__(idx)
|
|
138
|
+
|
|
139
|
+
def modify(self, date: Date) -> Date:
|
|
140
|
+
first_day = self._get_date(type(date), date.year, date.month, 1)
|
|
133
141
|
ref_date = first_day if self.idx > 0 else first_day + relativedelta(months=1)
|
|
134
142
|
return ref_date + relativedelta(days=self.incr)
|
|
135
143
|
|
|
@@ -143,24 +151,26 @@ class DayIdxOfWeek(_DayIdxOfCalendarUnit):
|
|
|
143
151
|
idx: 1 for first, 2 for second, etc. Or, -1 for last, -2 for second last, etc. Must not be 0
|
|
144
152
|
first_day_of_week: The day of week identified as the "first". Default is Monday
|
|
145
153
|
"""
|
|
146
|
-
|
|
154
|
+
_first_day_of_week: DayOfWeek
|
|
147
155
|
|
|
148
|
-
def
|
|
149
|
-
super().
|
|
150
|
-
self.
|
|
156
|
+
def __init__(self, idx: int, first_day_of_week: DayOfWeek = DayOfWeek.Monday) -> None:
|
|
157
|
+
super().__init__(idx)
|
|
158
|
+
self._first_day_of_week = first_day_of_week
|
|
159
|
+
self.first_dow_num = self._first_day_of_week.value
|
|
151
160
|
|
|
152
|
-
def modify(self, date:
|
|
161
|
+
def modify(self, date: Date) -> Date:
|
|
153
162
|
distance_from_first_day = (1 + date.weekday() - self.first_dow_num) % 7
|
|
154
163
|
total_incr = -distance_from_first_day + (7 if self.idx < 0 else 0) + self.incr
|
|
155
164
|
return date + relativedelta(days=total_incr)
|
|
156
165
|
|
|
157
166
|
|
|
158
|
-
@dataclass
|
|
159
167
|
class _OffsetUnits(DateModifier):
|
|
160
168
|
"""
|
|
161
169
|
Abstract DateModifier class to offset an input date by some number of some calendar unit
|
|
162
170
|
"""
|
|
163
|
-
offset: int
|
|
171
|
+
def __init__(self, offset: int) -> None:
|
|
172
|
+
super().__init__()
|
|
173
|
+
self.offset = offset
|
|
164
174
|
|
|
165
175
|
|
|
166
176
|
@dataclass
|
|
@@ -172,7 +182,10 @@ class OffsetYears(_OffsetUnits):
|
|
|
172
182
|
offset: The number of years to offset the input date.
|
|
173
183
|
"""
|
|
174
184
|
|
|
175
|
-
def
|
|
185
|
+
def __init__(self, offset: int) -> None:
|
|
186
|
+
super().__init__(offset)
|
|
187
|
+
|
|
188
|
+
def modify(self, date: Date) -> Date:
|
|
176
189
|
return date + relativedelta(years=self.offset)
|
|
177
190
|
|
|
178
191
|
|
|
@@ -185,7 +198,10 @@ class OffsetMonths(_OffsetUnits):
|
|
|
185
198
|
offset: The number of months to offset the input date.
|
|
186
199
|
"""
|
|
187
200
|
|
|
188
|
-
def
|
|
201
|
+
def __init__(self, offset: int) -> None:
|
|
202
|
+
super().__init__(offset)
|
|
203
|
+
|
|
204
|
+
def modify(self, date: Date) -> Date:
|
|
189
205
|
return date + relativedelta(months=self.offset)
|
|
190
206
|
|
|
191
207
|
|
|
@@ -198,7 +214,10 @@ class OffsetWeeks(_OffsetUnits):
|
|
|
198
214
|
offset: The number of weeks to offset the input date.
|
|
199
215
|
"""
|
|
200
216
|
|
|
201
|
-
def
|
|
217
|
+
def __init__(self, offset: int) -> None:
|
|
218
|
+
super().__init__(offset)
|
|
219
|
+
|
|
220
|
+
def modify(self, date: Date) -> Date:
|
|
202
221
|
return date + relativedelta(weeks=self.offset)
|
|
203
222
|
|
|
204
223
|
|
|
@@ -211,7 +230,10 @@ class OffsetDays(_OffsetUnits):
|
|
|
211
230
|
offset: The number of days to offset the input date.
|
|
212
231
|
"""
|
|
213
232
|
|
|
214
|
-
def
|
|
233
|
+
def __init__(self, offset: int) -> None:
|
|
234
|
+
super().__init__(offset)
|
|
235
|
+
|
|
236
|
+
def modify(self, date: Date) -> Date:
|
|
215
237
|
return date + relativedelta(days=self.offset)
|
|
216
238
|
|
|
217
239
|
|
|
@@ -223,13 +245,14 @@ class DateModPipeline(DateModifier):
|
|
|
223
245
|
Attributes:
|
|
224
246
|
modifiers: The list of DateModifier's to apply in sequence.
|
|
225
247
|
"""
|
|
226
|
-
|
|
248
|
+
_date_modifiers: Sequence[DateModifier]
|
|
227
249
|
|
|
228
|
-
def
|
|
229
|
-
|
|
250
|
+
def __init__(self, date_modifiers: Sequence[DateModifier]) -> None:
|
|
251
|
+
super().__init__()
|
|
252
|
+
self._date_modifiers = tuple(date_modifiers)
|
|
230
253
|
|
|
231
|
-
def modify(self, date:
|
|
232
|
-
for modifier in self.
|
|
254
|
+
def modify(self, date: Date) -> Date:
|
|
255
|
+
for modifier in self._date_modifiers:
|
|
233
256
|
date = modifier.modify(date)
|
|
234
257
|
return date
|
|
235
258
|
|
|
@@ -244,7 +267,7 @@ class DateModPipeline(DateModifier):
|
|
|
244
267
|
Returns:
|
|
245
268
|
A new sequence of DateModifier
|
|
246
269
|
"""
|
|
247
|
-
joined_modifiers = self.
|
|
270
|
+
joined_modifiers = self._date_modifiers + tuple(date_modifiers)
|
|
248
271
|
return joined_modifiers
|
|
249
272
|
|
|
250
273
|
def with_more_modifiers(self, date_modifiers: Sequence[DateModifier]):
|
|
@@ -260,7 +283,7 @@ class DateModPipeline(DateModifier):
|
|
|
260
283
|
joined_modifiers = self.get_joined_modifiers(date_modifiers)
|
|
261
284
|
return DateModPipeline(joined_modifiers)
|
|
262
285
|
|
|
263
|
-
def get_date_list(self, start_date:
|
|
286
|
+
def get_date_list(self, start_date: Date, step: _OffsetUnits) -> Sequence[Date]:
|
|
264
287
|
"""
|
|
265
288
|
This method modifies the input date, and returns all dates from the input date to the modified date,
|
|
266
289
|
incremented by a DateModifier step.
|
|
@@ -278,7 +301,7 @@ class DateModPipeline(DateModifier):
|
|
|
278
301
|
A list of datetime objects
|
|
279
302
|
"""
|
|
280
303
|
if step.offset == 0:
|
|
281
|
-
raise
|
|
304
|
+
raise u.ConfigurationError(f"The length of 'step' must not be zero")
|
|
282
305
|
|
|
283
306
|
output = []
|
|
284
307
|
end_date = self.modify(start_date)
|
|
@@ -291,16 +314,16 @@ class DateModPipeline(DateModifier):
|
|
|
291
314
|
return output
|
|
292
315
|
|
|
293
316
|
|
|
294
|
-
|
|
295
|
-
class _DateRepresentationModifier:
|
|
317
|
+
class _DateRepresentationModifier(metaclass=ABCMeta):
|
|
296
318
|
"""
|
|
297
319
|
Abstract class for modifying other representations of dates (such as string or unix timestemp)
|
|
298
320
|
"""
|
|
299
321
|
def __init__(self, date_modifiers: Sequence[DateModifier]):
|
|
300
322
|
self.date_modifier = DateModPipeline(date_modifiers)
|
|
301
323
|
|
|
324
|
+
@abstractmethod
|
|
302
325
|
def with_more_modifiers(self, date_modifiers: Sequence[DateModifier]):
|
|
303
|
-
|
|
326
|
+
pass
|
|
304
327
|
|
|
305
328
|
|
|
306
329
|
@dataclass
|
|
@@ -312,9 +335,12 @@ class DateStringModifier(_DateRepresentationModifier):
|
|
|
312
335
|
date_modifier: The DateModifier to apply on datetime objects
|
|
313
336
|
date_format: Format of the output date string. Default is '%Y-%m-%d'
|
|
314
337
|
"""
|
|
338
|
+
_date_modifiers: Sequence[DateModifier]
|
|
339
|
+
_date_format: str
|
|
340
|
+
|
|
315
341
|
def __init__(self, date_modifiers: Sequence[DateModifier], date_format: str = '%Y-%m-%d'):
|
|
316
342
|
super().__init__(date_modifiers)
|
|
317
|
-
self.
|
|
343
|
+
self._date_format = date_format
|
|
318
344
|
|
|
319
345
|
def with_more_modifiers(self, date_modifiers: Sequence[DateModifier]):
|
|
320
346
|
"""
|
|
@@ -327,11 +353,11 @@ class DateStringModifier(_DateRepresentationModifier):
|
|
|
327
353
|
A new DateStringModifier
|
|
328
354
|
"""
|
|
329
355
|
joined_modifiers = self.date_modifier.get_joined_modifiers(date_modifiers)
|
|
330
|
-
return DateStringModifier(joined_modifiers, self.
|
|
356
|
+
return DateStringModifier(joined_modifiers, self._date_format)
|
|
331
357
|
|
|
332
|
-
def _get_input_date_obj(self, date_str: str, input_format: str = None) ->
|
|
333
|
-
input_format = self.
|
|
334
|
-
return datetime.strptime(date_str, input_format)
|
|
358
|
+
def _get_input_date_obj(self, date_str: str, input_format: str = None) -> Date:
|
|
359
|
+
input_format = self._date_format if input_format is None else input_format
|
|
360
|
+
return datetime.strptime(date_str, input_format).date()
|
|
335
361
|
|
|
336
362
|
def modify(self, date_str: str, input_format: str = None) -> str:
|
|
337
363
|
"""
|
|
@@ -345,7 +371,7 @@ class DateStringModifier(_DateRepresentationModifier):
|
|
|
345
371
|
The resulting date string
|
|
346
372
|
"""
|
|
347
373
|
date_obj = self._get_input_date_obj(date_str, input_format)
|
|
348
|
-
return self.date_modifier.modify(date_obj).strftime(self.
|
|
374
|
+
return self.date_modifier.modify(date_obj).strftime(self._date_format)
|
|
349
375
|
|
|
350
376
|
def get_date_list(self, start_date_str: str, step: DateModifier, input_format: str = None) -> Sequence[str]:
|
|
351
377
|
"""
|
|
@@ -367,7 +393,7 @@ class DateStringModifier(_DateRepresentationModifier):
|
|
|
367
393
|
"""
|
|
368
394
|
curr_date = self._get_input_date_obj(start_date_str, input_format)
|
|
369
395
|
output = self.date_modifier.get_date_list(curr_date, step)
|
|
370
|
-
return [x.strftime(self.
|
|
396
|
+
return [x.strftime(self._date_format) for x in output]
|
|
371
397
|
|
|
372
398
|
|
|
373
399
|
@dataclass
|
|
@@ -379,6 +405,8 @@ class TimestampModifier(_DateRepresentationModifier):
|
|
|
379
405
|
date_modifier: The DateModifier to apply on datetime objects
|
|
380
406
|
date_format: Format of the date string. Default is '%Y-%m-%d'
|
|
381
407
|
"""
|
|
408
|
+
_date_modifiers: Sequence[DateModifier]
|
|
409
|
+
|
|
382
410
|
def __init__(self, date_modifiers: Sequence[DateModifier]):
|
|
383
411
|
super().__init__(date_modifiers)
|
|
384
412
|
|
|
@@ -406,7 +434,8 @@ class TimestampModifier(_DateRepresentationModifier):
|
|
|
406
434
|
The resulting timestamp
|
|
407
435
|
"""
|
|
408
436
|
date_obj = datetime.fromtimestamp(timestamp)
|
|
409
|
-
|
|
437
|
+
modified_date: datetime = self.date_modifier.modify(date_obj)
|
|
438
|
+
return modified_date.timestamp()
|
|
410
439
|
|
|
411
440
|
def get_date_list(self, start_timestamp: float, step: DateModifier) -> Sequence[float]:
|
|
412
441
|
"""
|
|
@@ -426,5 +455,5 @@ class TimestampModifier(_DateRepresentationModifier):
|
|
|
426
455
|
A list of timestamp as floats
|
|
427
456
|
"""
|
|
428
457
|
curr_date = datetime.fromtimestamp(start_timestamp)
|
|
429
|
-
output = self.date_modifier.get_date_list(curr_date, step)
|
|
458
|
+
output: Sequence[datetime] = self.date_modifier.get_date_list(curr_date, step)
|
|
430
459
|
return [x.timestamp() for x in output]
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
# Change here to use different python version (ex. 3.12-slim for version 3.12)
|
|
2
|
+
FROM python:3.11-slim
|
|
3
|
+
WORKDIR /app
|
|
4
|
+
|
|
5
|
+
# Needed if any python dependencies are installed from git, or for
|
|
6
|
+
# "squirrels load-modules" if there are modules in "squirrels.yml"
|
|
7
|
+
RUN apt-get update && apt-get install -y git
|
|
8
|
+
|
|
9
|
+
COPY requirements-lock.txt .
|
|
10
|
+
RUN pip install --no-cache-dir -r requirements-lock.txt
|
|
11
|
+
|
|
12
|
+
COPY . .
|
|
13
|
+
RUN squirrels load-modules
|
|
14
|
+
|
|
15
|
+
CMD ["squirrels", "run", "--host", "0.0.0.0", "--port", "4465"]
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
## Uses SQLAlchemy URLs. More details here: https://docs.sqlalchemy.org/en/latest/core/engines.html
|
|
2
|
+
db_connections:
|
|
3
|
+
- connection_name: default
|
|
4
|
+
credential_key: null
|
|
5
|
+
url: "sqlite://{username}:{password}@/./database/expenses.db" ## or use {{ sqlite_conn_str }} with the environcfg.yml file
|
|
6
|
+
|
|
7
|
+
|
|
Binary file
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
## Note: You can copy this file to the .squirrels folder in your home directory to make
|
|
2
|
+
## the configurations global for all squirrels projects on the current machine
|
|
3
|
+
|
|
4
|
+
## Fake users for local development testing
|
|
5
|
+
users:
|
|
6
|
+
johndoe:
|
|
7
|
+
is_internal: True
|
|
8
|
+
password: I<3Squirrels
|
|
9
|
+
full_name: John Doe
|
|
10
|
+
organization: org1
|
|
11
|
+
mattdoe:
|
|
12
|
+
is_internal: False
|
|
13
|
+
password: abcd5678
|
|
14
|
+
full_name: Matt Doe
|
|
15
|
+
organization: org2
|
|
16
|
+
|
|
17
|
+
## Custom environment variables / secrets
|
|
18
|
+
env_vars:
|
|
19
|
+
sqlite_conn_str: sqlite://{username}:{password}@/./database/expenses.db
|
|
20
|
+
|
|
21
|
+
## Database credentials
|
|
22
|
+
credentials:
|
|
23
|
+
db_user:
|
|
24
|
+
username: user1
|
|
25
|
+
password: pass1
|
|
26
|
+
|
|
27
|
+
## Predefined secrets used by the squirrels framework
|
|
28
|
+
secrets:
|
|
29
|
+
jwt_signature: ## generate a random 32 byte hex string here for jwt signatures. For instance, you can run "openssl rand -hex 32" in bash
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
from textwrap import dedent
|
|
2
|
+
import pandas as pd, squirrels as sr
|
|
3
|
+
|
|
4
|
+
|
|
5
|
+
def main(sqrl: sr.ModelArgs) -> pd.DataFrame:
|
|
6
|
+
"""
|
|
7
|
+
Create a database view model in Python by sending an external query to a database or API, and return a
|
|
8
|
+
pandas DataFrame of the result in this function. Since the result is loaded into server memory, be mindful of
|
|
9
|
+
the size of the results coming from the external query.
|
|
10
|
+
"""
|
|
11
|
+
|
|
12
|
+
""" If working with sqlalchemy ORMs, use 'sqrl.connections' to get a connection pool / engine """
|
|
13
|
+
# from typing import Union
|
|
14
|
+
# from sqlalchemy import Engine, Pool
|
|
15
|
+
# engine1: Union[Engine, Pool] = sqrl.connections[sqrl.connection_name] ## using the pre-assigned key
|
|
16
|
+
# engine2: Union[Engine, Pool] = sqrl.connections["my_connection_name"] ## or use any defined key
|
|
17
|
+
|
|
18
|
+
""" Example with building and running a sql query """
|
|
19
|
+
category_clause = f'AND Category IN ({sqrl.ctx["categories"]})\n' if sqrl.ctx["has_categories"] else ''
|
|
20
|
+
subcategory_clause = f'AND Subcategory IN ({sqrl.ctx["subcategories"]})\n' if sqrl.ctx["has_subcategories"] else ''
|
|
21
|
+
query = dedent(f"""
|
|
22
|
+
SELECT {sqrl.ctx["group_by_cols"]}
|
|
23
|
+
, sum(-Amount) as Total_Amount
|
|
24
|
+
FROM transactions
|
|
25
|
+
WHERE 1=1
|
|
26
|
+
{category_clause}{subcategory_clause}AND "Date" >= {sqrl.ctx["start_date"]}
|
|
27
|
+
AND "Date" <= {sqrl.ctx["end_date"]}
|
|
28
|
+
AND -Amount >= {sqrl.ctx["min_amount"]}
|
|
29
|
+
AND -Amount <= {sqrl.ctx["max_amount"]}
|
|
30
|
+
GROUP BY {sqrl.ctx["group_by_cols"]}
|
|
31
|
+
""").strip()
|
|
32
|
+
|
|
33
|
+
return sqrl.run_external_sql(query)
|
|
34
|
+
|
|
35
|
+
""" A 'connection_name' argument is available to use a different connection key """
|
|
36
|
+
# return sqrl.run_external_sql(query, connection_name="different_key")
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
SELECT {{ ctx["group_by_cols"] }}
|
|
2
|
+
, sum(-Amount) as Total_Amount
|
|
3
|
+
FROM transactions
|
|
4
|
+
WHERE 1=1
|
|
5
|
+
{%- if ctx["has_categories"] %}
|
|
6
|
+
AND Category IN ({{ ctx["categories"] }})
|
|
7
|
+
{%- endif %}
|
|
8
|
+
{%- if ctx["has_subcategories"] %}
|
|
9
|
+
AND Subcategory IN ({{ ctx["subcategories"] }})
|
|
10
|
+
{%- endif %}
|
|
11
|
+
AND "Date" >= {{ ctx["start_date"] }}
|
|
12
|
+
AND "Date" <= {{ ctx["end_date"] }}
|
|
13
|
+
AND -Amount >= {{ ctx["min_amount"] }}
|
|
14
|
+
AND -Amount <= {{ ctx["max_amount"] }}
|
|
15
|
+
GROUP BY {{ ctx["group_by_cols"] }}
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
from typing import Iterable
|
|
2
|
+
import pandas as pd, squirrels as sr
|
|
3
|
+
|
|
4
|
+
|
|
5
|
+
def dependencies(sqrl: sr.ModelDepsArgs) -> Iterable[str]:
|
|
6
|
+
"""
|
|
7
|
+
Define list of dependent models here. This will determine the dependencies first, at compile-time,
|
|
8
|
+
before running the model.
|
|
9
|
+
"""
|
|
10
|
+
return ["database_view1"]
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
def main(sqrl: sr.ModelArgs) -> pd.DataFrame:
|
|
14
|
+
"""
|
|
15
|
+
Create federated models by joining/processing dependent database views and/or other federated models to
|
|
16
|
+
form and return the result as a new pandas DataFrame.
|
|
17
|
+
"""
|
|
18
|
+
df = sqrl.ref("database_view1")
|
|
19
|
+
group_by_cols: str = sqrl.ctx["group_by_cols_list"]
|
|
20
|
+
return df.sort_values(group_by_cols, ascending=False)
|
|
@@ -0,0 +1,109 @@
|
|
|
1
|
+
parameters:
|
|
2
|
+
- name: group_by
|
|
3
|
+
type: SingleSelectParameter
|
|
4
|
+
factory: Create ## one of 'Create', 'CreateSimple', or 'CreateFromSource'
|
|
5
|
+
arguments: ## arguments to specify depend on values for 'type' and 'factory'
|
|
6
|
+
label: Group By
|
|
7
|
+
all_options:
|
|
8
|
+
- id: g0
|
|
9
|
+
label: Transaction
|
|
10
|
+
columns: [ID, Date] ## custom field
|
|
11
|
+
is_default: false ## optional, default, exists for SingleSelect or MultiSelect options only
|
|
12
|
+
user_groups: [] ## optional, default, exists for all parameter options
|
|
13
|
+
parent_option_ids: [] ## optional, default, exists for all parameter options
|
|
14
|
+
- id: g1
|
|
15
|
+
label: Date
|
|
16
|
+
columns: [Date]
|
|
17
|
+
- id: g2
|
|
18
|
+
label: Category
|
|
19
|
+
columns: [Category]
|
|
20
|
+
- id: g3
|
|
21
|
+
label: Subcategory
|
|
22
|
+
columns: [Category, Subcategory]
|
|
23
|
+
is_hidden: false ## optional, default, exists for all parameter types
|
|
24
|
+
user_attribute: null ## optional, default, exists for all parameter types
|
|
25
|
+
parent_name: null ## optional, default, exists for all parameter types
|
|
26
|
+
- name: start_date
|
|
27
|
+
type: DateParameter
|
|
28
|
+
factory: Create
|
|
29
|
+
arguments:
|
|
30
|
+
label: Start Date
|
|
31
|
+
all_options:
|
|
32
|
+
- default_date: 2023-01-01
|
|
33
|
+
date_format: '%Y-%m-%d' ## optional, default, format comes from python datetime, exists for Date and DateRange parameter options
|
|
34
|
+
- name: end_date
|
|
35
|
+
type: DateParameter
|
|
36
|
+
factory: Create
|
|
37
|
+
arguments:
|
|
38
|
+
label: End Date
|
|
39
|
+
all_options:
|
|
40
|
+
- default_date: 2023-12-31
|
|
41
|
+
- name: date_range
|
|
42
|
+
type: DateRangeParameter
|
|
43
|
+
factory: Create
|
|
44
|
+
arguments:
|
|
45
|
+
label: Date Range
|
|
46
|
+
all_options:
|
|
47
|
+
- default_start_date: 2023-01-01
|
|
48
|
+
default_end_date: 2023-12-31
|
|
49
|
+
- name: category
|
|
50
|
+
type: MultiSelectParameter
|
|
51
|
+
factory: CreateFromSource
|
|
52
|
+
arguments:
|
|
53
|
+
label: Category Filter
|
|
54
|
+
data_source:
|
|
55
|
+
table_or_query: categories
|
|
56
|
+
id_col: Category_ID
|
|
57
|
+
options_col: Category
|
|
58
|
+
order_by_col: null ## optional, default, exists for SingleSelect and MultiSelect
|
|
59
|
+
is_default_col: null ## optional, default, exists for SingleSelect and MultiSelect
|
|
60
|
+
custom_cols: {} ## optional, default, exists for SingleSelect and MultiSelect
|
|
61
|
+
include_all: true ## optional, default, exists for MultiSelect only
|
|
62
|
+
order_matters: false ## optional, default, exists for MultiSelect only
|
|
63
|
+
user_group_col: null ## optional, default, exists for all parameters
|
|
64
|
+
connection_name: default ## optional, default, exists for all parameters
|
|
65
|
+
- name: subcategory
|
|
66
|
+
type: MultiSelectParameter
|
|
67
|
+
factory: CreateFromSource
|
|
68
|
+
arguments:
|
|
69
|
+
label: Subcategory Filter
|
|
70
|
+
data_source:
|
|
71
|
+
table_or_query: subcategories
|
|
72
|
+
id_col: Subcategory_ID
|
|
73
|
+
options_col: Subcategory
|
|
74
|
+
parent_id_col: Category_ID ## optional, default is null, exists for all parameter types
|
|
75
|
+
parent_name: category
|
|
76
|
+
- name: min_filter
|
|
77
|
+
type: NumberParameter
|
|
78
|
+
factory: Create
|
|
79
|
+
arguments:
|
|
80
|
+
label: Amounts Greater Than
|
|
81
|
+
all_options:
|
|
82
|
+
- min_value: 0
|
|
83
|
+
max_value: 500
|
|
84
|
+
increment: 1 ## optional, default, exists for Number and NumRange options
|
|
85
|
+
default_value: null ## optional, default, exists for Number options only
|
|
86
|
+
- name: max_filter
|
|
87
|
+
type: NumberParameter
|
|
88
|
+
factory: CreateFromSource
|
|
89
|
+
arguments:
|
|
90
|
+
label: Amounts Less Than
|
|
91
|
+
data_source:
|
|
92
|
+
table_or_query: "SELECT 0 as min_value, max(-Amount) as max_value, 10 as increment FROM transactions WHERE Category <> 'Income'"
|
|
93
|
+
min_value_col: min_value
|
|
94
|
+
max_value_col: max_value
|
|
95
|
+
increment_col: increment ## optional, default is null
|
|
96
|
+
default_value_col: max_value ## optional, default is null
|
|
97
|
+
id_col: null ## optional, default, required for SingleSelect and MultiSelect, optional for all others
|
|
98
|
+
- name: between_filter
|
|
99
|
+
type: NumberRangeParameter
|
|
100
|
+
factory: Create
|
|
101
|
+
arguments:
|
|
102
|
+
label: Amounts Between
|
|
103
|
+
all_options:
|
|
104
|
+
- min_value: 0
|
|
105
|
+
max_value: 500
|
|
106
|
+
default_lower_value: 10 ## optional, default is null (or min_value), exists for NumRange options only
|
|
107
|
+
default_upper_value: 400 ## optional, default is null (or max_value), exists for NumRange options only
|
|
108
|
+
|
|
109
|
+
|
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
from typing import Union, Any
|
|
3
|
+
from squirrels import User as UserBase, WrongPassword
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
class User(UserBase):
|
|
7
|
+
def set_attributes(self, user_dict: dict[str, Any]) -> None:
|
|
8
|
+
"""
|
|
9
|
+
Use this method to add custom attributes in the User model that don't exist in UserBase (username, is_internal, etc.)
|
|
10
|
+
"""
|
|
11
|
+
self.organization = user_dict["organization"]
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
def get_user_if_valid(username: str, password: str) -> Union[User, WrongPassword, None]:
|
|
15
|
+
"""
|
|
16
|
+
This function allows the squirrels framework to know how to authenticate input username and password.
|
|
17
|
+
|
|
18
|
+
Return:
|
|
19
|
+
- User instance - if username and password are correct
|
|
20
|
+
- WrongPassword(username) - if username exists but password is incorrect
|
|
21
|
+
- None - if the username doesn't exist (and continue username search among "fake users" in environcfg.yml)
|
|
22
|
+
"""
|
|
23
|
+
mock_users_db = {
|
|
24
|
+
"johndoe": {
|
|
25
|
+
"username": "johndoe",
|
|
26
|
+
"is_admin": True,
|
|
27
|
+
"organization": "org1",
|
|
28
|
+
"hashed_password": str(hash("I<3Squirrels"))
|
|
29
|
+
},
|
|
30
|
+
"mattdoe": {
|
|
31
|
+
"username": "mattdoe",
|
|
32
|
+
"is_admin": False,
|
|
33
|
+
"organization": "org2",
|
|
34
|
+
"hashed_password": str(hash("abcd5678"))
|
|
35
|
+
}
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
if username in mock_users_db:
|
|
39
|
+
user_dict = mock_users_db[username]
|
|
40
|
+
hashed_pwd = user_dict["hashed_password"]
|
|
41
|
+
if str(hash(password)) == hashed_pwd:
|
|
42
|
+
is_admin = user_dict["is_admin"]
|
|
43
|
+
user = User(username, is_internal=is_admin)
|
|
44
|
+
return user.with_attributes(user_dict)
|
|
45
|
+
else:
|
|
46
|
+
return WrongPassword(username)
|
|
47
|
+
return None
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
from typing import Union
|
|
2
|
+
from sqlalchemy import create_engine, Engine, Pool
|
|
3
|
+
import squirrels as sr
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
def main(connections: dict[str, Union[Engine, Pool]], sqrl: sr.ConnectionsArgs) -> None:
|
|
7
|
+
"""
|
|
8
|
+
Define connections by adding sqlalchemy engines or connection pools to the dictionary "connections".
|
|
9
|
+
Note that all connections in a pool should be shareable across threads. No writes should be occuring on them.
|
|
10
|
+
"""
|
|
11
|
+
|
|
12
|
+
""" Example of getting the username and password """
|
|
13
|
+
# username, password = sqrl.get_credential('my_key')
|
|
14
|
+
|
|
15
|
+
""" SQLAlchemy URL for a connection pool / engine """
|
|
16
|
+
conn_str = 'sqlite:///./database/expenses.db'
|
|
17
|
+
|
|
18
|
+
""" Can also leverage the environcfg.yml file for connection details """
|
|
19
|
+
# conn_str_raw: str = sqrl.env_vars["sqlite_conn_str"]
|
|
20
|
+
# conn_str = conn_str_raw.format(username=username, password=password)
|
|
21
|
+
|
|
22
|
+
""" Assigning names to connection pool / engine """
|
|
23
|
+
connections["default"] = create_engine(conn_str)
|
|
24
|
+
|
|
25
|
+
""" Example of using QueuePool to use a custom db connector without a SQLAlchemy URL """
|
|
26
|
+
# from sqlalchemy import QueuePool
|
|
27
|
+
# connection_creator = lambda: sqlite3.connect("./database/expenses.db", check_same_thread=False)
|
|
28
|
+
# connections["default"] = QueuePool(connection_creator)
|