squirrels 0.5.0b3__py3-none-any.whl → 0.6.0.post0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- squirrels/__init__.py +4 -0
- squirrels/_api_routes/__init__.py +5 -0
- squirrels/_api_routes/auth.py +337 -0
- squirrels/_api_routes/base.py +196 -0
- squirrels/_api_routes/dashboards.py +156 -0
- squirrels/_api_routes/data_management.py +148 -0
- squirrels/_api_routes/datasets.py +220 -0
- squirrels/_api_routes/project.py +289 -0
- squirrels/_api_server.py +440 -792
- squirrels/_arguments/__init__.py +0 -0
- squirrels/_arguments/{_init_time_args.py → init_time_args.py} +23 -43
- squirrels/_arguments/{_run_time_args.py → run_time_args.py} +32 -68
- squirrels/_auth.py +590 -264
- squirrels/_command_line.py +130 -58
- squirrels/_compile_prompts.py +147 -0
- squirrels/_connection_set.py +16 -15
- squirrels/_constants.py +36 -11
- squirrels/_dashboards.py +179 -0
- squirrels/_data_sources.py +40 -34
- squirrels/_dataset_types.py +16 -11
- squirrels/_env_vars.py +209 -0
- squirrels/_exceptions.py +9 -37
- squirrels/_http_error_responses.py +52 -0
- squirrels/_initializer.py +7 -6
- squirrels/_logging.py +121 -0
- squirrels/_manifest.py +155 -77
- squirrels/_mcp_server.py +578 -0
- squirrels/_model_builder.py +11 -55
- squirrels/_model_configs.py +5 -5
- squirrels/_model_queries.py +1 -1
- squirrels/_models.py +276 -143
- squirrels/_package_data/base_project/.env +1 -24
- squirrels/_package_data/base_project/.env.example +31 -17
- squirrels/_package_data/base_project/connections.yml +4 -3
- squirrels/_package_data/base_project/dashboards/dashboard_example.py +13 -7
- squirrels/_package_data/base_project/dashboards/dashboard_example.yml +6 -6
- squirrels/_package_data/base_project/docker/Dockerfile +2 -2
- squirrels/_package_data/base_project/docker/compose.yml +1 -1
- squirrels/_package_data/base_project/duckdb_init.sql +1 -0
- squirrels/_package_data/base_project/models/builds/build_example.py +2 -2
- squirrels/_package_data/base_project/models/dbviews/dbview_example.sql +7 -2
- squirrels/_package_data/base_project/models/dbviews/dbview_example.yml +16 -10
- squirrels/_package_data/base_project/models/federates/federate_example.py +27 -17
- squirrels/_package_data/base_project/models/federates/federate_example.sql +3 -7
- squirrels/_package_data/base_project/models/federates/federate_example.yml +7 -7
- squirrels/_package_data/base_project/models/sources.yml +5 -6
- squirrels/_package_data/base_project/parameters.yml +24 -38
- squirrels/_package_data/base_project/pyconfigs/connections.py +8 -3
- squirrels/_package_data/base_project/pyconfigs/context.py +26 -14
- squirrels/_package_data/base_project/pyconfigs/parameters.py +124 -81
- squirrels/_package_data/base_project/pyconfigs/user.py +48 -15
- squirrels/_package_data/base_project/resources/public/.gitkeep +0 -0
- squirrels/_package_data/base_project/seeds/seed_categories.yml +1 -1
- squirrels/_package_data/base_project/seeds/seed_subcategories.yml +1 -1
- squirrels/_package_data/base_project/squirrels.yml.j2 +21 -31
- squirrels/_package_data/templates/login_successful.html +53 -0
- squirrels/_package_data/templates/squirrels_studio.html +22 -0
- squirrels/_parameter_configs.py +43 -22
- squirrels/_parameter_options.py +1 -1
- squirrels/_parameter_sets.py +41 -30
- squirrels/_parameters.py +560 -123
- squirrels/_project.py +487 -277
- squirrels/_py_module.py +71 -10
- squirrels/_request_context.py +33 -0
- squirrels/_schemas/__init__.py +0 -0
- squirrels/_schemas/auth_models.py +83 -0
- squirrels/_schemas/query_param_models.py +70 -0
- squirrels/_schemas/request_models.py +26 -0
- squirrels/_schemas/response_models.py +286 -0
- squirrels/_seeds.py +52 -13
- squirrels/_sources.py +29 -23
- squirrels/_utils.py +221 -42
- squirrels/_version.py +1 -3
- squirrels/arguments.py +7 -2
- squirrels/auth.py +4 -0
- squirrels/connections.py +2 -0
- squirrels/dashboards.py +3 -1
- squirrels/data_sources.py +6 -0
- squirrels/parameter_options.py +5 -0
- squirrels/parameters.py +5 -0
- squirrels/types.py +10 -3
- squirrels-0.6.0.post0.dist-info/METADATA +148 -0
- squirrels-0.6.0.post0.dist-info/RECORD +101 -0
- {squirrels-0.5.0b3.dist-info → squirrels-0.6.0.post0.dist-info}/WHEEL +1 -1
- squirrels/_api_response_models.py +0 -190
- squirrels/_dashboard_types.py +0 -82
- squirrels/_dashboards_io.py +0 -79
- squirrels-0.5.0b3.dist-info/METADATA +0 -110
- squirrels-0.5.0b3.dist-info/RECORD +0 -80
- /squirrels/_package_data/base_project/{assets → resources}/expenses.db +0 -0
- /squirrels/_package_data/base_project/{assets → resources}/weather.db +0 -0
- {squirrels-0.5.0b3.dist-info → squirrels-0.6.0.post0.dist-info}/entry_points.txt +0 -0
- {squirrels-0.5.0b3.dist-info → squirrels-0.6.0.post0.dist-info}/licenses/LICENSE +0 -0
|
@@ -1,30 +1,7 @@
|
|
|
1
1
|
# Custom environment variables
|
|
2
|
-
SQLITE_URI="sqlite:///{project_path}/
|
|
2
|
+
SQLITE_URI="sqlite:///{project_path}/resources/expenses.db"
|
|
3
3
|
|
|
4
4
|
# Secrets used by the Squirrels framework that are NOT SAFE TO INCLUDE IN VERSION CONTROL
|
|
5
5
|
# Required if your project uses authentication. Otherwise, optional.
|
|
6
6
|
SQRL_SECRET__KEY="{{ random_secret_key }}"
|
|
7
7
|
SQRL_SECRET__ADMIN_PASSWORD="{{ random_admin_password }}"
|
|
8
|
-
|
|
9
|
-
# Optional variables used by the Squirrels framework that are safe to include in version control if desired
|
|
10
|
-
# (default values are shown below)
|
|
11
|
-
SQRL_AUTH__DB_FILE_PATH="target/auth.sqlite"
|
|
12
|
-
SQRL_AUTH__TOKEN_EXPIRE_MINUTES="30"
|
|
13
|
-
|
|
14
|
-
SQRL_PARAMETERS__CACHE_SIZE="1024"
|
|
15
|
-
SQRL_PARAMETERS__CACHE_TTL_MINUTES="60"
|
|
16
|
-
|
|
17
|
-
SQRL_DATASETS__CACHE_SIZE="128"
|
|
18
|
-
SQRL_DATASETS__CACHE_TTL_MINUTES="60"
|
|
19
|
-
|
|
20
|
-
SQRL_DASHBOARDS__CACHE_SIZE="128"
|
|
21
|
-
SQRL_DASHBOARDS__CACHE_TTL_MINUTES="60"
|
|
22
|
-
|
|
23
|
-
SQRL_SEEDS__INFER_SCHEMA="true"
|
|
24
|
-
SQRL_SEEDS__NA_VALUES=["NA"] # must be a JSON list
|
|
25
|
-
|
|
26
|
-
SQRL_TEST_SETS__DEFAULT_NAME_USED="default"
|
|
27
|
-
|
|
28
|
-
SQRL_CONNECTIONS__DEFAULT_NAME_USED="default"
|
|
29
|
-
|
|
30
|
-
SQRL_DUCKDB_VENV__DB_FILE_PATH="target/venv.duckdb"
|
|
@@ -1,30 +1,44 @@
|
|
|
1
1
|
# Custom environment variables
|
|
2
|
-
SQLITE_URI="sqlite:///{project_path}/
|
|
2
|
+
SQLITE_URI="sqlite:///{project_path}/resources/expenses.db"
|
|
3
3
|
|
|
4
4
|
# Secrets used by the Squirrels framework that are NOT SAFE TO INCLUDE IN VERSION CONTROL
|
|
5
5
|
# Required if your project uses authentication. Otherwise, optional.
|
|
6
|
-
SQRL_SECRET__KEY="" # a random 32 byte hex string -
|
|
6
|
+
SQRL_SECRET__KEY="" # a random 32 byte hex string - you can generate this is by running `python -c "import secrets; print(secrets.token_hex(32))"`
|
|
7
7
|
SQRL_SECRET__ADMIN_PASSWORD=""
|
|
8
8
|
|
|
9
|
-
# Optional variables used by the Squirrels framework that are safe to include in version control if desired
|
|
10
|
-
# (default values are shown below)
|
|
11
|
-
SQRL_AUTH__DB_FILE_PATH="target/auth.sqlite"
|
|
12
|
-
SQRL_AUTH__TOKEN_EXPIRE_MINUTES="30"
|
|
9
|
+
# # Optional variables used by the Squirrels framework that are safe to include in version control if desired
|
|
10
|
+
# # (default values are shown below)
|
|
11
|
+
# SQRL_AUTH__DB_FILE_PATH="{project_path}/target/auth.sqlite"
|
|
12
|
+
# SQRL_AUTH__TOKEN_EXPIRE_MINUTES="30"
|
|
13
|
+
# SQRL_AUTH__ALLOWED_ORIGINS_FOR_COOKIES="https://squirrels-analytics.github.io"
|
|
13
14
|
|
|
14
|
-
SQRL_PARAMETERS__CACHE_SIZE="1024"
|
|
15
|
-
SQRL_PARAMETERS__CACHE_TTL_MINUTES="60"
|
|
15
|
+
# SQRL_PARAMETERS__CACHE_SIZE="1024"
|
|
16
|
+
# SQRL_PARAMETERS__CACHE_TTL_MINUTES="60"
|
|
17
|
+
# SQRL_PARAMETERS__DATASOURCE_REFRESH_MINUTES="60"
|
|
16
18
|
|
|
17
|
-
SQRL_DATASETS__CACHE_SIZE="128"
|
|
18
|
-
SQRL_DATASETS__CACHE_TTL_MINUTES="60"
|
|
19
|
+
# SQRL_DATASETS__CACHE_SIZE="128"
|
|
20
|
+
# SQRL_DATASETS__CACHE_TTL_MINUTES="60"
|
|
21
|
+
# SQRL_DATASETS__MAX_ROWS_FOR_AI="100"
|
|
22
|
+
# SQRL_DATASETS__MAX_ROWS_OUTPUT="100000"
|
|
23
|
+
# SQRL_DATASETS__SQL_TIMEOUT_SECONDS="2"
|
|
19
24
|
|
|
20
|
-
SQRL_DASHBOARDS__CACHE_SIZE="128"
|
|
21
|
-
SQRL_DASHBOARDS__CACHE_TTL_MINUTES="60"
|
|
25
|
+
# SQRL_DASHBOARDS__CACHE_SIZE="128"
|
|
26
|
+
# SQRL_DASHBOARDS__CACHE_TTL_MINUTES="60"
|
|
22
27
|
|
|
23
|
-
|
|
24
|
-
SQRL_SEEDS__NA_VALUES=["NA"] # must be a JSON list
|
|
28
|
+
# SQRL_PERMISSIONS__ELEVATED_ACCESS_LEVEL="admin" # one of "admin", "member", "guest"
|
|
25
29
|
|
|
26
|
-
|
|
30
|
+
# SQRL_SEEDS__INFER_SCHEMA="true"
|
|
31
|
+
# SQRL_SEEDS__NA_VALUES=["NA"] # must be a JSON list
|
|
27
32
|
|
|
28
|
-
SQRL_CONNECTIONS__DEFAULT_NAME_USED="default"
|
|
33
|
+
# SQRL_CONNECTIONS__DEFAULT_NAME_USED="default"
|
|
29
34
|
|
|
30
|
-
|
|
35
|
+
# SQRL_VDL__CATALOG_DB_PATH="ducklake:{project_path}/target/vdl_catalog.duckdb"
|
|
36
|
+
# SQRL_VDL__DATA_PATH="{project_path}/target/vdl_data/"
|
|
37
|
+
|
|
38
|
+
# SQRL_STUDIO__BASE_URL="https://squirrels-analytics.github.io/squirrels-studio-v2"
|
|
39
|
+
|
|
40
|
+
# SQRL_LOGGING__LEVEL="INFO" # one of "DEBUG", "INFO", "WARNING"
|
|
41
|
+
# SQRL_LOGGING__FORMAT="text"
|
|
42
|
+
# SQRL_LOGGING__TO_FILE="false"
|
|
43
|
+
# SQRL_LOGGING__FILE_SIZE_MB="50"
|
|
44
|
+
# SQRL_LOGGING__FILE_BACKUP_COUNT="1"
|
|
@@ -1,7 +1,8 @@
|
|
|
1
1
|
## Connection URIs are usually in format "dialect://username:password@host:port/database" for database connections
|
|
2
2
|
## However, subtle differences exist depending on the "type" specified. For example, sqlite URIs are slightly different.
|
|
3
|
-
## sqlalchemy: sqlite:///relative/path/to/database.db
|
|
4
|
-
## connectorx/adbc: sqlite://relative/path/to/database.db (adbc URI format matches connectorx
|
|
3
|
+
## sqlalchemy: sqlite:///{project_path}/relative/path/to/database.db
|
|
4
|
+
## connectorx/adbc: sqlite://{project_path}/relative/path/to/database.db (adbc URI format matches connectorx)
|
|
5
|
+
## duckdb: sqlite:{project_path}/relative/path/to/database.db
|
|
5
6
|
## Refer to specific documentation for supported databases by type (with URI examples):
|
|
6
7
|
## sqlalchemy: https://docs.sqlalchemy.org/en/latest/core/engines.html#database-urls
|
|
7
8
|
## connectorx: https://sfu-db.github.io/connector-x/databases.html
|
|
@@ -9,7 +10,7 @@
|
|
|
9
10
|
connections:
|
|
10
11
|
- name: default
|
|
11
12
|
label: SQLite Expenses Database
|
|
12
|
-
type: sqlalchemy ## one of: sqlalchemy, connectorx, or
|
|
13
|
+
type: sqlalchemy ## one of: sqlalchemy, connectorx, adbc, or duckdb
|
|
13
14
|
uri: {{ env_vars.SQLITE_URI }} ## using Jinja to substitute environment variables
|
|
14
15
|
|
|
15
16
|
|
|
@@ -1,14 +1,20 @@
|
|
|
1
|
-
from squirrels import
|
|
2
|
-
from
|
|
1
|
+
from squirrels.arguments import DashboardArgs
|
|
2
|
+
from squirrels.dashboards import PngDashboard, HtmlDashboard
|
|
3
|
+
from matplotlib import pyplot as plt, figure as fg, axes as a
|
|
4
|
+
import asyncio
|
|
3
5
|
|
|
4
6
|
|
|
5
|
-
async def main(sqrl:
|
|
6
|
-
|
|
7
|
-
|
|
7
|
+
async def main(sqrl: DashboardArgs) -> PngDashboard | HtmlDashboard:
|
|
8
|
+
# Get dataset instances concurrently
|
|
9
|
+
all_dataframes = await asyncio.gather(
|
|
10
|
+
sqrl.dataset("grouped_expenses", fixed_parameters={"group_by": "month"}),
|
|
11
|
+
sqrl.dataset("grouped_expenses", fixed_parameters={"group_by": "subcat"})
|
|
12
|
+
)
|
|
13
|
+
spending_by_month_df, spending_by_subcategory_df = all_dataframes
|
|
8
14
|
|
|
9
15
|
# Create a figure with two subplots
|
|
10
16
|
fig, (ax0, ax1) = plt.subplots(2, 1, figsize=(8, 8), height_ratios=(1, 2))
|
|
11
|
-
fig:
|
|
17
|
+
fig: fg.Figure; ax0: a.Axes; ax1: a.Axes
|
|
12
18
|
fig.tight_layout(pad=4, h_pad=6)
|
|
13
19
|
|
|
14
20
|
# Create a bar chart of spending by month
|
|
@@ -31,4 +37,4 @@ async def main(sqrl: args.DashboardArgs) -> d.PngDashboard:
|
|
|
31
37
|
subcategory_pandas.plot(y="total_amount", kind='pie', ax=ax1, autopct=autopct, legend=False, ylabel="")
|
|
32
38
|
ax1.set_title("Spending by Subcategory")
|
|
33
39
|
|
|
34
|
-
return
|
|
40
|
+
return PngDashboard(fig)
|
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
label: Dashboard
|
|
1
|
+
label: Expense Dashboard
|
|
2
2
|
|
|
3
|
-
description: This is
|
|
3
|
+
description: This is a dashboard showing the total expense amounts by month as a line chart and by subcategory as a pie chart
|
|
4
4
|
|
|
5
5
|
scope: protected
|
|
6
6
|
|
|
@@ -12,11 +12,11 @@ parameters:
|
|
|
12
12
|
|
|
13
13
|
depends_on:
|
|
14
14
|
- name: dataset_example_month
|
|
15
|
-
dataset:
|
|
15
|
+
dataset: grouped_expenses
|
|
16
16
|
fixed_parameters:
|
|
17
|
-
|
|
17
|
+
group_by: month (Month)
|
|
18
18
|
|
|
19
19
|
- name: dataset_example_subcategory
|
|
20
|
-
dataset:
|
|
20
|
+
dataset: grouped_expenses
|
|
21
21
|
fixed_parameters:
|
|
22
|
-
|
|
22
|
+
group_by: subcat (Subcategory)
|
|
@@ -1,8 +1,8 @@
|
|
|
1
|
-
from squirrels import
|
|
1
|
+
from squirrels.arguments import BuildModelArgs
|
|
2
2
|
import polars as pl, pandas as pd
|
|
3
3
|
|
|
4
4
|
|
|
5
|
-
def main(sqrl:
|
|
5
|
+
def main(sqrl: BuildModelArgs) -> pl.LazyFrame | pl.DataFrame | pd.DataFrame:
|
|
6
6
|
"""
|
|
7
7
|
Create a build model by joining/processing sources or other build models to form a new
|
|
8
8
|
Python DataFrame (using polars LazyFrame, polars DataFrame, or pandas DataFrame).
|
|
@@ -1,7 +1,12 @@
|
|
|
1
1
|
{#- SQLite dialect (based on connection used) -#}
|
|
2
2
|
|
|
3
|
-
SELECT
|
|
4
|
-
,
|
|
3
|
+
SELECT
|
|
4
|
+
date,
|
|
5
|
+
printf('%.2f', amount) as amount,
|
|
6
|
+
CASE
|
|
7
|
+
WHEN '{{ user.custom_fields.role }}' = 'manager' THEN description
|
|
8
|
+
ELSE '***MASKED***'
|
|
9
|
+
END as description
|
|
5
10
|
|
|
6
11
|
FROM {{ source("src_transactions") }}
|
|
7
12
|
|
|
@@ -1,26 +1,32 @@
|
|
|
1
1
|
description: |
|
|
2
|
-
This is an example of a database view model. It
|
|
2
|
+
This is an example of a database view model. It shows transaction details including date, amount, and description.
|
|
3
|
+
Description is masked for non-manager users.
|
|
3
4
|
|
|
4
5
|
Parameters are available to filter the date and amount of the transactions.
|
|
5
6
|
|
|
6
|
-
connection: default
|
|
7
|
+
connection: default # optional - if not provided, will use default connection specified in the SQRL_CONNECTIONS__DEFAULT_NAME_USED setting
|
|
7
8
|
|
|
8
|
-
translate_to_duckdb:
|
|
9
|
+
translate_to_duckdb: false # optional - default is false - if true, then the model will be translated to duckdb if dialect is supported by sqlglot
|
|
9
10
|
|
|
10
|
-
depends_on:
|
|
11
|
+
depends_on: # optional - Squirrels is able to derive this from the "source" macro in the SQL file
|
|
11
12
|
- src_transactions
|
|
12
13
|
|
|
13
14
|
columns:
|
|
14
|
-
- name:
|
|
15
|
-
type: string
|
|
16
|
-
description: The months for which the amount is aggregated by, in descending order
|
|
17
|
-
category: dimension
|
|
15
|
+
- name: date
|
|
18
16
|
depends_on:
|
|
19
17
|
- src_transactions.date
|
|
18
|
+
pass_through: true
|
|
20
19
|
|
|
21
|
-
- name:
|
|
20
|
+
- name: amount
|
|
22
21
|
type: float
|
|
23
|
-
description: The
|
|
22
|
+
description: The amount of the transaction, formatted to 2 decimal places
|
|
24
23
|
category: measure
|
|
25
24
|
depends_on:
|
|
26
25
|
- src_transactions.amount
|
|
26
|
+
|
|
27
|
+
- name: description
|
|
28
|
+
type: string
|
|
29
|
+
description: The description of the transaction (masked for non-manager users)
|
|
30
|
+
category: dimension
|
|
31
|
+
depends_on:
|
|
32
|
+
- src_transactions.description
|
|
@@ -1,8 +1,8 @@
|
|
|
1
|
-
from squirrels import
|
|
1
|
+
from squirrels.arguments import ModelArgs
|
|
2
2
|
import polars as pl, pandas as pd
|
|
3
3
|
|
|
4
4
|
|
|
5
|
-
def main(sqrl:
|
|
5
|
+
def main(sqrl: ModelArgs) -> pl.LazyFrame | pl.DataFrame | pd.DataFrame:
|
|
6
6
|
"""
|
|
7
7
|
Create federated models by joining/processing dependent models (sources, seeds, builds, dbviews, other federates, etc.) to
|
|
8
8
|
form a new Python DataFrame (using polars LazyFrame, polars DataFrame, or pandas DataFrame).
|
|
@@ -16,26 +16,36 @@ def main(sqrl: args.ModelArgs) -> pl.LazyFrame | pl.DataFrame | pd.DataFrame:
|
|
|
16
16
|
(pl.col("amount") <= sqrl.ctx["max_amount_from_range"])
|
|
17
17
|
)
|
|
18
18
|
|
|
19
|
-
if sqrl.ctx
|
|
19
|
+
if sqrl.ctx.get("has_categories"):
|
|
20
20
|
categories: list[str] = sqrl.ctx["categories"]
|
|
21
21
|
df = df.filter(pl.col("category_id").is_in(categories))
|
|
22
22
|
|
|
23
|
-
if sqrl.ctx
|
|
23
|
+
if sqrl.ctx.get("has_subcategories"):
|
|
24
24
|
subcategories: list[str] = sqrl.ctx["subcategories"]
|
|
25
25
|
df = df.filter(pl.col("subcategory_id").is_in(subcategories))
|
|
26
26
|
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
)
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
27
|
+
df = df.rename(sqrl.ctx.get("column_to_alias_mapping", {}))
|
|
28
|
+
|
|
29
|
+
dimension_cols: list[str] | None = sqrl.ctx.get("group_by_cols")
|
|
30
|
+
decimal_type = pl.Decimal(precision=15, scale=2)
|
|
31
|
+
if dimension_cols is not None:
|
|
32
|
+
df = df.group_by(dimension_cols).agg(
|
|
33
|
+
pl.sum("amount").cast(decimal_type).alias("total_amount")
|
|
34
|
+
)
|
|
35
|
+
else:
|
|
36
|
+
df = df.with_columns(
|
|
37
|
+
pl.col("amount").cast(decimal_type).alias("total_amount")
|
|
38
|
+
)
|
|
39
|
+
|
|
40
|
+
order_by_cols: list[str] = sqrl.ctx.get("order_by_cols")
|
|
41
|
+
if order_by_cols is not None:
|
|
42
|
+
df = df.select(*order_by_cols, "total_amount").sort(order_by_cols, descending=True)
|
|
43
|
+
|
|
44
|
+
# Apply mask_column_function to description column if it exists
|
|
45
|
+
mask_column_func = sqrl.ctx.get("mask_column_function")
|
|
46
|
+
if "description" in order_by_cols and mask_column_func:
|
|
47
|
+
df = df.with_columns(
|
|
48
|
+
pl.col("description").map_elements(mask_column_func, return_dtype=pl.String).alias("description")
|
|
49
|
+
)
|
|
40
50
|
|
|
41
51
|
return df
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{#- DuckDB dialect -#}
|
|
2
2
|
|
|
3
3
|
SELECT {{ ctx.select_dim_cols | join }}
|
|
4
|
-
, CAST(
|
|
4
|
+
, CAST({{ ctx.aggregator }}(amount) AS DECIMAL(15, 2)) as total_amount
|
|
5
5
|
|
|
6
6
|
{# ref() can be used on a sources, seeds, builds, dbviews, or other federate models -#}
|
|
7
7
|
FROM {{ ref("build_example") }} AS a
|
|
@@ -14,12 +14,8 @@ WHERE {{ date_and_amount_filters(use_from_range=true) }}
|
|
|
14
14
|
AND subcategory_id IN ({{ ctx.subcategories | quote_and_join }})
|
|
15
15
|
{%- endif %}
|
|
16
16
|
|
|
17
|
+
{%- if ctx.group_by_cols %}
|
|
17
18
|
GROUP BY {{ ctx.group_by_cols | join }}
|
|
19
|
+
{%- endif %}
|
|
18
20
|
|
|
19
21
|
ORDER BY {{ ctx.order_by_cols_desc | join }}
|
|
20
|
-
|
|
21
|
-
{%- if ctx.limit %}
|
|
22
|
-
|
|
23
|
-
LIMIT {{ ctx.limit }}
|
|
24
|
-
|
|
25
|
-
{%- endif %}
|
|
@@ -11,7 +11,7 @@ eager: false # optional - defaults to false. Only applies to SQL m
|
|
|
11
11
|
columns:
|
|
12
12
|
- name: date
|
|
13
13
|
type: string
|
|
14
|
-
condition:
|
|
14
|
+
condition: ['group_by == "Transaction"']
|
|
15
15
|
description: The date of the transaction in 'YYYY-MM-DD' format, in descending order
|
|
16
16
|
category: dimension
|
|
17
17
|
depends_on:
|
|
@@ -19,15 +19,15 @@ columns:
|
|
|
19
19
|
|
|
20
20
|
- name: description
|
|
21
21
|
type: string
|
|
22
|
-
condition:
|
|
23
|
-
description: The description of the transaction
|
|
22
|
+
condition: ['group_by == "Transaction"']
|
|
23
|
+
description: The description of the transaction (masked for non-manager users)
|
|
24
24
|
category: dimension
|
|
25
25
|
depends_on:
|
|
26
26
|
- build_example.description
|
|
27
27
|
|
|
28
28
|
- name: day
|
|
29
29
|
type: string
|
|
30
|
-
condition:
|
|
30
|
+
condition: ['group_by == "Day"']
|
|
31
31
|
description: The day for which the amount is aggregated by, in descending order
|
|
32
32
|
category: dimension
|
|
33
33
|
depends_on:
|
|
@@ -35,7 +35,7 @@ columns:
|
|
|
35
35
|
|
|
36
36
|
- name: month
|
|
37
37
|
type: string
|
|
38
|
-
condition:
|
|
38
|
+
condition: ['group_by == "Month"']
|
|
39
39
|
description: The month for which the amount is aggregated by, in descending order
|
|
40
40
|
category: dimension
|
|
41
41
|
depends_on:
|
|
@@ -43,7 +43,7 @@ columns:
|
|
|
43
43
|
|
|
44
44
|
- name: category
|
|
45
45
|
type: string
|
|
46
|
-
condition:
|
|
46
|
+
condition: ['group_by == "Transaction"', 'group_by == "Category"', 'group_by == "Subcategory"']
|
|
47
47
|
description: The category for which the amount is aggregated by
|
|
48
48
|
category: dimension
|
|
49
49
|
depends_on:
|
|
@@ -51,7 +51,7 @@ columns:
|
|
|
51
51
|
|
|
52
52
|
- name: subcategory
|
|
53
53
|
type: string
|
|
54
|
-
condition:
|
|
54
|
+
condition: ['group_by == "Transaction"', 'group_by == "Subcategory"']
|
|
55
55
|
description: The subcategory for which the amount is aggregated by
|
|
56
56
|
category: dimension
|
|
57
57
|
depends_on:
|
|
@@ -3,16 +3,15 @@ sources:
|
|
|
3
3
|
description: "The source table for transactions" # optional
|
|
4
4
|
connection: default # optional - if not provided, will use the connection named "default" or the default connection specified in settings
|
|
5
5
|
table: expenses # optional - if not provided, will use the "name" field of the source
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
# For performance reasons, avoid specifying primary_key for large tables if upserts are not required
|
|
9
|
-
primary_key: [id] # optional - if not provided, then this is an insert-only table for incremental loads - otherwise, this uses upsert
|
|
6
|
+
load_to_vdl: true # optional - default is false - other than dbview models (with translate_to_duckdb set to false), other models can only reference this source if load_to_vdl is true or connection type is duckdb
|
|
10
7
|
|
|
11
8
|
update_hints:
|
|
12
9
|
increasing_column: date # optional - if not provided, will always do full refresh, otherwise uses this column for incremental loads
|
|
13
|
-
strictly_increasing: false # optional - default is true - if false, then maximum value of column is removed before incremental
|
|
10
|
+
strictly_increasing: false # optional - default is true - if false, then maximum value of column is removed before incremental load is performed
|
|
14
11
|
|
|
15
|
-
|
|
12
|
+
primary_key: [id] # optional - if not provided, then this is an insert-only table for incremental loads
|
|
13
|
+
|
|
14
|
+
columns: # optional - if load_to_vdl is true, then only the columns listed here are loaded to the Virtual Data Lake (VDL)
|
|
16
15
|
- name: id
|
|
17
16
|
type: string
|
|
18
17
|
description: The unique identifier for the transaction
|
|
@@ -5,47 +5,33 @@ parameters:
|
|
|
5
5
|
name: group_by
|
|
6
6
|
label: Group By
|
|
7
7
|
description: Dimension(s) to aggregate by ## optional, default is empty string
|
|
8
|
-
user_attribute:
|
|
8
|
+
user_attribute: access_level ## optional, default is null
|
|
9
9
|
all_options:
|
|
10
10
|
- id: trans
|
|
11
11
|
label: Transaction
|
|
12
12
|
columns: ["id", "date", "category", "subcategory", "description"] ## custom field
|
|
13
13
|
aliases: ["_id", "date", "category", "subcategory", "description"] ## custom field (any alias starting with "_" will not be selected - see context.py for implementation)
|
|
14
|
-
is_default: false ## optional, default
|
|
15
|
-
user_groups: ["
|
|
16
|
-
parent_option_ids: [] ## optional, default
|
|
14
|
+
is_default: false ## optional, shown is default - exists for SingleSelect or MultiSelect options only
|
|
15
|
+
user_groups: ["admin"] ## optional, default is empty list
|
|
16
|
+
parent_option_ids: [] ## optional, shown is default - exists for all parameter options
|
|
17
17
|
- id: day
|
|
18
18
|
label: Day
|
|
19
19
|
columns: [date]
|
|
20
20
|
aliases: [day]
|
|
21
|
-
user_groups: ["
|
|
21
|
+
user_groups: ["admin", "member"]
|
|
22
22
|
- id: month
|
|
23
23
|
label: Month
|
|
24
24
|
columns: [month]
|
|
25
|
-
user_groups: ["
|
|
25
|
+
user_groups: ["admin", "member", "guest"]
|
|
26
26
|
- id: cat
|
|
27
27
|
label: Category
|
|
28
28
|
columns: [category]
|
|
29
|
-
user_groups: ["
|
|
29
|
+
user_groups: ["admin", "member", "guest"]
|
|
30
30
|
- id: subcat
|
|
31
31
|
label: Subcategory
|
|
32
32
|
columns: [category, subcategory]
|
|
33
|
-
user_groups: ["
|
|
34
|
-
parent_name: null ## optional, default
|
|
35
|
-
|
|
36
|
-
- type: NumberParameter
|
|
37
|
-
factory: CreateWithOptions
|
|
38
|
-
arguments:
|
|
39
|
-
name: limit
|
|
40
|
-
label: Max Number of Rows
|
|
41
|
-
description: Maximum number of rows to return
|
|
42
|
-
parent_name: group_by
|
|
43
|
-
all_options:
|
|
44
|
-
- min_value: 0
|
|
45
|
-
max_value: 1000
|
|
46
|
-
increment: 10
|
|
47
|
-
default_value: 1000
|
|
48
|
-
parent_option_ids: trans
|
|
33
|
+
user_groups: ["admin", "member", "guest"]
|
|
34
|
+
parent_name: null ## optional, shown is default - exists for all parameter types
|
|
49
35
|
|
|
50
36
|
- type: DateParameter
|
|
51
37
|
factory: CreateFromSource
|
|
@@ -92,14 +78,14 @@ parameters:
|
|
|
92
78
|
table_or_query: seed_categories
|
|
93
79
|
id_col: category_id
|
|
94
80
|
options_col: category
|
|
95
|
-
|
|
96
|
-
order_by_col: null ## optional, default
|
|
97
|
-
is_default_col: null ## optional, default
|
|
98
|
-
custom_cols: {} ## optional, default
|
|
99
|
-
include_all: true ## optional, default
|
|
100
|
-
order_matters: false ## optional, default
|
|
101
|
-
user_group_col: null ## optional, default
|
|
102
|
-
connection_name: default ## optional, default
|
|
81
|
+
source: seeds ## optional, default is "connection" - must be one of "connection", "seeds", or "vdl" - exists for data_source of any parameters
|
|
82
|
+
order_by_col: null ## optional, shown is default - exists for data_source of SingleSelect and MultiSelect
|
|
83
|
+
is_default_col: null ## optional, shown is default - exists for data_source of SingleSelect and MultiSelect
|
|
84
|
+
custom_cols: {} ## optional, shown is default - exists for data_source of SingleSelect and MultiSelect
|
|
85
|
+
include_all: true ## optional, shown is default - exists for data_source of MultiSelect only
|
|
86
|
+
order_matters: false ## optional, shown is default - exists for data_source of MultiSelect only
|
|
87
|
+
user_group_col: null ## optional, shown is default - exists for data_source of any parameters
|
|
88
|
+
connection_name: default ## optional, shown is default - exists for data_source of any parameters
|
|
103
89
|
|
|
104
90
|
- type: MultiSelectParameter
|
|
105
91
|
factory: CreateFromSource
|
|
@@ -112,8 +98,8 @@ parameters:
|
|
|
112
98
|
table_or_query: seed_subcategories
|
|
113
99
|
id_col: subcategory_id
|
|
114
100
|
options_col: subcategory
|
|
115
|
-
|
|
116
|
-
parent_id_col: category_id ## optional, default is null
|
|
101
|
+
source: seeds
|
|
102
|
+
parent_id_col: category_id ## optional, default is null - exists for all parameter types
|
|
117
103
|
|
|
118
104
|
- type: NumberParameter
|
|
119
105
|
factory: CreateWithOptions
|
|
@@ -124,8 +110,8 @@ parameters:
|
|
|
124
110
|
all_options:
|
|
125
111
|
- min_value: 0
|
|
126
112
|
max_value: 300
|
|
127
|
-
increment: 10 ## optional, default is 1
|
|
128
|
-
default_value: null ## optional, default
|
|
113
|
+
increment: 10 ## optional, default is 1 - exists for Number and NumberRange options
|
|
114
|
+
default_value: null ## optional, shown is default - exists for Number options only
|
|
129
115
|
|
|
130
116
|
- type: NumberParameter
|
|
131
117
|
factory: CreateFromSource
|
|
@@ -139,7 +125,7 @@ parameters:
|
|
|
139
125
|
max_value_col: max_value
|
|
140
126
|
increment_col: increment ## optional, default is null
|
|
141
127
|
default_value_col: max_value ## optional, default is null
|
|
142
|
-
id_col: null ## optional, default
|
|
128
|
+
id_col: null ## optional, shown is default - required for SingleSelect and MultiSelect, optional for others
|
|
143
129
|
|
|
144
130
|
- type: NumberRangeParameter
|
|
145
131
|
factory: CreateWithOptions
|
|
@@ -150,7 +136,7 @@ parameters:
|
|
|
150
136
|
all_options:
|
|
151
137
|
- min_value: 0
|
|
152
138
|
max_value: 300
|
|
153
|
-
default_lower_value: 0 ## optional, default is null (or min_value)
|
|
154
|
-
default_upper_value: 300 ## optional, default is null (or max_value)
|
|
139
|
+
default_lower_value: 0 ## optional, default is null (or min_value) - exists for NumberRange options only
|
|
140
|
+
default_upper_value: 300 ## optional, default is null (or max_value) - exists for NumberRange options only
|
|
155
141
|
|
|
156
142
|
|
|
@@ -1,8 +1,9 @@
|
|
|
1
1
|
from typing import Any
|
|
2
|
-
from squirrels import
|
|
2
|
+
from squirrels.arguments import ConnectionsArgs
|
|
3
|
+
from squirrels.connections import ConnectionProperties, ConnectionTypeEnum
|
|
3
4
|
|
|
4
5
|
|
|
5
|
-
def main(connections: dict[str,
|
|
6
|
+
def main(connections: dict[str, ConnectionProperties | Any], sqrl: ConnectionsArgs) -> None:
|
|
6
7
|
"""
|
|
7
8
|
Define sqlalchemy engines by adding them to the "connections" dictionary
|
|
8
9
|
"""
|
|
@@ -10,5 +11,9 @@ def main(connections: dict[str, cn.ConnectionProperties | Any], sqrl: args.Conne
|
|
|
10
11
|
conn_str: str = sqrl.env_vars["SQLITE_URI"].format(project_path=sqrl.project_path)
|
|
11
12
|
|
|
12
13
|
## Assigning names to connection engines
|
|
13
|
-
connections["default"] =
|
|
14
|
+
connections["default"] = ConnectionProperties(
|
|
15
|
+
label="SQLite Expenses Database",
|
|
16
|
+
type=ConnectionTypeEnum.SQLALCHEMY,
|
|
17
|
+
uri=conn_str
|
|
18
|
+
)
|
|
14
19
|
|