squirrels 0.1.0__py3-none-any.whl → 0.6.0.post0__py3-none-any.whl

squirrels/_package_data/base_project/.env

@@ -0,0 +1,7 @@
+# Custom environment variables
+SQLITE_URI="sqlite:///{project_path}/resources/expenses.db"
+
+# Secrets used by the Squirrels framework that are NOT SAFE TO INCLUDE IN VERSION CONTROL
+# Required if your project uses authentication. Otherwise, optional.
+SQRL_SECRET__KEY="{{ random_secret_key }}"
+SQRL_SECRET__ADMIN_PASSWORD="{{ random_admin_password }}"

squirrels/_package_data/base_project/.env.example

@@ -0,0 +1,44 @@
+# Custom environment variables
+SQLITE_URI="sqlite:///{project_path}/resources/expenses.db"
+
+# Secrets used by the Squirrels framework that are NOT SAFE TO INCLUDE IN VERSION CONTROL
+# Required if your project uses authentication. Otherwise, optional.
+SQRL_SECRET__KEY="" # a random 32 byte hex string - you can generate this is by running `python -c "import secrets; print(secrets.token_hex(32))"`
+SQRL_SECRET__ADMIN_PASSWORD=""
+
+# # Optional variables used by the Squirrels framework that are safe to include in version control if desired
+# # (default values are shown below)
+# SQRL_AUTH__DB_FILE_PATH="{project_path}/target/auth.sqlite"
+# SQRL_AUTH__TOKEN_EXPIRE_MINUTES="30"
+# SQRL_AUTH__ALLOWED_ORIGINS_FOR_COOKIES="https://squirrels-analytics.github.io"
+
+# SQRL_PARAMETERS__CACHE_SIZE="1024"
+# SQRL_PARAMETERS__CACHE_TTL_MINUTES="60"
+# SQRL_PARAMETERS__DATASOURCE_REFRESH_MINUTES="60"
+
+# SQRL_DATASETS__CACHE_SIZE="128"
+# SQRL_DATASETS__CACHE_TTL_MINUTES="60"
+# SQRL_DATASETS__MAX_ROWS_FOR_AI="100"
+# SQRL_DATASETS__MAX_ROWS_OUTPUT="100000"
+# SQRL_DATASETS__SQL_TIMEOUT_SECONDS="2"
+
+# SQRL_DASHBOARDS__CACHE_SIZE="128"
+# SQRL_DASHBOARDS__CACHE_TTL_MINUTES="60"
+
+# SQRL_PERMISSIONS__ELEVATED_ACCESS_LEVEL="admin" # one of "admin", "member", "guest"
+
+# SQRL_SEEDS__INFER_SCHEMA="true"
+# SQRL_SEEDS__NA_VALUES=["NA"] # must be a JSON list
+
+# SQRL_CONNECTIONS__DEFAULT_NAME_USED="default"
+
+# SQRL_VDL__CATALOG_DB_PATH="ducklake:{project_path}/target/vdl_catalog.duckdb"
+# SQRL_VDL__DATA_PATH="{project_path}/target/vdl_data/"
+
+# SQRL_STUDIO__BASE_URL="https://squirrels-analytics.github.io/squirrels-studio-v2"
+
+# SQRL_LOGGING__LEVEL="INFO" # one of "DEBUG", "INFO", "WARNING"
+# SQRL_LOGGING__FORMAT="text"
+# SQRL_LOGGING__TO_FILE="false"
+# SQRL_LOGGING__FILE_SIZE_MB="50"
+# SQRL_LOGGING__FILE_BACKUP_COUNT="1"

squirrels/_package_data/base_project/connections.yml

@@ -0,0 +1,16 @@
+## Connection URIs are usually in format "dialect://username:password@host:port/database" for database connections
+## However, subtle differences exist depending on the "type" specified. For example, sqlite URIs are slightly different.
+##   sqlalchemy:      sqlite:///{project_path}/relative/path/to/database.db
+##   connectorx/adbc: sqlite://{project_path}/relative/path/to/database.db  (adbc URI format matches connectorx)
+##   duckdb:          sqlite:{project_path}/relative/path/to/database.db
+## Refer to specific documentation for supported databases by type (with URI examples):
+##   sqlalchemy: https://docs.sqlalchemy.org/en/latest/core/engines.html#database-urls
+##   connectorx: https://sfu-db.github.io/connector-x/databases.html
+##   adbc:       https://arrow.apache.org/adbc/ (see connectorx documentation for URI examples)
+connections:
+  - name: default
+    label: SQLite Expenses Database
+    type: sqlalchemy                      ## one of: sqlalchemy, connectorx, adbc, or duckdb
+    uri: {{ env_vars.SQLITE_URI }}        ## using Jinja to substitute environment variables
+
+

squirrels/_package_data/base_project/dashboards/dashboard_example.py

@@ -0,0 +1,40 @@
+from squirrels.arguments import DashboardArgs
+from squirrels.dashboards import PngDashboard, HtmlDashboard
+from matplotlib import pyplot as plt, figure as fg, axes as a
+import asyncio
+
+
+async def main(sqrl: DashboardArgs) -> PngDashboard | HtmlDashboard:
+    # Get dataset instances concurrently
+    all_dataframes = await asyncio.gather(
+        sqrl.dataset("grouped_expenses", fixed_parameters={"group_by": "month"}),
+        sqrl.dataset("grouped_expenses", fixed_parameters={"group_by": "subcat"})
+    )
+    spending_by_month_df, spending_by_subcategory_df = all_dataframes
+
+    # Create a figure with two subplots
+    fig, (ax0, ax1) = plt.subplots(2, 1, figsize=(8, 8), height_ratios=(1, 2))
+    fig: fg.Figure; ax0: a.Axes; ax1: a.Axes
+    fig.tight_layout(pad=4, h_pad=6)
+
+    # Create a bar chart of spending by month
+
+    # Convert to pandas and ensure total_amount is numeric
+    spending_by_month_pandas = spending_by_month_df.sort("month").to_pandas()
+    spending_by_month_pandas["total_amount"] = spending_by_month_pandas["total_amount"].astype(float)
+
+    spending_by_month_pandas.plot(x="month", y="total_amount", ax=ax0)
+    ax0.set_title("Spending by Month")
+
+    # Create a pie chart of spending by subcategory
+
+    # Convert to pandas and ensure total_amount is numeric
+    subcategory_pandas = spending_by_subcategory_df.sort("total_amount", descending=True).to_pandas()
+    subcategory_pandas["total_amount"] = subcategory_pandas["total_amount"].astype(float)
+    subcategory_pandas.set_index("subcategory", inplace=True)
+    
+    autopct = lambda pct: ('%.1f%%' % pct) if pct > 6 else ''
+    subcategory_pandas.plot(y="total_amount", kind='pie', ax=ax1, autopct=autopct, legend=False, ylabel="")
+    ax1.set_title("Spending by Subcategory")
+    
+    return PngDashboard(fig)

squirrels/_package_data/base_project/dashboards/dashboard_example.yml

@@ -0,0 +1,22 @@
+label: Expense Dashboard
+
+description: This is a dashboard showing the total expense amounts by month as a line chart and by subcategory as a pie chart
+
+scope: protected
+
+format: png
+
+parameters:
+  - date_range
+  - category
+
+depends_on:
+  - name: dataset_example_month
+    dataset: grouped_expenses
+    fixed_parameters:
+      group_by: month (Month)
+  
+  - name: dataset_example_subcategory
+    dataset: grouped_expenses
+    fixed_parameters:
+      group_by: subcat (Subcategory)

squirrels/_package_data/base_project/docker/.dockerignore

@@ -0,0 +1,16 @@
+**/__pycache__/
+
+# common virtual environment names
+.venv/
+venv/
+
+# squirrels files to ignore
+.env
+.env.local
+duckdb_init.sql
+logs/
+target/
+sqrl_packages/
+
+# additional files for docker to ignore
+.git/

squirrels/_package_data/base_project/docker/Dockerfile

@@ -0,0 +1,16 @@
+# Change here to use different python version (ex. 3.11-slim for version 3.11)
+FROM python:3.12-slim
+WORKDIR /app
+
+COPY . .
+
+# Only needed if there are python dependencies installed using git, or for the
+# "sqrl deps" command if there are packages defined in "squirrels.yml"
+RUN apt-get update && apt-get install -y git
+
+RUN pip install --no-cache-dir -r requirements.txt
+
+RUN sqrl deps
+
+EXPOSE 8000
+CMD ["sqrl", "run", "--build", "--host", "0.0.0.0", "--port", "8000"]

squirrels/_package_data/base_project/docker/compose.yml

@@ -0,0 +1,7 @@
+services:
+  squirrels:
+    build: .
+    ports:
+      - "8000:8000"
+    volumes:
+      - ./.env:/app/.env

squirrels/_package_data/base_project/duckdb_init.sql

@@ -0,0 +1,10 @@
+-- SQL statements that run at the start of every DuckDB session
+-- If the VDL catalog path is provided, the project will ATTACH it as 'vdl' (READ_ONLY)
+
+-- Example:
+
+-- SET threads = 4;
+
+-- SET temp_directory = '/path/to/tmp/';
+
+-- CREATE SECRET (TYPE S3, PROVIDER CREDENTIAL_CHAIN);

squirrels/_package_data/base_project/gitignore

@@ -0,0 +1,13 @@
+**/__pycache__/
+
+# common virtual environment names
+.venv/
+venv/
+
+# squirrels files to ignore
+.env
+.env.local
+duckdb_init.sql
+logs/
+target/
+sqrl_packages/

squirrels/_package_data/base_project/macros/macros_example.sql

@@ -0,0 +1,17 @@
+{%- macro date_and_amount_filters(use_from_range) -%}
+    {%- if use_from_range -%}
+
+    date >= {{ ctx.start_date_from_range | quote }}
+    AND date <= {{ ctx.end_date_from_range | quote }}
+    AND amount >= {{ ctx.min_amount_from_range }}
+    AND amount <= {{ ctx.max_amount_from_range }}
+
+    {%- else -%}
+
+    date >= {{ ctx.start_date | quote }}
+    AND date <= {{ ctx.end_date | quote }}
+    AND amount >= {{ ctx.min_amount }}
+    AND amount <= {{ ctx.max_amount }}
+
+    {%- endif -%}
+{%- endmacro -%}

squirrels/_package_data/base_project/models/builds/build_example.py

@@ -0,0 +1,26 @@
+from squirrels.arguments import BuildModelArgs
+import polars as pl, pandas as pd
+
+
+def main(sqrl: BuildModelArgs) -> pl.LazyFrame | pl.DataFrame | pd.DataFrame:
+    """
+    Create a build model by joining/processing sources or other build models to form a new
+    Python DataFrame (using polars LazyFrame, polars DataFrame, or pandas DataFrame).
+    """
+    # sqrl.ref() can be used on a sources, seeds, or other build models
+    expenses_df = sqrl.ref("src_transactions")
+    categories_df = sqrl.ref("seed_categories")
+    subcategories_df = sqrl.ref("seed_subcategories")
+
+    df = expenses_df \
+        .join(subcategories_df, on="subcategory_id", how="left") \
+        .join(categories_df, on="category_id", how="left")
+
+    df = df.with_columns(
+        pl.col("date").dt.strftime("%Y-%m").alias("month"),
+        pl.col("date").dt.strftime("%Y-%m-%d").alias("date"),
+    )
+
+    return df.select(
+        "id", "date", "month", "category_id", "category", "subcategory_id", "subcategory", "amount", "description"
+    )

squirrels/_package_data/base_project/models/builds/build_example.sql

@@ -0,0 +1,16 @@
+{#- DuckDB dialect -#}
+
+SELECT a.id,
+    STRFTIME(a.date, '%Y-%m-%d') AS date,
+    STRFTIME(a.date, '%Y-%m') AS month,
+    c.category_id,
+    c.category,
+    b.subcategory_id,
+    b.subcategory,
+    a.amount,
+    a.description
+
+{# ref() can be used on a sources, seeds, or other build models -#}
+FROM {{ ref("src_transactions") }} AS a 
+    LEFT JOIN {{ ref("seed_subcategories") }} AS b ON a.subcategory_id = b.subcategory_id
+    LEFT JOIN {{ ref("seed_categories") }} AS c ON b.category_id = c.category_id

squirrels/_package_data/base_project/models/builds/build_example.yml

@@ -0,0 +1,57 @@
+description: |
+  This is an example of a build model. It adds a new column called "month" to the source table "src_transactions".
+
+materialization: TABLE     # optional - defaults to "VIEW" for SQL models, ignored and always a "TABLE" for Python models
+
+depends_on:                # optional for SQL models - the "ref" macro also adds to this set
+  - src_transactions
+  - seed_categories
+  - seed_subcategories
+
+columns:
+  - name: id
+    depends_on:
+      - src_transactions.id
+    pass_through: true
+
+  - name: date
+    type: string
+    description: The day of the transaction as a string in 'YYYY-MM-DD' format
+    depends_on:
+      - src_transactions.date
+
+  - name: month
+    type: string
+    description: The month of the transaction as a string in 'YYYY-MM' format
+    depends_on:
+      - src_transactions.date
+  
+  - name: category_id
+    depends_on:
+      - seed_categories.category_id
+    pass_through: true
+
+  - name: category
+    depends_on:
+      - seed_categories.category
+    pass_through: true
+  
+  - name: subcategory_id
+    depends_on:
+      - seed_subcategories.subcategory_id
+    pass_through: true
+
+  - name: subcategory
+    depends_on:
+      - seed_subcategories.subcategory
+    pass_through: true
+
+  - name: amount
+    depends_on:
+      - src_transactions.amount
+    pass_through: true
+
+  - name: description
+    depends_on:
+      - src_transactions.description
+    pass_through: true

squirrels/_package_data/base_project/models/dbviews/dbview_example.sql

@@ -0,0 +1,17 @@
+{#- SQLite dialect (based on connection used) -#}
+
+SELECT 
+    date,
+    printf('%.2f', amount) as amount,
+    CASE 
+        WHEN '{{ user.custom_fields.role }}' = 'manager' THEN description
+        ELSE '***MASKED***'
+    END as description
+
+FROM {{ source("src_transactions") }}
+
+WHERE {{ date_and_amount_filters(use_from_range=false) }}
+
+GROUP BY 1
+
+ORDER BY 1 DESC

squirrels/_package_data/base_project/models/dbviews/dbview_example.yml

@@ -0,0 +1,32 @@
+description: |
+  This is an example of a database view model. It shows transaction details including date, amount, and description.
+  Description is masked for non-manager users.
+
+  Parameters are available to filter the date and amount of the transactions.
+
+connection: default         # optional - if not provided, will use default connection specified in the SQRL_CONNECTIONS__DEFAULT_NAME_USED setting
+
+translate_to_duckdb: false  # optional - default is false - if true, then the model will be translated to duckdb if dialect is supported by sqlglot
+
+depends_on:                 # optional - Squirrels is able to derive this from the "source" macro in the SQL file
+  - src_transactions
+
+columns:
+  - name: date
+    depends_on:
+      - src_transactions.date
+    pass_through: true
+  
+  - name: amount
+    type: float
+    description: The amount of the transaction, formatted to 2 decimal places
+    category: measure
+    depends_on:
+      - src_transactions.amount
+      
+  - name: description
+    type: string
+    description: The description of the transaction (masked for non-manager users)
+    category: dimension
+    depends_on:
+      - src_transactions.description

squirrels/_package_data/base_project/models/federates/federate_example.py

@@ -0,0 +1,51 @@
+from squirrels.arguments import ModelArgs
+import polars as pl, pandas as pd
+
+
+def main(sqrl: ModelArgs) -> pl.LazyFrame | pl.DataFrame | pd.DataFrame:
+    """
+    Create federated models by joining/processing dependent models (sources, seeds, builds, dbviews, other federates, etc.) to
+    form a new Python DataFrame (using polars LazyFrame, polars DataFrame, or pandas DataFrame).
+    """
+    df = sqrl.ref("build_example")
+
+    df = df.filter(
+        (pl.col("date") >= sqrl.ctx["start_date_from_range"]) &
+        (pl.col("date") <= sqrl.ctx["end_date_from_range"]) &
+        (pl.col("amount") >= sqrl.ctx["min_amount_from_range"]) &
+        (pl.col("amount") <= sqrl.ctx["max_amount_from_range"])
+    )
+
+    if sqrl.ctx.get("has_categories"):
+        categories: list[str] = sqrl.ctx["categories"]
+        df = df.filter(pl.col("category_id").is_in(categories))
+
+    if sqrl.ctx.get("has_subcategories"):
+        subcategories: list[str] = sqrl.ctx["subcategories"]
+        df = df.filter(pl.col("subcategory_id").is_in(subcategories))
+
+    df = df.rename(sqrl.ctx.get("column_to_alias_mapping", {}))
+
+    dimension_cols: list[str] | None = sqrl.ctx.get("group_by_cols")
+    decimal_type = pl.Decimal(precision=15, scale=2)
+    if dimension_cols is not None:
+        df = df.group_by(dimension_cols).agg(
+            pl.sum("amount").cast(decimal_type).alias("total_amount")
+        )
+    else:
+        df = df.with_columns(
+            pl.col("amount").cast(decimal_type).alias("total_amount")
+        )
+    
+    order_by_cols: list[str] = sqrl.ctx.get("order_by_cols")
+    if order_by_cols is not None:
+        df = df.select(*order_by_cols, "total_amount").sort(order_by_cols, descending=True)
+
+    # Apply mask_column_function to description column if it exists
+    mask_column_func = sqrl.ctx.get("mask_column_function")
+    if "description" in order_by_cols and mask_column_func:
+        df = df.with_columns(
+            pl.col("description").map_elements(mask_column_func, return_dtype=pl.String).alias("description")
+        )
+
+    return df

squirrels/_package_data/base_project/models/federates/federate_example.sql

@@ -0,0 +1,21 @@
+{#- DuckDB dialect -#}
+
+SELECT {{ ctx.select_dim_cols | join }}
+    , CAST({{ ctx.aggregator }}(amount) AS DECIMAL(15, 2)) as total_amount
+
+{# ref() can be used on a sources, seeds, builds, dbviews, or other federate models -#}
+FROM {{ ref("build_example") }} AS a
+
+WHERE {{ date_and_amount_filters(use_from_range=true) }}
+{%- if ctx.has_categories %} 
+    AND category_id IN ({{ ctx.categories | quote_and_join }}) 
+{%- endif %}
+{%- if ctx.has_subcategories %} 
+    AND subcategory_id IN ({{ ctx.subcategories | quote_and_join }}) 
+{%- endif %}
+
+{%- if ctx.group_by_cols %}
+GROUP BY {{ ctx.group_by_cols | join }}
+{%- endif %}
+
+ORDER BY {{ ctx.order_by_cols_desc | join }}

squirrels/_package_data/base_project/models/federates/federate_example.yml

@@ -0,0 +1,65 @@
+description: |
+  This is an example of a federate view model. It takes the build example model and groups or filters the results based on the parameter selections provided.
+
+  Parameters are available to specify the group by dimension and filter by date, amount of the transaction, category, and subcategory.
+
+depends_on:                # optional for SQL models - the "ref" macro also adds to this set
+  - build_example
+
+eager: false               # optional - defaults to false. Only applies to SQL models.
+
+columns:
+  - name: date
+    type: string
+    condition: ['group_by == "Transaction"']
+    description: The date of the transaction in 'YYYY-MM-DD' format, in descending order
+    category: dimension
+    depends_on:
+      - build_example.date
+  
+  - name: description
+    type: string
+    condition: ['group_by == "Transaction"']
+    description: The description of the transaction (masked for non-manager users)
+    category: dimension
+    depends_on:
+      - build_example.description
+  
+  - name: day
+    type: string
+    condition: ['group_by == "Day"']
+    description: The day for which the amount is aggregated by, in descending order
+    category: dimension
+    depends_on:
+      - build_example.date
+  
+  - name: month
+    type: string
+    condition: ['group_by == "Month"']
+    description: The month for which the amount is aggregated by, in descending order
+    category: dimension
+    depends_on:
+      - build_example.month
+
+  - name: category
+    type: string
+    condition: ['group_by == "Transaction"', 'group_by == "Category"', 'group_by == "Subcategory"']
+    description: The category for which the amount is aggregated by
+    category: dimension
+    depends_on:
+      - build_example.category
+  
+  - name: subcategory
+    type: string
+    condition: ['group_by == "Transaction"', 'group_by == "Subcategory"']
+    description: The subcategory for which the amount is aggregated by
+    category: dimension
+    depends_on:
+      - build_example.subcategory
+
+  - name: total_amount
+    type: float
+    description: The total amount spent by the group by dimension
+    category: measure
+    depends_on:
+      - build_example.total_amount

squirrels/_package_data/base_project/models/sources.yml

@@ -0,0 +1,38 @@
+sources:
+  - name: src_transactions
+    description: "The source table for transactions"  # optional
+    connection: default           # optional - if not provided, will use the connection named "default" or the default connection specified in settings
+    table: expenses               # optional - if not provided, will use the "name" field of the source
+    load_to_vdl: true             # optional - default is false - other than dbview models (with translate_to_duckdb set to false), other models can only reference this source if load_to_vdl is true or connection type is duckdb
+    
+    update_hints:
+      increasing_column: date     # optional - if not provided, will always do full refresh, otherwise uses this column for incremental loads
+      strictly_increasing: false  # optional - default is true - if false, then maximum value of column is removed before incremental load is performed
+
+    primary_key: [id]             # optional - if not provided, then this is an insert-only table for incremental loads
+    
+    columns:                      # optional - if load_to_vdl is true, then only the columns listed here are loaded to the Virtual Data Lake (VDL)
+      - name: id
+        type: string
+        description: The unique identifier for the transaction
+        category: dimension
+
+      - name: date
+        type: date
+        description: The date of the transaction
+        category: dimension
+
+      - name: subcategory_id
+        type: string
+        description: The ID of the subcategory of the transaction
+        category: dimension
+
+      - name: amount
+        type: float
+        description: The amount of the transaction
+        category: measure
+
+      - name: description
+        type: string
+        description: The description of the transaction
+        category: dimension