policyengine-household-analytics 0.29.6__tar.gz

policyengine_household_analytics-0.29.6/.gitignore

@@ -0,0 +1,16 @@
+**/__pycache__
+*.egg-info
+.pytest_cache
+.mypy_cache
+.vscode
+**/*.db
+**/*.db-journal
+dist/*
+**/*.rdb
+**/*.h5
+**/*.csv.gz
+.env
+.ds_store
+
+# Generated by modal-deploy-release.sh for Modal image builds
+requirements-modal-*.txt

policyengine_household_analytics-0.29.6/PKG-INFO

@@ -0,0 +1,27 @@
+Metadata-Version: 2.4
+Name: policyengine-household-analytics
+Version: 0.29.6
+Summary: Analytics contract and persistence for the PolicyEngine Household API
+Author-email: PolicyEngine <hello@policyengine.org>
+Requires-Python: >=3.12
+Requires-Dist: alembic>=1.13.0
+Requires-Dist: cloud-sql-python-connector
+Requires-Dist: flask-sqlalchemy>=3
+Requires-Dist: flask>=2.2
+Requires-Dist: policyengine-household-common==0.29.6
+Requires-Dist: pymysql
+Requires-Dist: sqlalchemy>=2
+Description-Content-Type: text/markdown
+
+# policyengine-household-analytics
+
+Analytics contract and persistence for the PolicyEngine Household API: the
+calculate-analytics event schema, SQLAlchemy ORM models, analytics database
+setup, and the Alembic migration scripts (shipped as package data).
+
+The slim Cloud Run analytics writer installs this lib, so its dependency
+closure deliberately excludes numpy, country model packages, and modal.
+
+Published to PyPI because `policyengine-household-api` depends on it. It is
+not a standalone product; its API follows the needs of the Household API
+services.

policyengine_household_analytics-0.29.6/README.md

@@ -0,0 +1,12 @@
+# policyengine-household-analytics
+
+Analytics contract and persistence for the PolicyEngine Household API: the
+calculate-analytics event schema, SQLAlchemy ORM models, analytics database
+setup, and the Alembic migration scripts (shipped as package data).
+
+The slim Cloud Run analytics writer installs this lib, so its dependency
+closure deliberately excludes numpy, country model packages, and modal.
+
+Published to PyPI because `policyengine-household-api` depends on it. It is
+not a standalone product; its API follows the needs of the Household API
+services.

policyengine_household_analytics-0.29.6/policyengine_household_analytics/alembic/env.py

@@ -0,0 +1,90 @@
+"""Alembic environment for the household API analytics database."""
+
+from __future__ import annotations
+
+from logging.config import fileConfig
+
+from alembic import context
+from sqlalchemy import create_engine, pool, text
+
+from policyengine_household_analytics.analytics_setup import (  # noqa: E402
+    get_analytics_database_uri,
+    getconn,
+)
+from policyengine_household_analytics.analytics_setup import db  # noqa: E402
+import policyengine_household_analytics.orm  # noqa: F401,E402
+
+
+config = context.config
+
+if config.config_file_name is not None:
+    fileConfig(config.config_file_name, disable_existing_loggers=False)
+
+target_metadata = db.metadata
+
+
+def _database_uri() -> str:
+    x_args = context.get_x_argument(as_dictionary=True)
+    return x_args.get("database_url") or get_analytics_database_uri()
+
+
+def run_migrations_offline() -> None:
+    context.configure(
+        url=_database_uri(),
+        target_metadata=target_metadata,
+        literal_binds=True,
+        dialect_opts={"paramstyle": "named"},
+    )
+
+    with context.begin_transaction():
+        context.run_migrations()
+
+
+def _run_migrations_with_lock(connection) -> None:
+    context.configure(
+        connection=connection,
+        target_metadata=target_metadata,
+        render_as_batch=connection.dialect.name == "sqlite",
+    )
+
+    if connection.dialect.name != "mysql":
+        with context.begin_transaction():
+            context.run_migrations()
+        return
+
+    lock_name = "policyengine_household_api_alembic"
+    lock_acquired = connection.execute(
+        text("SELECT GET_LOCK(:lock_name, 30)"),
+        {"lock_name": lock_name},
+    ).scalar()
+    if lock_acquired != 1:
+        raise RuntimeError("Could not acquire analytics migration lock")
+
+    try:
+        with context.begin_transaction():
+            context.run_migrations()
+    finally:
+        connection.execute(
+            text("SELECT RELEASE_LOCK(:lock_name)"),
+            {"lock_name": lock_name},
+        )
+
+
+def run_migrations_online() -> None:
+    database_uri = _database_uri()
+    engine_kwargs = {"poolclass": pool.NullPool}
+    if database_uri == "mysql+pymysql://":
+        engine_kwargs["creator"] = lambda: getconn(
+            require_analytics_enabled=False
+        )
+
+    connectable = create_engine(database_uri, **engine_kwargs)
+
+    with connectable.connect() as connection:
+        _run_migrations_with_lock(connection)
+
+
+if context.is_offline_mode():
+    run_migrations_offline()
+else:
+    run_migrations_online()

policyengine_household_analytics-0.29.6/policyengine_household_analytics/alembic/script.py.mako

@@ -0,0 +1,24 @@
+"""${message}
+
+Revision ID: ${up_revision}
+Revises: ${down_revision | comma,n}
+Create Date: ${create_date}
+"""
+
+from alembic import op
+import sqlalchemy as sa
+${imports if imports else ""}
+
+
+revision = ${repr(up_revision)}
+down_revision = ${repr(down_revision)}
+branch_labels = ${repr(branch_labels)}
+depends_on = ${repr(depends_on)}
+
+
+def upgrade() -> None:
+    ${upgrades if upgrades else "pass"}
+
+
+def downgrade() -> None:
+    ${downgrades if downgrades else "pass"}

policyengine_household_analytics-0.29.6/policyengine_household_analytics/alembic/versions/20260508_0001_existing_visits_baseline.py

@@ -0,0 +1,32 @@
+"""Baseline existing visits analytics table.
+
+Revision ID: 20260508_0001
+Revises:
+Create Date: 2026-05-08 00:00:00
+"""
+
+from alembic import op
+import sqlalchemy as sa
+
+
+revision = "20260508_0001"
+down_revision = None
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    op.create_table(
+        "visits",
+        sa.Column("id", sa.Integer(), primary_key=True),
+        sa.Column("client_id", sa.String(length=255), nullable=False),
+        sa.Column("datetime", sa.DateTime(), nullable=True),
+        sa.Column("api_version", sa.String(length=32), nullable=True),
+        sa.Column("endpoint", sa.String(length=64), nullable=True),
+        sa.Column("method", sa.String(length=32), nullable=True),
+        sa.Column("content_length_bytes", sa.Integer(), nullable=True),
+    )
+
+
+def downgrade() -> None:
+    op.drop_table("visits")

policyengine_household_analytics-0.29.6/policyengine_household_analytics/alembic/versions/20260508_0002_calculate_variable_usage.py

@@ -0,0 +1,143 @@
+"""calculate variable usage
+
+Revision ID: 20260508_0002
+Revises: 20260508_0001
+Create Date: 2026-05-08 20:05:06.418279
+"""
+
+from alembic import op
+import sqlalchemy as sa
+
+
+revision = "20260508_0002"
+down_revision = "20260508_0001"
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.create_table(
+        "calculate_requests",
+        sa.Column("id", sa.Integer(), nullable=False),
+        sa.Column("visit_id", sa.Integer(), nullable=False),
+        sa.Column("request_uuid", sa.String(length=36), nullable=False),
+        sa.Column("client_id", sa.String(length=255), nullable=True),
+        sa.Column("api_version", sa.String(length=32), nullable=True),
+        sa.Column("country_id", sa.String(length=16), nullable=False),
+        sa.Column("model_version", sa.String(length=64), nullable=True),
+        sa.Column("endpoint", sa.String(length=64), nullable=True),
+        sa.Column("method", sa.String(length=16), nullable=False),
+        sa.Column("content_length_bytes", sa.Integer(), nullable=True),
+        sa.Column("response_status_code", sa.Integer(), nullable=True),
+        sa.Column("distinct_variable_count", sa.Integer(), nullable=False),
+        sa.Column("unsupported_variable_count", sa.Integer(), nullable=False),
+        sa.Column(
+            "deprecated_allowlisted_variable_count",
+            sa.Integer(),
+            nullable=False,
+        ),
+        sa.Column("created_at", sa.DateTime(), nullable=False),
+        sa.ForeignKeyConstraint(
+            ["visit_id"],
+            ["visits.id"],
+        ),
+        sa.PrimaryKeyConstraint("id"),
+        sa.UniqueConstraint("request_uuid"),
+    )
+    with op.batch_alter_table("calculate_requests", schema=None) as batch_op:
+        batch_op.create_index(
+            "ix_calculate_requests_client_created",
+            ["client_id", "created_at"],
+            unique=False,
+        )
+        batch_op.create_index(
+            "ix_calculate_requests_country_created",
+            ["country_id", "created_at"],
+            unique=False,
+        )
+        batch_op.create_index(
+            "ix_calculate_requests_visit_id", ["visit_id"], unique=False
+        )
+
+    op.create_table(
+        "calculate_request_variables",
+        sa.Column("id", sa.Integer(), nullable=False),
+        sa.Column("request_id", sa.Integer(), nullable=False),
+        sa.Column("client_id", sa.String(length=255), nullable=True),
+        sa.Column("created_at", sa.DateTime(), nullable=False),
+        sa.Column("country_id", sa.String(length=16), nullable=False),
+        sa.Column("api_version", sa.String(length=32), nullable=True),
+        sa.Column("model_version", sa.String(length=64), nullable=True),
+        sa.Column("response_status_code", sa.Integer(), nullable=True),
+        sa.Column("variable_name", sa.String(length=255), nullable=False),
+        sa.Column("entity_type", sa.String(length=64), nullable=False),
+        sa.Column("source", sa.String(length=32), nullable=False),
+        sa.Column("period_granularity", sa.String(length=16), nullable=False),
+        sa.Column("entity_count", sa.Integer(), nullable=False),
+        sa.Column("period_count", sa.Integer(), nullable=False),
+        sa.Column("occurrence_count", sa.Integer(), nullable=False),
+        sa.Column("availability_status", sa.String(length=32), nullable=False),
+        sa.ForeignKeyConstraint(
+            ["request_id"],
+            ["calculate_requests.id"],
+        ),
+        sa.PrimaryKeyConstraint("id"),
+        sa.UniqueConstraint(
+            "request_id",
+            "variable_name",
+            "entity_type",
+            "source",
+            name="ux_calc_vars_request_variable_entity_source",
+        ),
+    )
+    with op.batch_alter_table(
+        "calculate_request_variables", schema=None
+    ) as batch_op:
+        batch_op.create_index(
+            "ix_calc_vars_client_variable_created",
+            ["client_id", "variable_name", "created_at"],
+            unique=False,
+        )
+        batch_op.create_index(
+            "ix_calc_vars_country_model_variable",
+            ["country_id", "model_version", "variable_name"],
+            unique=False,
+        )
+        batch_op.create_index(
+            "ix_calc_vars_variable_created",
+            ["variable_name", "created_at"],
+            unique=False,
+        )
+
+    with op.batch_alter_table("visits", schema=None) as batch_op:
+        batch_op.alter_column(
+            "client_id", existing_type=sa.VARCHAR(length=255), nullable=True
+        )
+
+    # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.execute("UPDATE visits SET client_id = '' WHERE client_id IS NULL")
+    with op.batch_alter_table("visits", schema=None) as batch_op:
+        batch_op.alter_column(
+            "client_id", existing_type=sa.VARCHAR(length=255), nullable=False
+        )
+
+    with op.batch_alter_table(
+        "calculate_request_variables", schema=None
+    ) as batch_op:
+        batch_op.drop_index("ix_calc_vars_variable_created")
+        batch_op.drop_index("ix_calc_vars_country_model_variable")
+        batch_op.drop_index("ix_calc_vars_client_variable_created")
+
+    op.drop_table("calculate_request_variables")
+    with op.batch_alter_table("calculate_requests", schema=None) as batch_op:
+        batch_op.drop_index("ix_calculate_requests_visit_id")
+        batch_op.drop_index("ix_calculate_requests_country_created")
+        batch_op.drop_index("ix_calculate_requests_client_created")
+
+    op.drop_table("calculate_requests")
+    # ### end Alembic commands ###

policyengine_household_analytics-0.29.6/policyengine_household_analytics/alembic/versions/20260512_0003_cap_analytics_variable_names.py

@@ -0,0 +1,76 @@
+"""cap analytics variable names
+
+Downgrade note:
+    This migration is not data-preserving for variable names longer than
+    250 characters. Those names are capped before storage in this revision,
+    so downgrading deletes rows marked as truncated before restoring the old
+    per-request uniqueness constraint.
+
+Revision ID: 20260512_0003
+Revises: 20260508_0002
+Create Date: 2026-05-12 13:42:10.751334
+"""
+
+from alembic import op
+import sqlalchemy as sa
+
+
+revision = "20260512_0003"
+down_revision = "20260508_0002"
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    with op.batch_alter_table(
+        "calculate_request_variables", schema=None
+    ) as batch_op:
+        batch_op.add_column(
+            sa.Column(
+                "variable_name_truncated",
+                sa.Boolean(),
+                server_default=sa.false(),
+                nullable=False,
+            )
+        )
+        batch_op.create_index(
+            "ix_calc_vars_request_id",
+            ["request_id"],
+            unique=False,
+        )
+
+    with op.batch_alter_table(
+        "calculate_request_variables", schema=None
+    ) as batch_op:
+        batch_op.drop_constraint(
+            batch_op.f("ux_calc_vars_request_variable_entity_source"),
+            type_="unique",
+        )
+
+    # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    calculate_request_variables = sa.table(
+        "calculate_request_variables",
+        sa.column("variable_name_truncated", sa.Boolean()),
+    )
+    op.execute(
+        calculate_request_variables.delete().where(
+            calculate_request_variables.c.variable_name_truncated.is_(True)
+        )
+    )
+
+    with op.batch_alter_table(
+        "calculate_request_variables", schema=None
+    ) as batch_op:
+        batch_op.create_unique_constraint(
+            batch_op.f("ux_calc_vars_request_variable_entity_source"),
+            ["request_id", "variable_name", "entity_type", "source"],
+        )
+        batch_op.drop_index("ix_calc_vars_request_id")
+        batch_op.drop_column("variable_name_truncated")
+
+    # ### end Alembic commands ###

policyengine_household_analytics-0.29.6/policyengine_household_analytics/alembic/versions/20260519_0004_calculate_analytics_routing_version.py

@@ -0,0 +1,77 @@
+"""add calculate analytics routing version
+
+Revision ID: 20260519_0004
+Revises: 20260512_0003
+Create Date: 2026-05-19 22:05:42.821147
+"""
+
+from alembic import op
+import sqlalchemy as sa
+
+
+revision = "20260519_0004"
+down_revision = "20260512_0003"
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    with op.batch_alter_table(
+        "calculate_request_variables", schema=None
+    ) as batch_op:
+        batch_op.add_column(
+            sa.Column("requested_version", sa.String(length=64), nullable=True)
+        )
+        batch_op.add_column(
+            sa.Column("resolved_channel", sa.String(length=16), nullable=True)
+        )
+        batch_op.create_index(
+            "ix_calc_vars_channel_created",
+            ["resolved_channel", "created_at"],
+            unique=False,
+        )
+        batch_op.create_index(
+            "ix_calc_vars_requested_created",
+            ["requested_version", "created_at"],
+            unique=False,
+        )
+
+    with op.batch_alter_table("calculate_requests", schema=None) as batch_op:
+        batch_op.add_column(
+            sa.Column("requested_version", sa.String(length=64), nullable=True)
+        )
+        batch_op.add_column(
+            sa.Column("resolved_channel", sa.String(length=16), nullable=True)
+        )
+        batch_op.create_index(
+            "ix_calculate_requests_channel_created",
+            ["resolved_channel", "created_at"],
+            unique=False,
+        )
+        batch_op.create_index(
+            "ix_calculate_requests_requested_created",
+            ["requested_version", "created_at"],
+            unique=False,
+        )
+
+    # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    with op.batch_alter_table("calculate_requests", schema=None) as batch_op:
+        batch_op.drop_index("ix_calculate_requests_requested_created")
+        batch_op.drop_index("ix_calculate_requests_channel_created")
+        batch_op.drop_column("resolved_channel")
+        batch_op.drop_column("requested_version")
+
+    with op.batch_alter_table(
+        "calculate_request_variables", schema=None
+    ) as batch_op:
+        batch_op.drop_index("ix_calc_vars_requested_created")
+        batch_op.drop_index("ix_calc_vars_channel_created")
+        batch_op.drop_column("resolved_channel")
+        batch_op.drop_column("requested_version")
+
+    # ### end Alembic commands ###

policyengine_household_analytics-0.29.6/policyengine_household_analytics/analytics_setup.py

@@ -0,0 +1,433 @@
+"""
+Analytics database setup with opt-in configuration support.
+This module provides conditional analytics database connectivity based on configuration.
+"""
+
+import os
+import logging
+import re
+from functools import cache
+from policyengine_household_common.config_loader import get_config_value
+from policyengine_household_common.analytics_migration import (
+    ANALYTICS_ALEMBIC_MINIMUM_REVISION,
+)
+from google.cloud.sql.connector import Connector
+from google.cloud.sql.connector import IPTypes
+from pathlib import Path
+from sqlalchemy import inspect
+from sqlalchemy.orm import DeclarativeBase
+from flask_sqlalchemy import SQLAlchemy
+
+logger = logging.getLogger(__name__)
+
+# Global variable to store whether analytics is enabled
+_analytics_enabled = None
+_analytics_schema_ready = True
+_connector = None
+
+ANALYTICS_DATABASE_URL_ENV_VAR = "ANALYTICS_DATABASE_URL"
+ANALYTICS_DATABASE_NAME = "user_analytics"
+ALEMBIC_REVISION_ID_PATTERN = re.compile(r"^\d{8}_\d{4}$")
+REQUIRED_ANALYTICS_COLUMNS = {
+    "visits": {
+        "id",
+        "client_id",
+        "datetime",
+        "api_version",
+        "endpoint",
+        "method",
+        "content_length_bytes",
+    },
+    "calculate_requests": {
+        "id",
+        "visit_id",
+        "request_uuid",
+        "client_id",
+        "api_version",
+        "country_id",
+        "model_version",
+        "requested_version",
+        "resolved_channel",
+        "endpoint",
+        "method",
+        "content_length_bytes",
+        "response_status_code",
+        "distinct_variable_count",
+        "unsupported_variable_count",
+        "deprecated_allowlisted_variable_count",
+        "created_at",
+    },
+    "calculate_request_variables": {
+        "id",
+        "request_id",
+        "client_id",
+        "created_at",
+        "country_id",
+        "api_version",
+        "model_version",
+        "requested_version",
+        "resolved_channel",
+        "response_status_code",
+        "variable_name",
+        "variable_name_truncated",
+        "entity_type",
+        "source",
+        "period_granularity",
+        "entity_count",
+        "period_count",
+        "occurrence_count",
+        "availability_status",
+    },
+}
+
+
+# Configure db schema, but don't initialize db itself
+class Base(DeclarativeBase):
+    pass
+
+
+db = SQLAlchemy(model_class=Base)
+
+
+def configure_analytics_db_if_enabled(app):
+    """
+    Register the analytics database extension with Flask, if enabled.
+
+    This does not connect to the database or validate schema readiness. Flask
+    extensions must be registered before the first request is handled, but the
+    API request path should not depend on analytics storage being reachable.
+    """
+    if not is_analytics_enabled():
+        return
+
+    if "sqlalchemy" in app.extensions:
+        return
+
+    database_uri = get_analytics_database_uri()
+    app.config["SQLALCHEMY_DATABASE_URI"] = database_uri
+    if database_uri == "mysql+pymysql://":
+        app.config["SQLALCHEMY_ENGINE_OPTIONS"] = {"creator": getconn}
+
+    db.init_app(app)
+
+
+def initialize_analytics_db_if_enabled(app):
+    """
+    Initialize database configuration for the Flask app, if enabled.
+    Return a bool corresponding with whether or not it's enabled,
+    as well as the SQLAlchemy instance if it is enabled or None if not.
+
+    Args:
+        app: Flask application instance
+    """
+    if not is_analytics_enabled():
+        return
+
+    configure_analytics_db_if_enabled(app)
+
+    with app.app_context():
+        schema_ready = check_analytics_schema_ready()
+        set_analytics_schema_ready(schema_ready)
+        if not schema_ready:
+            raise RuntimeError(
+                "Analytics is enabled but the analytics database schema is "
+                "not ready. Run `uv run alembic upgrade head` and verify "
+                "database connectivity before starting the API."
+            )
+
+
+def get_local_analytics_database_path() -> Path:
+    # Debug-mode-only sqlite file. Resolved from the working directory so
+    # this lib never imports the core package (the writer image installs the
+    # analytics lib without the core package present).
+    return Path.cwd() / "local" / "policyengine_analytics.db"
+
+
+def get_analytics_database_uri() -> str:
+    database_url = os.getenv(
+        ANALYTICS_DATABASE_URL_ENV_VAR
+    ) or get_config_value("analytics.database.url", "")
+    if database_url:
+        return str(database_url)
+
+    if get_config_value("app.debug", False):
+        db_path = get_local_analytics_database_path()
+        should_reset = os.getenv("RESET_ANALYTICS", "").lower() in (
+            "1",
+            "true",
+            "yes",
+        ) or get_config_value("analytics.reset", False)
+        if should_reset and db_path.exists():
+            db_path.unlink()
+        db_path.parent.mkdir(parents=True, exist_ok=True)
+        return f"sqlite:///{db_path}"
+
+    return "mysql+pymysql://"
+
+
+def set_analytics_schema_ready(is_ready: bool) -> None:
+    global _analytics_schema_ready
+    _analytics_schema_ready = is_ready
+
+
+def is_analytics_schema_ready() -> bool:
+    return _analytics_schema_ready
+
+
+def check_analytics_schema_ready() -> bool:
+    try:
+        inspector = inspect(db.engine)
+        missing = list(_missing_required_schema(inspector))
+    except Exception as e:
+        logger.error(f"Could not inspect analytics database schema: {e}")
+        return False
+
+    if missing:
+        logger.error(
+            "Analytics database schema is not ready; run "
+            "`uv run alembic upgrade head`. Missing: " + ", ".join(missing)
+        )
+        return False
+
+    try:
+        alembic_version = _alembic_version()
+    except Exception as e:
+        logger.error(f"Could not inspect analytics migration version: {e}")
+        return False
+
+    if not _alembic_revision_is_compatible(alembic_version):
+        logger.error(
+            "Analytics database schema is not at or after required Alembic "
+            f"revision {ANALYTICS_ALEMBIC_MINIMUM_REVISION}; current "
+            "revision is "
+            f"{alembic_version or 'missing'}. Run `uv run alembic upgrade head`."
+        )
+        return False
+
+    return True
+
+
+def _missing_required_schema(inspector):
+    required_columns = {"visits": REQUIRED_ANALYTICS_COLUMNS["visits"]}
+    if _collect_variable_usage_enabled():
+        required_columns["calculate_requests"] = REQUIRED_ANALYTICS_COLUMNS[
+            "calculate_requests"
+        ]
+        required_columns["calculate_request_variables"] = (
+            REQUIRED_ANALYTICS_COLUMNS["calculate_request_variables"]
+        )
+
+    for table_name, column_names in required_columns.items():
+        if not inspector.has_table(table_name):
+            yield table_name
+            continue
+
+        existing_columns = {
+            column["name"] for column in inspector.get_columns(table_name)
+        }
+        for missing_column in sorted(column_names - existing_columns):
+            yield f"{table_name}.{missing_column}"
+
+
+def _collect_variable_usage_enabled() -> bool:
+    value = get_config_value("analytics.collect_variable_usage", True)
+    if isinstance(value, str):
+        return value.lower() not in {"0", "false", "no"}
+    return bool(value)
+
+
+def _alembic_version() -> str | None:
+    inspector = inspect(db.engine)
+    if not inspector.has_table("alembic_version"):
+        return None
+
+    with db.engine.connect() as connection:
+        return connection.exec_driver_sql(
+            "SELECT version_num FROM alembic_version"
+        ).scalar()
+
+
+def _alembic_revision_is_compatible(
+    current_revision: str | None,
+    minimum_revision: str = ANALYTICS_ALEMBIC_MINIMUM_REVISION,
+) -> bool:
+    if not current_revision:
+        return False
+    if current_revision == minimum_revision:
+        return True
+    if _known_revision_descends_from(current_revision, minimum_revision):
+        return True
+    return _revision_id_is_later_than(current_revision, minimum_revision)
+
+
+def _known_revision_descends_from(
+    current_revision: str,
+    minimum_revision: str,
+) -> bool:
+    try:
+        script = _alembic_script_directory()
+        current = script.get_revision(current_revision)
+        minimum = script.get_revision(minimum_revision)
+    except Exception as e:
+        logger.debug(f"Could not resolve Alembic revision locally: {e}")
+        return False
+
+    if current is None or minimum is None:
+        return False
+
+    pending = [current]
+    seen = set()
+    while pending:
+        revision = pending.pop()
+        if revision.revision == minimum.revision:
+            return True
+        if revision.revision in seen:
+            continue
+        seen.add(revision.revision)
+        for down_revision in _down_revisions(revision.down_revision):
+            try:
+                parent = script.get_revision(down_revision)
+            except Exception as e:
+                logger.debug(
+                    "Could not resolve Alembic parent revision "
+                    f"{down_revision}: {e}"
+                )
+                return False
+            if parent is not None:
+                pending.append(parent)
+    return False
+
+
+@cache
+def _alembic_script_directory():
+    from alembic.config import Config
+    from alembic.script import ScriptDirectory
+
+    # Migration scripts ship as package data, so revision ancestry checks
+    # work in every runtime that installs this lib — previously they resolved
+    # from the repo root and silently fell back to string comparison in
+    # images that shipped no alembic/ directory.
+    alembic_config = Config()
+    alembic_config.set_main_option(
+        "script_location",
+        str(Path(__file__).resolve().parent / "alembic"),
+    )
+    return ScriptDirectory.from_config(alembic_config)
+
+
+def _down_revisions(down_revision):
+    if down_revision is None:
+        return ()
+    if isinstance(down_revision, str):
+        return (down_revision,)
+    return tuple(down_revision)
+
+
+def _revision_id_is_later_than(
+    current_revision: str,
+    minimum_revision: str,
+) -> bool:
+    return (
+        bool(ALEMBIC_REVISION_ID_PATTERN.fullmatch(current_revision))
+        and bool(ALEMBIC_REVISION_ID_PATTERN.fullmatch(minimum_revision))
+        and current_revision > minimum_revision
+    )
+
+
+def is_analytics_enabled() -> bool:
+    """
+    Check if analytics is enabled based on configuration.
+
+    Returns:
+        bool: True if analytics is enabled, False otherwise
+    """
+    global _analytics_enabled
+
+    if _analytics_enabled is not None:
+        return _analytics_enabled
+
+    # Try to get from config first (future use when app migrates to ConfigLoader)
+    _analytics_enabled = get_config_value("analytics.enabled", False)
+    return _analytics_enabled
+
+
+def get_analytics_connector(require_analytics_enabled: bool = True):
+    """
+    Get the Google Cloud SQL connector for analytics.
+    Returns None if analytics is disabled.
+
+    Returns:
+        Connector or None: The connector instance if analytics is enabled, None otherwise
+    """
+    global _connector
+
+    if require_analytics_enabled and not is_analytics_enabled():
+        return None
+
+    if _connector is None:
+        try:
+            _connector = Connector()
+        except Exception as e:
+            logger.error(f"Failed to initialize analytics connector: {e}")
+            return None
+
+    return _connector
+
+
+def getconn(require_analytics_enabled: bool = True):
+    """
+    Get a connection to the analytics database.
+    Returns None if analytics is disabled.
+
+    Returns:
+        Connection or None: Database connection if analytics is enabled, None otherwise
+    """
+    if require_analytics_enabled and not is_analytics_enabled():
+        return None
+
+    connector = get_analytics_connector(require_analytics_enabled)
+    if not connector:
+        return None
+
+    try:
+        connection_name = get_config_value(
+            "analytics.database.connection_name",
+        )
+        username = get_config_value(
+            "analytics.database.username",
+        )
+        password = get_config_value(
+            "analytics.database.password",
+        )
+
+        if not connection_name or not username or not password:
+            logger.error(
+                "Analytics enabled but problem with one or more of the following configuration values: connection_name, username, password"
+            )
+            return None
+
+        conn = connector.connect(
+            connection_name,
+            "pymysql",
+            user=username,
+            password=password,
+            db=ANALYTICS_DATABASE_NAME,
+            ip_type=IPTypes.PUBLIC,
+        )
+
+        return conn
+
+    except Exception as e:
+        logger.error(f"Failed to connect to analytics database: {e}")
+        return None
+
+
+def cleanup():
+    """Clean up the analytics connector if it exists."""
+    global _connector
+    if _connector:
+        try:
+            _connector.close()
+        except Exception:
+            pass
+        _connector = None

policyengine_household_analytics-0.29.6/policyengine_household_analytics/events.py

@@ -0,0 +1,17 @@
+from __future__ import annotations
+
+from typing import Literal
+
+from pydantic import BaseModel, ConfigDict
+
+from policyengine_household_common.models.analytics import AnalyticsContext
+
+
+class CalculateAnalyticsEvent(BaseModel):
+    """Value-free analytics event persisted outside the request path."""
+
+    model_config = ConfigDict(frozen=True)
+
+    schema_version: Literal[1] = 1
+    context: AnalyticsContext
+    response_status_code: int | None = None

policyengine_household_analytics-0.29.6/policyengine_household_analytics/orm.py

@@ -0,0 +1,183 @@
+from policyengine_household_analytics.analytics_setup import db
+from policyengine_household_common.models.analytics import (
+    AnalyticsHttpMethod,
+    AvailabilityStatus,
+    ModalResolvedChannel,
+    PeriodGranularity,
+    VariableSource,
+)
+from sqlalchemy import (
+    Boolean,
+    DateTime,
+    ForeignKey,
+    Index,
+    Integer,
+    String,
+)
+from sqlalchemy.orm import mapped_column
+
+
+def _enum_values(enum_type) -> tuple[str, ...]:
+    return tuple(member.value for member in enum_type)
+
+
+class Visit(db.Model):
+    # Note that the model represents one visit,
+    # while the table name is plural
+    __tablename__ = "visits"
+    id = mapped_column(Integer, primary_key=True)
+    client_id = mapped_column(String(255), nullable=True)
+    datetime = mapped_column(DateTime)
+    api_version = mapped_column(String(32))
+    endpoint = mapped_column(String(64))
+    method = mapped_column(
+        String(32),
+        info={"options": _enum_values(AnalyticsHttpMethod)},
+    )
+    content_length_bytes = mapped_column(Integer)
+
+
+class CalculateRequest(db.Model):
+    """One analytics record for an inbound /calculate request."""
+
+    __tablename__ = "calculate_requests"
+
+    id = mapped_column(Integer, primary_key=True)
+    visit_id = mapped_column(
+        Integer,
+        ForeignKey("visits.id"),
+        nullable=False,
+    )
+    request_uuid = mapped_column(String(36), nullable=False, unique=True)
+    client_id = mapped_column(String(255), nullable=True)
+    api_version = mapped_column(String(32), nullable=True)
+    country_id = mapped_column(String(16), nullable=False)
+    model_version = mapped_column(String(64), nullable=True)
+    requested_version = mapped_column(String(64), nullable=True)
+    resolved_channel = mapped_column(
+        String(16),
+        nullable=True,
+        info={"options": _enum_values(ModalResolvedChannel)},
+    )
+    endpoint = mapped_column(String(64), nullable=True)
+    method = mapped_column(
+        String(16),
+        nullable=False,
+        info={"options": _enum_values(AnalyticsHttpMethod)},
+    )
+    content_length_bytes = mapped_column(Integer, nullable=True)
+    response_status_code = mapped_column(Integer, nullable=True)
+    distinct_variable_count = mapped_column(Integer, nullable=False, default=0)
+    unsupported_variable_count = mapped_column(
+        Integer, nullable=False, default=0
+    )
+    deprecated_allowlisted_variable_count = mapped_column(
+        Integer, nullable=False, default=0
+    )
+    created_at = mapped_column(DateTime, nullable=False)
+
+    __table_args__ = (
+        Index(
+            "ix_calculate_requests_client_created", "client_id", "created_at"
+        ),
+        Index("ix_calculate_requests_visit_id", "visit_id"),
+        Index(
+            "ix_calculate_requests_country_created",
+            "country_id",
+            "created_at",
+        ),
+        Index(
+            "ix_calculate_requests_channel_created",
+            "resolved_channel",
+            "created_at",
+        ),
+        Index(
+            "ix_calculate_requests_requested_created",
+            "requested_version",
+            "created_at",
+        ),
+    )
+
+
+class CalculateRequestVariable(db.Model):
+    """A grouped variable-usage row derived from one calculate request."""
+
+    __tablename__ = "calculate_request_variables"
+
+    id = mapped_column(Integer, primary_key=True)
+    request_id = mapped_column(
+        Integer,
+        ForeignKey("calculate_requests.id"),
+        nullable=False,
+    )
+    client_id = mapped_column(String(255), nullable=True)
+    created_at = mapped_column(DateTime, nullable=False)
+    country_id = mapped_column(String(16), nullable=False)
+    api_version = mapped_column(String(32), nullable=True)
+    model_version = mapped_column(String(64), nullable=True)
+    requested_version = mapped_column(String(64), nullable=True)
+    resolved_channel = mapped_column(
+        String(16),
+        nullable=True,
+        info={"options": _enum_values(ModalResolvedChannel)},
+    )
+    response_status_code = mapped_column(Integer, nullable=True)
+    variable_name = mapped_column(String(255), nullable=False)
+    variable_name_truncated = mapped_column(
+        Boolean,
+        nullable=False,
+        default=False,
+    )
+    entity_type = mapped_column(String(64), nullable=False)
+    source = mapped_column(
+        String(32),
+        nullable=False,
+        info={"options": _enum_values(VariableSource)},
+    )
+    period_granularity = mapped_column(
+        String(16),
+        nullable=False,
+        info={"options": _enum_values(PeriodGranularity)},
+    )
+    entity_count = mapped_column(Integer, nullable=False, default=0)
+    period_count = mapped_column(Integer, nullable=False, default=0)
+    occurrence_count = mapped_column(Integer, nullable=False, default=0)
+    availability_status = mapped_column(
+        String(32),
+        nullable=False,
+        info={"options": _enum_values(AvailabilityStatus)},
+    )
+
+    # Do not make request_id + variable_name unique: overlong variable names
+    # are intentionally capped before persistence, so different originals can
+    # share one stored representation.
+    __table_args__ = (
+        Index("ix_calc_vars_request_id", "request_id"),
+        Index(
+            "ix_calc_vars_variable_created",
+            "variable_name",
+            "created_at",
+        ),
+        Index(
+            "ix_calc_vars_client_variable_created",
+            "client_id",
+            "variable_name",
+            "created_at",
+        ),
+        Index(
+            "ix_calc_vars_country_model_variable",
+            "country_id",
+            "model_version",
+            "variable_name",
+        ),
+        Index(
+            "ix_calc_vars_channel_created",
+            "resolved_channel",
+            "created_at",
+        ),
+        Index(
+            "ix_calc_vars_requested_created",
+            "requested_version",
+            "created_at",
+        ),
+    )

policyengine_household_analytics-0.29.6/policyengine_household_analytics/persistence.py

@@ -0,0 +1,177 @@
+from __future__ import annotations
+
+import logging
+
+from sqlalchemy.exc import IntegrityError
+
+from policyengine_household_analytics.events import CalculateAnalyticsEvent
+from policyengine_household_analytics.analytics_setup import db
+from policyengine_household_analytics.orm import (
+    CalculateRequest,
+    CalculateRequestVariable,
+    Visit,
+)
+from policyengine_household_common.models.analytics import (
+    AnalyticsContext,
+    VariableUsageSummary,
+)
+from policyengine_household_common.variable_usage_analytics import (
+    stored_variable_name,
+)
+
+logger = logging.getLogger(__name__)
+
+
+def record_calculate_analytics_event(
+    event: CalculateAnalyticsEvent,
+) -> None:
+    record_analytics(event.context, event.response_status_code)
+
+
+def record_analytics(
+    context: AnalyticsContext | None,
+    response_status_code: int | None,
+) -> None:
+    if context is None:
+        return
+
+    if _calculate_request_exists(context):
+        return
+
+    try:
+        visit = _build_visit(context)
+        db.session.add(visit)
+        db.session.flush()
+        visit_id = getattr(visit, "id", None)
+
+        variable_summaries = context.variable_summaries
+        calculate_request = _build_calculate_request(
+            context,
+            response_status_code,
+            variable_summaries,
+            visit_id,
+        )
+
+        if calculate_request is not None:
+            db.session.add(calculate_request)
+            db.session.flush()
+            for summary in variable_summaries:
+                db.session.add(
+                    _build_calculate_request_variable(
+                        calculate_request,
+                        summary,
+                    )
+                )
+
+        db.session.commit()
+    except IntegrityError:
+        db.session.rollback()
+        if _calculate_request_exists(context):
+            return
+        logger.exception("Failed to log analytics due to integrity error")
+        raise
+    except Exception as e:
+        db.session.rollback()
+        logger.error(f"Failed to log analytics: {e}")
+        raise
+
+
+def _calculate_request_exists(context: AnalyticsContext) -> bool:
+    if not context.record_calculate_request:
+        return False
+    return (
+        db.session.query(CalculateRequest)
+        .filter_by(request_uuid=context.request_uuid)
+        .first()
+        is not None
+    )
+
+
+def _build_visit(context: AnalyticsContext) -> Visit:
+    visit = Visit()
+    visit.client_id = context.client_id
+    visit.api_version = context.api_version
+    visit.endpoint = context.endpoint
+    visit.method = context.method.value
+    visit.content_length_bytes = context.content_length_bytes
+    visit.datetime = context.created_at
+    return visit
+
+
+def _build_calculate_request(
+    context: AnalyticsContext,
+    response_status_code: int | None,
+    variable_summaries: tuple[VariableUsageSummary, ...],
+    visit_id: int | None,
+) -> CalculateRequest | None:
+    if not context.record_calculate_request:
+        return None
+    if visit_id is None:
+        raise ValueError("Visit ID is required for calculate analytics")
+
+    distinct_variable_names = {
+        summary.variable_name for summary in variable_summaries
+    }
+    unsupported_variable_names = {
+        summary.variable_name
+        for summary in variable_summaries
+        if summary.availability_status == "unsupported"
+    }
+    deprecated_allowlisted_variable_names = {
+        summary.variable_name
+        for summary in variable_summaries
+        if summary.availability_status == "deprecated_allowlisted"
+    }
+
+    calculate_request = CalculateRequest()
+    calculate_request.visit_id = visit_id
+    calculate_request.request_uuid = context.request_uuid
+    calculate_request.client_id = context.client_id
+    calculate_request.api_version = context.api_version
+    calculate_request.country_id = context.country_id
+    calculate_request.model_version = context.model_version
+    calculate_request.requested_version = context.requested_version
+    calculate_request.resolved_channel = (
+        context.resolved_channel.value if context.resolved_channel else None
+    )
+    calculate_request.endpoint = context.endpoint
+    calculate_request.method = context.method.value
+    calculate_request.content_length_bytes = context.content_length_bytes
+    calculate_request.response_status_code = response_status_code
+    calculate_request.distinct_variable_count = len(distinct_variable_names)
+    calculate_request.unsupported_variable_count = len(
+        unsupported_variable_names
+    )
+    calculate_request.deprecated_allowlisted_variable_count = len(
+        deprecated_allowlisted_variable_names
+    )
+    calculate_request.created_at = context.created_at
+    return calculate_request
+
+
+def _build_calculate_request_variable(
+    calculate_request: CalculateRequest,
+    summary: VariableUsageSummary,
+) -> CalculateRequestVariable:
+    variable = CalculateRequestVariable()
+    variable.request_id = calculate_request.id
+    variable.client_id = calculate_request.client_id
+    variable.created_at = calculate_request.created_at
+    variable.country_id = calculate_request.country_id
+    variable.api_version = calculate_request.api_version
+    variable.model_version = calculate_request.model_version
+    variable.requested_version = calculate_request.requested_version
+    variable.resolved_channel = calculate_request.resolved_channel
+    variable.response_status_code = calculate_request.response_status_code
+    (
+        variable.variable_name,
+        variable.variable_name_truncated,
+    ) = stored_variable_name(summary.variable_name)
+    variable.entity_type = summary.entity_type
+    variable.source = summary.source.value
+    variable.period_granularity = summary.period_granularity.value
+    variable.entity_count = summary.entity_count
+    variable.period_count = summary.period_count
+    variable.occurrence_count = summary.occurrence_count
+    variable.availability_status = summary.availability_status.value
+    return variable

policyengine_household_analytics-0.29.6/pyproject.toml

@@ -0,0 +1,32 @@
+[project]
+name = "policyengine-household-analytics"
+version = "0.29.6"
+description = "Analytics contract and persistence for the PolicyEngine Household API"
+readme = "README.md"
+authors = [
+    { name = "PolicyEngine", email = "hello@policyengine.org" },
+]
+requires-python = ">=3.12"
+# The slim analytics writer image installs this lib: keep numpy, country
+# model packages, and modal out of this closure.
+dependencies = [
+    "policyengine-household-common==0.29.6",
+    "alembic>=1.13.0",
+    "cloud-sql-python-connector",
+    "flask>=2.2",
+    "flask-sqlalchemy>=3",
+    "pymysql",
+    "sqlalchemy>=2",
+]
+
+[tool.uv.sources]
+policyengine-household-common = { workspace = true }
+
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[tool.hatch.build.targets.wheel]
+# Includes the alembic/ migration scripts as package data, so revision
+# ancestry checks work in every runtime that installs this lib.
+packages = ["policyengine_household_analytics"]