meta-edc 1.0.6__py3-none-any.whl → 1.1.0__py3-none-any.whl

meta_analytics/notebooks/monitoring_report.ipynb

@@ -0,0 +1,1751 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "0",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "%%capture\n",
+    "import os\n",
+    "from pathlib import Path\n",
+    "import pandas as pd\n",
+    "from dj_notebook import activate\n",
+    "import numpy as np\n",
+    "from django_pandas.io import read_frame\n",
+    "\n",
+    "env_file = os.environ[\"META_ENV\"]\n",
+    "reports_folder = Path(os.environ[\"META_REPORTS_FOLDER\"])\n",
+    "analysis_folder = Path(os.environ[\"META_ANALYSIS_FOLDER\"])\n",
+    "pharmacy_folder = Path(os.environ[\"META_PHARMACY_FOLDER\"])\n",
+    "plus = activate(dotenv_file=env_file)\n",
+    "pd.set_option('future.no_silent_downcasting', True)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "1",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import pdfkit\n",
+    "from typing import Callable\n",
+    "from datetime import date\n",
+    "from edc_pdutils.dataframes import get_subject_visit\n",
+    "from meta_visit_schedule.constants import MONTH15, MONTH18, MONTH21, MONTH27, MONTH30, MONTH33, MONTH39\n",
+    "from meta_analytics.dataframes import GlucoseEndpointsByDate\n",
+    "from scipy.stats import chi2\n",
+    "from great_tables import loc, style, md\n",
+    "from meta_analytics.dataframes import get_eos_df\n",
+    "from meta_analytics.utils import df_as_great_table, df_as_great_table2\n",
+    "from meta_prn.models import LossToFollowup\n",
+    "from edc_visit_schedule.models import SubjectScheduleHistory\n",
+    "from edc_appointment.analytics import get_appointment_df\n",
+    "from edc_appointment.constants import NEW_APPT, CANCELLED_APPT, ONTIME_APPT, MISSED_APPT\n",
+    "from meta_consent.models import SubjectConsentV1Ext\n",
+    "from meta_analytics.dataframes import get_glucose_df, get_screening_df\n",
+    "\n",
+    "from edc_appointment.constants import SCHEDULED_APPT, UNSCHEDULED_APPT # noqa\n",
+    "from edc_constants.constants import YES # noqa"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "2",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "html_data = []\n",
+    "data_download_date= date(2025,6, 17)\n",
+    "cutoff_date = date(2025,6, 17)\n",
+    "end_of_trial_date= date(2026,7, 1)\n",
+    "document_title = f\"<h2>Monitoring Report: {cutoff_date.strftime('%B %Y')}</h2><h5>Data Download: {data_download_date.strftime('%d %B %Y')}</h5>\"\n",
+    "study_title = 'META3 - Metformin treatment for diabetes prevention in Africa'\n",
+    "pdf_filename = f\"monitoring_report_{cutoff_date.strftime('%Y%m%d')}.pdf\"\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "3",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# 105-30-0288-5 should also be late excluded based on the haemoglobin 4.8 presented at baseline\n",
+    "\n",
+    "df_visit = get_subject_visit(\"meta_subject.subjectvisit\")\n",
+    "late_exlusion_offstudy_reasons = ['Patient fulfilled late exclusion criteria (due to abnormal blood values or raised blood pressure at enrolment']\n",
+    "df_eos = get_eos_df()\n",
+    "df_eos_excluded = (\n",
+    "    df_eos\n",
+    "    .query(\"offstudy_reason.isin(@late_exlusion_offstudy_reasons)\")\n",
+    "    .copy()\n",
+    "    .reset_index()\n",
+    ")\n",
+    "df_visit = (\n",
+    "    df_visit\n",
+    "    .merge(df_eos_excluded[[\"subject_identifier\", \"offstudy_datetime\", \"offstudy_reason\"]], on=\"subject_identifier\", how=\"left\", indicator=True)\n",
+    "    .query(\"_merge=='left_only'\")\n",
+    "    .drop(columns=[\"_merge\"])\n",
+    ")\n",
+    "\n",
+    "df_visit = df_visit[df_visit.appt_datetime.dt.date<=cutoff_date]\n",
+    "\n",
+    "df_appointments = get_appointment_df()\n",
+    "df_appointments[\"site_id\"] = df_appointments.site_id.astype(str)\n",
+    "df_appointments = (\n",
+    "    df_appointments\n",
+    "    .merge(df_eos_excluded[[\"subject_identifier\", \"offstudy_datetime\", \"offstudy_reason\"]], on=\"subject_identifier\", how=\"left\", indicator=True)\n",
+    "    .query(\"_merge=='left_only'\")\n",
+    "    .drop(columns=[\"_merge\"])\n",
+    ")\n",
+    "\n",
+    "\n",
+    "cls = GlucoseEndpointsByDate()\n",
+    "cls.run()\n",
+    "df_endpoint = cls.endpoint_only_df.copy()\n",
+    "df_glucose = get_glucose_df()\n",
+    "# df_glucose_fbg = get_glucose_fbg_df()\n",
+    "# df_glucose = pd.concat([df_glucose, df_glucose_fbg])\n",
+    "\n",
+    "\n",
+    "enrolled = df_visit.copy()\n",
+    "enrolled[\"site_id\"] = enrolled[\"site_id\"].astype(str)\n",
+    "enrolled_pivot = (\n",
+    "    enrolled\n",
+    "    .query(\"visit_code==1000.0\").groupby([\"site_id\"])\n",
+    "    .size()\n",
+    "    .reset_index()\n",
+    "    .pivot_table(columns=\"site_id\", values=0, observed=True)\n",
+    ")\n",
+    "enrolled_pivot.columns.name=\"\"\n",
+    "enrolled_pivot[\"total\"] = enrolled_pivot[[\"10\", \"20\",\"30\",\"40\",\"60\"]].sum(axis=1)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "4",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# df_eos.query(\"offstudy_reason.isin(@late_exlusion_offstudy_reasons)\")[[\"subject_identifier\", \"offstudy_datetime\", \"offstudy_reason\"]].to_stata(\n",
+    "#         path=analysis_folder / \"late_exclusions.dta\",\n",
+    "#         variable_labels={},\n",
+    "#         version=118,\n",
+    "#         write_index=False,\n",
+    "#     )"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "5",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "column_headers = {\"label\": \"Label\", \"visit_code\": \"Visit code\", \"10\": \"Hindu Mandal\", \"20\": \"Amana\", \"30\": \"Temeke\", \"40\": \"Mwananyamala\", \"60\": \"Mnazi Moja\", \"total\": \"Total\"}\n",
+    "column_headers_with_str = {\"label\": \"Label\", \"10_str\": \"Hindu Mandal\", \"20_str\": \"Amana\", \"30_str\": \"Temeke\", \"40_str\": \"Mwananyamala\", \"60_str\": \"Mnazi Moja\", \"total_str\": \"Total\"}"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "6",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Table 1a Visits completed to date\n",
+    "\n",
+    "df_tbl1 = df_visit[(df_visit.visit_code_sequence==0) & (df_visit.appt_timing==ONTIME_APPT) & ~(df_visit.appt_status.isin([NEW_APPT, CANCELLED_APPT]))].groupby(by=[\"visit_code\", \"site_id\"]).size().to_frame().reset_index()\n",
+    "\n",
+    "df_tbl1.columns = [\"visit_code\", \"site_id\", \"visits\"]\n",
+    "df1 = df_tbl1.pivot(index=\"visit_code\", columns=\"site_id\", values=\"visits\").reset_index()\n",
+    "df1.columns.name = None\n",
+    "df1.columns = ['visit_code', \"10\", \"20\", \"30\", \"40\", \"60\"]\n",
+    "df1['total'] = df1[['10', '20', '30', '40', '60']].sum(axis=1)\n",
+    "df1.fillna(0, inplace=True)\n",
+    "df_attended = df1.copy().reset_index(drop=True)\n",
+    "df_attended = df_attended.fillna(0.0)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "7",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "gt = df_as_great_table(\n",
+    "    df_attended[[\"visit_code\", \"10\", \"20\", \"30\", \"40\", \"60\", \"total\"]],\n",
+    "    title=\"Table 1a: Visits completed to date\"\n",
+    ")\n",
+    "gt = (\n",
+    "    gt\n",
+    "    .cols_label({k:v for k, v in column_headers.items() if k!=\"label\"})\n",
+    "    .cols_align(align=\"center\", columns=[\"10\", \"20\", \"30\", \"40\", \"60\", \"total\"])\n",
+    "    .cols_align(align=\"left\", columns=[\"visit_code\"])\n",
+    "    .data_color(\n",
+    "        columns=[\"visit_code\"],\n",
+    "        palette=[\"lavender\", \"thistle\"],\n",
+    "        domain=[2000, 5000],\n",
+    "        na_color=\"white\"\n",
+    "    )\n",
+    "    .tab_source_note(source_note=f\"Excludes visit reports submitted for participants eventually withdrawn on late exclusion criteria.\")\n",
+    ")\n",
+    "html_data.append(gt.as_raw_html())\n",
+    "gt.show()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "8",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Table 1b Total scheduled appointments\n",
+    "df_appt_pivot = (\n",
+    "    df_appointments.query(\"appt_reason==@SCHEDULED_APPT\")\n",
+    "    .merge(df_eos_excluded[[\"subject_identifier\"]], on=\"subject_identifier\", how=\"left\", indicator=True)\n",
+    "    .query(\"_merge=='left_only'\")\n",
+    "    .drop(columns=[\"_merge\"])\n",
+    "    .reset_index(drop=True)\n",
+    "    .groupby([\"visit_code\", \"site_id\"])\n",
+    "    .size()\n",
+    "    .to_frame()\n",
+    "    .reset_index()\n",
+    "    .pivot(index=\"visit_code\", columns=\"site_id\", values=0)\n",
+    "    .reset_index()\n",
+    "    .fillna(0)\n",
+    ")\n",
+    "\n",
+    "df_appt_pivot[\"total\"] = df_appt_pivot.iloc[:,1:].sum(axis=1)\n",
+    "df_appt_pivot.columns.name = None\n",
+    "gt = df_as_great_table(\n",
+    "    df_appt_pivot,\n",
+    "    title=\"Table 1b: Total appointments\",\n",
+    "    subtitle=\"Total possible appointments not including unscheduled appointments\"\n",
+    "\n",
+    ")\n",
+    "gt = (\n",
+    "    gt\n",
+    "    .cols_label({k:v for k, v in column_headers.items() if k!=\"label\"})\n",
+    "    .cols_align(align=\"center\", columns=[\"10\", \"20\", \"30\", \"40\", \"60\", \"total\"])\n",
+    "    .cols_align(align=\"left\", columns=[\"visit_code\"])\n",
+    "    .data_color(\n",
+    "        columns=[\"visit_code\"],\n",
+    "        palette=[\"lavender\", \"thistle\"],\n",
+    "        domain=[2000, 5000],\n",
+    "        na_color=\"white\"\n",
+    "    )\n",
+    ")\n",
+    "html_data.append(gt.as_raw_html())\n",
+    "gt.show()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "9",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Table 1c Past scheduled appointments -- no information provided\n",
+    "df_appt_pivot = (\n",
+    "    df_appointments.query(\"appt_datetime<@cutoff_date and appt_reason==@SCHEDULED_APPT and appt_timing==@ONTIME_APPT and appt_status.isin([@NEW_APPT])\")\n",
+    "    .merge(df_eos_excluded[[\"subject_identifier\"]], on=\"subject_identifier\", how=\"left\", indicator=True)\n",
+    "    .query(\"_merge=='left_only'\")\n",
+    "    .drop(columns=[\"_merge\"])\n",
+    "    .reset_index(drop=True)\n",
+    "    .groupby([\"visit_code\", \"site_id\"])\n",
+    "    .size()\n",
+    "    .to_frame()\n",
+    "    .reset_index()\n",
+    "    .pivot(index=\"visit_code\", columns=\"site_id\", values=0)\n",
+    "    .reset_index()\n",
+    "    .fillna(0)\n",
+    ")\n",
+    "df_appt_pivot[\"total\"] = df_appt_pivot.iloc[:,1:].sum(axis=1)\n",
+    "df_appt_pivot.columns.name = None\n",
+    "gt = df_as_great_table(\n",
+    "    df_appt_pivot,\n",
+    "    title=\"Table 1c: Past appointments not attended/not reported\",\n",
+    "    subtitle=\"Expected by now but no information provided by site\",\n",
+    ")\n",
+    "gt = (\n",
+    "    gt\n",
+    "    .cols_label({k:v for k, v in column_headers.items() if k!=\"label\"})\n",
+    "    .cols_align(align=\"center\", columns=[\"10\", \"20\", \"30\", \"40\", \"60\", \"total\"])\n",
+    "    .cols_align(align=\"left\", columns=[\"visit_code\"])\n",
+    "    .data_color(\n",
+    "        columns=[\"visit_code\"],\n",
+    "        palette=[\"lavender\", \"thistle\"],\n",
+    "        domain=[2000, 5000],\n",
+    "        na_color=\"white\"\n",
+    "    )\n",
+    "    .tab_source_note(source_note=f\"Scheduled appointment date is before {cutoff_date.strftime('%d %B %Y')}.\")\n",
+    ")\n",
+    "html_data.append(gt.as_raw_html())\n",
+    "gt.show()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "10",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Table 1d Unscheduled appointments\n",
+    "df_appt = (\n",
+    "    df_appointments.query(\"appt_reason==@UNSCHEDULED_APPT and appt_timing==@ONTIME_APPT and appt_status!=@NEW_APPT\")\n",
+    "     .merge(df_eos_excluded[[\"subject_identifier\"]], on=\"subject_identifier\", how=\"left\", indicator=True)\n",
+    "    .query(\"_merge=='left_only'\")\n",
+    "    .drop(columns=[\"_merge\"])\n",
+    "    .reset_index(drop=True)\n",
+    "   .copy()\n",
+    "    .reset_index(drop=True)\n",
+    ")\n",
+    "df_appt['visit_code'] = df_appt['visit_code'].astype(int)\n",
+    "df_appt['visit_code'] = df_appt['visit_code'].astype(str)\n",
+    "\n",
+    "subjects_with_unscheduled = df_appt.subject_identifier.nunique()\n",
+    "\n",
+    "df_appt_pivot = (\n",
+    "    df_appt\n",
+    "    .groupby([\"visit_code\", \"site_id\"])\n",
+    "    .size()\n",
+    "    .to_frame()\n",
+    "    .reset_index()\n",
+    "    .pivot(index=\"visit_code\", columns=\"site_id\", values=0)\n",
+    "    .reset_index()\n",
+    "    .fillna(0)\n",
+    ")\n",
+    "df_appt_pivot[\"total\"] = df_appt_pivot.iloc[:,1:].sum(axis=1)\n",
+    "df_appt_pivot.columns.name = None\n",
+    "df_appt_pivot[[\"10\", \"20\", \"30\", \"40\", \"60\", \"total\"]] = df_appt_pivot[[\"10\", \"20\", \"30\", \"40\", \"60\", \"total\"]].astype('float64')\n",
+    "\n",
+    "\n",
+    "# add totals row\n",
+    "sum_row = df_appt_pivot.select_dtypes(include='float64').sum()\n",
+    "sum_row['visit_code'] = 'Total'\n",
+    "sum_row_df = pd.DataFrame(sum_row).T\n",
+    "df_appt_pivot = pd.concat([df_appt_pivot, sum_row_df], axis=0).reset_index(drop=True)\n",
+    "\n",
+    "gt = df_as_great_table(\n",
+    "    df_appt_pivot,\n",
+    "    title=\"Table 1d: Unscheduled appointments\",\n",
+    "    subtitle=\"Appointments with sequence>0 grouped by visit code\",\n",
+    ")\n",
+    "gt = (\n",
+    "    gt\n",
+    "    .cols_label({k:v for k, v in column_headers.items() if k!=\"label\"})\n",
+    "    .cols_align(align=\"center\", columns=[\"10\", \"20\", \"30\", \"40\", \"60\", \"total\"])\n",
+    "    .cols_align(align=\"left\", columns=[\"visit_code\"])\n",
+    "    .data_color(\n",
+    "        columns=[\"visit_code\"],\n",
+    "        palette=[\"lavender\", \"thistle\"],\n",
+    "        domain=[2000, 5000],\n",
+    "        na_color=\"white\"\n",
+    "    )\n",
+    "    .fmt_number(columns=[\"10\", \"20\", \"30\", \"40\", \"60\", \"total\"], decimals=0)\n",
+    "    .tab_source_note(source_note=f\"{subjects_with_unscheduled} participants had at least one unscheduled appointment.\")\n",
+    ")\n",
+    "html_data.append(gt.as_raw_html())\n",
+    "gt.show()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "11",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Table 1e Future scheduled appointments\n",
+    "df_appt_pivot = (\n",
+    "    df_appointments.query(\"@cutoff_date<=appt_datetime<@end_of_trial_date and appt_reason==@SCHEDULED_APPT and appt_timing==@ONTIME_APPT and appt_status.isin([@NEW_APPT])\")\n",
+    "    .merge(df_eos_excluded[[\"subject_identifier\"]], on=\"subject_identifier\", how=\"left\", indicator=True)\n",
+    "    .query(\"_merge=='left_only'\")\n",
+    "    .drop(columns=[\"_merge\"])\n",
+    "    .reset_index(drop=True)\n",
+    "    .groupby([\"visit_code\", \"site_id\"])\n",
+    "    .size()\n",
+    "    .to_frame()\n",
+    "    .reset_index()\n",
+    "    .pivot(index=\"visit_code\", columns=\"site_id\", values=0)\n",
+    "    .reset_index()\n",
+    "    .fillna(0)\n",
+    ")\n",
+    "df_appt_pivot[\"total\"] = df_appt_pivot.iloc[:,1:].sum(axis=1)\n",
+    "df_appt_pivot.columns.name = None\n",
+    "gt = df_as_great_table(\n",
+    "    df_appt_pivot,\n",
+    "    title=\"Table 1e: Future appointments\",\n",
+    ")\n",
+    "gt = (\n",
+    "    gt\n",
+    "    .cols_label({k:v for k, v in column_headers.items() if k!=\"label\"})\n",
+    "    .cols_align(align=\"center\", columns=[\"10\", \"20\", \"30\", \"40\", \"60\", \"total\"])\n",
+    "    .cols_align(align=\"left\", columns=[\"visit_code\"])\n",
+    "    .data_color(\n",
+    "        columns=[\"visit_code\"],\n",
+    "        palette=[\"lavender\", \"thistle\"],\n",
+    "        domain=[2000, 5000],\n",
+    "        na_color=\"white\"\n",
+    "    )\n",
+    "    .fmt_number(columns=[\"10\", \"20\", \"30\", \"40\", \"60\", \"total\"], decimals=0)\n",
+    "    .tab_source_note(source_note=f\"Scheduled appointment date is on or after {cutoff_date.strftime('%d %B %Y')} and before {end_of_trial_date.strftime('%d %B %Y')}.\")\n",
+    ")\n",
+    "html_data.append(gt.as_raw_html())\n",
+    "gt.show()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "12",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Table 2 Visits Missed to Date as % of Visits Attended + Visits Missed\n",
+    "subject_count = (\n",
+    "    df_visit\n",
+    "    .merge(df_eos_excluded[[\"subject_identifier\"]], on=\"subject_identifier\", how=\"left\", indicator=True)\n",
+    "    .query(\"_merge=='left_only'\")\n",
+    "    .drop(columns=[\"_merge\"])\n",
+    "    .reset_index(drop=True)\n",
+    "    .query(\"visit_code_sequence==0 and appt_timing==@MISSED_APPT and ~appt_status.isin([@NEW_APPT, @CANCELLED_APPT])\")\n",
+    ").subject_identifier.nunique()\n",
+    "df_tbl = (\n",
+    "    df_visit[(df_visit.visit_code_sequence==0) & (df_visit.appt_timing==MISSED_APPT) & ~(df_visit.appt_status.isin([NEW_APPT, CANCELLED_APPT]))]\n",
+    "    .merge(df_eos_excluded[[\"subject_identifier\"]], on=\"subject_identifier\", how=\"left\", indicator=True)\n",
+    "    .query(\"_merge=='left_only'\")\n",
+    "    .drop(columns=[\"_merge\"])\n",
+    "    .reset_index(drop=True)\n",
+    "    .groupby(by=[\"visit_code\", \"site_id\"])\n",
+    "    .size()\n",
+    "    .to_frame()\n",
+    "    .reset_index()\n",
+    ")\n",
+    "df_tbl.columns = [\"visit_code\", \"site_id\", \"visits\"]\n",
+    "df_tbl_pivot = df_tbl.pivot(index=\"visit_code\", columns=\"site_id\", values=\"visits\").reset_index()\n",
+    "df_tbl_pivot.columns.name = None\n",
+    "df_tbl_pivot.columns = ['visit_code', \"10\", \"20\", \"30\", \"40\", \"60\"]\n",
+    "df_tbl_pivot['total'] = df_tbl_pivot[['10', '20', '30', '40', '60']].sum(axis=1)\n",
+    "df_missed = (\n",
+    "    df_tbl_pivot\n",
+    "    .fillna(0)\n",
+    "    .copy()\n",
+    "    .set_index([\"visit_code\"])\n",
+    ")\n",
+    "\n",
+    "df_attended_display = df_attended.copy()\n",
+    "df_attended_display = (\n",
+    "    df_attended_display\n",
+    "    .set_index([\"visit_code\"])\n",
+    ")\n",
+    "\n",
+    "attended_and_missed = df_attended_display + df_missed\n",
+    "attended_and_missed = (\n",
+    "    attended_and_missed\n",
+    "    .fillna(0)\n",
+    "    .reset_index()\n",
+    "    .set_index([\"visit_code\"])\n",
+    ")\n",
+    "\n",
+    "attended_and_missed_perc = df_missed/attended_and_missed\n",
+    "attended_and_missed_perc = (\n",
+    "    attended_and_missed_perc\n",
+    "    .fillna(0)\n",
+    "    .reset_index()\n",
+    "    .set_index([\"visit_code\"])\n",
+    ")\n",
+    "\n",
+    "df_result = df_missed.merge(attended_and_missed_perc, on=[\"visit_code\"], suffixes=(\"\", \"_perc\"))\n",
+    "for col in  [\"10\", \"20\", \"30\", \"40\", \"60\", \"total\"]:\n",
+    "    col_perc = f\"{col}_perc\"\n",
+    "    df_result[col] = df_result.apply(lambda x: f\"{x[col]} ({x[col_perc]*100:.2f})\", axis=1)\n",
+    "df_result = df_result.reset_index().sort_values(by=[\"visit_code\"], ascending=True)\n",
+    "df_result = df_result.fillna(0.0)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "13",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "df_table = df_result[[\"visit_code\", \"10\", \"20\", \"30\", \"40\", \"60\", \"total\"]].copy()\n",
+    "gt = df_as_great_table(\n",
+    "    df_table,\n",
+    "    title=\"Table 2a: Visits Missed to Date\",\n",
+    "    subtitle=\"as % of Visits Attended + Visits Missed\"\n",
+    ")\n",
+    "gt = (\n",
+    "    gt\n",
+    "    .cols_label({k:v for k, v in column_headers.items() if k!=\"label\"})\n",
+    "    .cols_align(align=\"center\", columns=[\"10\", \"20\", \"30\", \"40\", \"60\", \"total\"])\n",
+    "    .cols_align(align=\"left\", columns=[\"visit_code\", \"label\"])\n",
+    "    .tab_style(\n",
+    "        style=[style.fill(color=\"snow\"), style.text(color=\"black\")],\n",
+    "        locations=loc.body(\n",
+    "            columns=[0],\n",
+    "            rows=list(range(0, len(df_table))),\n",
+    "        ),\n",
+    "    )\n",
+    "    .tab_source_note(source_note=f\"{subject_count} participants had at least one missed visit.\")\n",
+    "\n",
+    ")\n",
+    "html_data.append(gt.as_raw_html())\n",
+    "gt.show()\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "14",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Table 2b: Number of missed visits by participant\n",
+    "subject_count = (\n",
+    "    df_visit\n",
+    "    .merge(df_eos_excluded[[\"subject_identifier\"]], on=\"subject_identifier\", how=\"left\", indicator=True)\n",
+    "    .query(\"_merge=='left_only'\")\n",
+    "    .drop(columns=[\"_merge\"])\n",
+    "    .reset_index(drop=True)\n",
+    "    .query(\"visit_code_sequence==0 and appt_timing==@MISSED_APPT and ~appt_status.isin([@NEW_APPT, @CANCELLED_APPT])\")\n",
+    ").subject_identifier.nunique()\n",
+    "df_tbl = (\n",
+    "    df_visit[(df_visit.visit_code_sequence==0) & (df_visit.appt_timing==MISSED_APPT) & ~(df_visit.appt_status.isin([NEW_APPT, CANCELLED_APPT]))]\n",
+    "    .merge(df_eos_excluded[[\"subject_identifier\"]], on=\"subject_identifier\", how=\"left\", indicator=True)\n",
+    "    .query(\"_merge=='left_only'\")\n",
+    "    .drop(columns=[\"_merge\"])\n",
+    "    .reset_index(drop=True)\n",
+    "    .groupby(by=[\"subject_identifier\", \"site_id\"])\n",
+    "    .size()\n",
+    "    .to_frame()\n",
+    "    .reset_index()\n",
+    ")\n",
+    "df_tbl.columns = [\"subject_identifier\", \"site_id\", \"missed_count\"]\n",
+    "df_tbl[\"category\"] = pd.cut(df_tbl[\"missed_count\"], bins=[0, 1, 3, 5, 7, 100], labels=[\"Missed at least 1\", \"2 to 3\", \"4 to 5\", \"6 to 7\", \"missed more than 7\"])\n",
+    "df_tbl_pivot = df_tbl.pivot_table(index=\"category\", columns=\"site_id\", values=\"missed_count\", observed=False, aggfunc=\"count\").reset_index()\n",
+    "\n",
+    "df_tbl_pivot['total'] = df_tbl_pivot.select_dtypes(include='int').sum(axis=1, skipna=True)\n",
+    "\n",
+    "sum_row = df_tbl_pivot.select_dtypes(include='int64').sum()\n",
+    "sum_row['category'] = 'Total'\n",
+    "\n",
+    "\n",
+    "df_tbl_pivot = (\n",
+    "    pd.concat([df_tbl_pivot, sum_row.to_frame().T], axis=0)\n",
+    "    .rename(columns={10: \"10\", 20: \"20\", 30: \"30\", 40: \"40\", 60: \"60\"})\n",
+    ")\n",
+    "\n",
+    "gt = df_as_great_table(\n",
+    "    df_tbl_pivot,\n",
+    "    title=\"Table 2b: Number of participants who missed one or more visits\",\n",
+    ")\n",
+    "gt = (\n",
+    "    gt\n",
+    "    .cols_label({\"category\": \"Category\", **{k:v for k, v in column_headers.items() if k not in [\"visit_code\", \"label\"]}})\n",
+    "    .cols_align(align=\"center\", columns=[\"10\", \"20\", \"30\", \"40\", \"60\", \"total\"])\n",
+    "    .cols_align(align=\"left\", columns=[\"category\"])\n",
+    "    .tab_style(\n",
+    "        style=[style.fill(color=\"snow\"), style.text(color=\"black\")],\n",
+    "        locations=loc.body(\n",
+    "            columns=[0],\n",
+    "            rows=list(range(0, len(df_table))),\n",
+    "        ),\n",
+    "    )\n",
+    ")\n",
+    "html_data.append(gt.as_raw_html())\n",
+    "gt.show()\n",
+    "\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "15",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# func for tables 3,4,5\n",
+    "def get_row_df(row_df:pd.DataFrame, label:str=None, **kwargs)->pd.DataFrame:\n",
+    "    row_df = row_df.groupby(by=[\"site_id\"]).site_id.count().to_frame(name=\"n\")\n",
+    "    row_df[\"label\"] = label\n",
+    "    row_df = row_df.reset_index()\n",
+    "    row_df = row_df.pivot(index=\"label\", values=\"n\", columns=\"site_id\").reset_index()\n",
+    "    row_df.columns.name = \"\"\n",
+    "    all_sites = [10, 20, 30, 40, 60]\n",
+    "    for site in all_sites:\n",
+    "        if site not in row_df.columns:\n",
+    "            row_df[site] = None\n",
+    "    row_df = row_df.reset_index(drop=True)\n",
+    "    return row_df\n",
+    "\n",
+    "\n",
+    "def get_table_df(\n",
+    "        df_source:pd.DataFrame,\n",
+    "        visit_code:float|None=None,\n",
+    "        month_label:str|None=None,\n",
+    "        visit_codes:list[float]|None=None,\n",
+    "        get_row_func:Callable|None=None,\n",
+    "        category_labels:list[str]|None=None,\n",
+    ")->pd.DataFrame:\n",
+    "    get_row_df_func = get_row_func or get_row_df\n",
+    "    if visit_code:\n",
+    "        df_month = df_source[df_source.visit_code==visit_code].copy()\n",
+    "    elif visit_codes:\n",
+    "        df_month = df_source[df_source.visit_code.isin(visit_codes)].copy()\n",
+    "    elif month_label:\n",
+    "        df_month = df_source.copy()\n",
+    "\n",
+    "    \n",
+    "    row_df = df_month.copy()\n",
+    "    table_df = get_row_df_func(row_df, \"Total (n)\", category_labels=category_labels)\n",
+    "    \n",
+    "    row_df = df_month.query(\"ogtt_value<7.8 and fbg_value<6.1\").copy()\n",
+    "    table_df = pd.concat([table_df, get_row_df_func(row_df, \"OGTT <7.8; FBG <6.1\", category_labels=category_labels)])\n",
+    "    \n",
+    "    row_df = df_month[(df_month.ogtt_value<7.8) & (df_month.fbg_value>=6.1) & (df_month.fbg_value<7.0)].copy()\n",
+    "    table_df = pd.concat([table_df, get_row_df_func(row_df, \"OGTT <7.8; FBG >=6.1 <7.0\", category_labels=category_labels)])\n",
+    "    \n",
+    "    row_df = df_month[(df_month.ogtt_value<7.8) & (df_month.fbg_value>=7.0)].copy()\n",
+    "    table_df = pd.concat([table_df, get_row_df_func(row_df, \"OGTT <7.8; FBG >=7.0\", category_labels=category_labels)])\n",
+    "    \n",
+    "    row_df = df_month[(df_month.ogtt_value>=7.8) & (df_month.ogtt_value<11.1) & (df_month.fbg_value<6.1)].copy()\n",
+    "    table_df = pd.concat([table_df, get_row_df_func(row_df, \"OGTT ≥7.8 to <11.1; FBG <6.1\", category_labels=category_labels)])\n",
+    "    \n",
+    "    row_df = df_month[(df_month.ogtt_value>=7.8) & (df_month.ogtt_value<11.1) & (df_month.fbg_value>=6.1) & (df_month.fbg_value<7.0)].copy()\n",
+    "    table_df = pd.concat([table_df, get_row_df_func(row_df, \"OGTT ≥7.8 to <11.1; FBG >=6.1 <7.0\", category_labels=category_labels)])\n",
+    "    \n",
+    "    row_df = df_month[(df_month.ogtt_value>=7.8) & (df_month.ogtt_value<11.1) & (df_month.fbg_value>=7.0)].copy()\n",
+    "    table_df = pd.concat([table_df, get_row_df_func(row_df, \"OGTT ≥7.8 to <11.1; FBG >=7.0\", category_labels=category_labels)])\n",
+    "    \n",
+    "    row_df = df_month[(df_month.ogtt_value>=11.1) & (df_month.fbg_value<6.1)].copy()\n",
+    "    table_df = pd.concat([table_df, get_row_df_func(row_df, \"OGTT ≥11.1; FBG <6.1\", category_labels=category_labels)])\n",
+    "    \n",
+    "    row_df = df_month[(df_month.ogtt_value>=11.1) & (df_month.fbg_value>=6.1) & (df_month.fbg_value<7.0)].copy()\n",
+    "    table_df = pd.concat([table_df, get_row_df_func(row_df, \"OGTT ≥11.1; FBG >=6.1 <7.0\", category_labels=category_labels)])\n",
+    "    \n",
+    "    row_df = df_month[(df_month.ogtt_value>=11.1) & (df_month.fbg_value>=7.0)].copy()\n",
+    "    table_df = pd.concat([table_df, get_row_df_func(row_df, \"OGTT ≥11.1; FBG >=7.0\", category_labels=category_labels)])\n",
+    "\n",
+    "    row_df = df_month[(df_month.ogtt_value.isna())].copy()\n",
+    "    table_df = pd.concat([table_df, get_row_df_func(row_df, \"Missing OGTT\", category_labels=category_labels)])\n",
+    "    return table_df\n",
+    "\n",
+    "\n",
+    "def format_table_df(tbl_df, add_totals:bool|None=None):\n",
+    "    \"\"\"Pivot on site\"\"\"\n",
+    "    add_totals = True if add_totals is None else add_totals\n",
+    "    tbl_df = tbl_df.fillna(0.0)\n",
+    "    tbl_df[\"total\"] = tbl_df.iloc[:,1:].sum(axis=1)\n",
+    "    tbl_df = tbl_df.reset_index(drop=True)\n",
+    "\n",
+    "    if add_totals:\n",
+    "        df_last = tbl_df[1:].sum().to_frame()\n",
+    "        df_last.loc[\"label\"] = np.nan\n",
+    "        df_last = df_last.reset_index()\n",
+    "        df_last.columns = [\"label\", \"value\"]\n",
+    "        df_last = df_last.pivot_table(columns=\"label\",  values=\"value\").reset_index(drop=True)\n",
+    "        df_last.columns.name = \"\"\n",
+    "        df_last[\"label\"] = \"Totals\"\n",
+    "\n",
+    "        tbl_df = pd.concat([tbl_df, df_last])\n",
+    "        tbl_df = tbl_df.reset_index(drop=True)\n",
+    "\n",
+    "    tbl_df.columns = [\"label\", \"10\", \"20\", \"30\", \"40\", \"60\", \"total\"]\n",
+    "\n",
+    "    for site in [\"10\", \"20\", \"30\", \"40\", \"60\", \"total\"]:\n",
+    "        tbl_df[f\"{site}_perc\"] = (tbl_df[site]/tbl_df.iloc[0][site]) * 100 if tbl_df.iloc[0][site]>0 else 0\n",
+    "        tbl_df[f\"{site}_perc_str\"] = tbl_df[f\"{site}_perc\"].map('{:.1f}'.format)\n",
+    "\n",
+    "\n",
+    "    for site in [\"10\", \"20\", \"30\", \"40\", \"60\", \"total\"]:\n",
+    "        tbl_df[f\"{site}_str\"] = tbl_df[[f\"{site}\", f\"{site}_perc_str\"]].apply(lambda x: ' ('.join(x.astype(str)), axis=1)\n",
+    "        tbl_df[f\"{site}_str\"] = tbl_df[f\"{site}_str\"] + \")\"\n",
+    "\n",
+    "    cols = [\"label\", *[f\"{site}_str\" for site in [\"10\", \"20\", \"30\", \"40\", \"60\", \"total\"]]]\n",
+    "    tbl_df1 = tbl_df[cols]\n",
+    "    tbl_df1.loc[tbl_df.label==\"Total (n)\"] = tbl_df.iloc[0][[\"label\", \"10\", \"20\", \"30\", \"40\", \"60\", \"total\"]].to_list()\n",
+    "    return tbl_df1\n",
+    "\n",
+    "def format_table_with_bmi_df(tbl_df, add_totals:bool|None=None, category_labels:list[str]=None):\n",
+    "    \"\"\"Pivot on BMI categories\"\"\"\n",
+    "\n",
+    "    add_totals = True if add_totals is None else add_totals\n",
+    "    tbl_df = tbl_df.fillna(0.0)\n",
+    "    tbl_df[\"total\"] = tbl_df.iloc[:,1:].sum(axis=1)\n",
+    "    tbl_df = tbl_df.reset_index(drop=True)\n",
+    "\n",
+    "    if add_totals:\n",
+    "        df_last = tbl_df[1:].sum().to_frame()\n",
+    "        df_last.loc[\"label\"] = np.nan\n",
+    "        df_last = df_last.reset_index()\n",
+    "        df_last.columns = [\"label\", \"value\"]\n",
+    "        df_last = df_last.pivot_table(columns=\"label\",  values=\"value\").reset_index(drop=True)\n",
+    "        df_last.columns.name = \"\"\n",
+    "        df_last[\"label\"] = \"Totals\"\n",
+    "\n",
+    "        tbl_df = pd.concat([tbl_df, df_last])\n",
+    "        tbl_df = tbl_df.reset_index(drop=True)\n",
+    "\n",
+    "    tbl_df.columns = [\"label\", *category_labels, \"total\"]\n",
+    "\n",
+    "    for label in [*category_labels, \"total\"]:\n",
+    "        tbl_df[f\"{label}_perc\"] = (tbl_df[label]/tbl_df.iloc[0][label]) * 100 if tbl_df.iloc[0][label]>0 else 0\n",
+    "        tbl_df[f\"{label}_perc_str\"] = tbl_df[f\"{label}_perc\"].map('{:.1f}'.format)\n",
+    "\n",
+    "    for cat in [*category_labels, \"total\"]:\n",
+    "        tbl_df[f\"{label}_str\"] = tbl_df[[f\"{label}\", f\"{label}_perc_str\"]].apply(lambda x: ' ('.join(x.astype(str)), axis=1)\n",
+    "        tbl_df[f\"{label}_str\"] = tbl_df[f\"{label}_str\"] + \")\"\n",
+    "\n",
+    "    cols = [\"label\", *[f\"{label}_str\" for label in [*category_labels, \"total\"]]]\n",
+    "    tbl_df1 = tbl_df[cols]\n",
+    "    tbl_df1.loc[tbl_df.label==\"Total (n)\"] = tbl_df.iloc[0][[\"label\", *category_labels, \"total\"]].to_list()\n",
+    "    return tbl_df1\n",
+    "\n",
+    "\n",
+    "def get_row_by_df(row_df:pd.DataFrame, label:str, category_labels:list[str])->pd.DataFrame:\n",
+    "    # if label not in category_labels:\n",
+    "    #     raise ValueError(f\"Invalid label. Expected one of {category_labels}. Got {label}.\")\n",
+    "    row_df = row_df.groupby(by=[\"site_id\"]).site_id.count().to_frame(name=\"n\")\n",
+    "    row_df[\"label\"] = label\n",
+    "    row_df = row_df.reset_index()\n",
+    "    row_df = row_df.pivot(index=\"label\", values=\"n\", columns=\"site_id\").reset_index()\n",
+    "    row_df.columns.name = \"\"\n",
+    "\n",
+    "    for label in category_labels:\n",
+    "        if label not in row_df.columns:\n",
+    "            row_df[label] = None\n",
+    "    row_df = row_df.reset_index(drop=True)\n",
+    "    return row_df"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "16",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def get_fbg_value(r):\n",
+    "    if not pd.isna(r[\"converted_fbg2_value\"]):\n",
+    "        return r[\"converted_fbg2_value\"]\n",
+    "    return r[\"converted_fbg_value\"]\n",
+    "\n",
+    "def get_ogtt_value(r):\n",
+    "    if not pd.isna(r[\"converted_ogtt2_value\"]):\n",
+    "        return r[\"converted_ogtt2_value\"]\n",
+    "    return r[\"converted_ogtt_value\"]\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "17",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Table 3: OGTT and FBG at Enrolment\n",
+    "\n",
+    "subjects = df_visit.subject_identifier.unique()\n",
+    "df_screening = get_screening_df().query(\"consented==True and subject_identifier.isin(@subjects)\")\n",
+    "df_screening[\"visit_code\"] = \"Enrol\"\n",
+    "df_screening[\"fbg_value\"] = df_screening.apply(get_fbg_value, axis=1)\n",
+    "df_screening[\"ogtt_value\"] = df_screening.apply(get_ogtt_value, axis=1)\n",
+    "df_screening[\"site_id\"] = df_screening.site.astype(int)\n",
+    "df_screening = df_screening.drop(columns=[\"site\"])\n",
+    "df_table3 = get_table_df(df_screening, month_label=\"enrol\")\n",
+    "df_table3 = format_table_df(df_table3)\n",
+    "df_table3 = df_table3.fillna(0.0)\n",
+    "gt = df_as_great_table(df_table3, title=\"Table 3a: OGTT and FBG at Screening / Enrolment\")\n",
+    "\n",
+    "column_headers_enrol = {k:v for k,v in column_headers_with_str.items() if k not in \"visit_code\"}\n",
+    "gt = (\n",
+    "    gt\n",
+    "    .cols_label(column_headers_enrol)\n",
+    "    .cols_align(align=\"center\", columns=[\"10_str\", \"20_str\", \"30_str\", \"40_str\", \"60_str\", \"total_str\"])\n",
+    "    .cols_align(align=\"left\", columns=[\"label\"])\n",
+    "    .cols_width(cases={\"label\": \"35%\"})\n",
+    "    .tab_source_note(source_note=\"Excluding patients eventually withdrawn for `late exclusion` criteria\")\n",
+    ")\n",
+    "html_data.append(gt.as_raw_html())\n",
+    "gt.show()\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "18",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# bmi_categories:\n",
+    "#     1 calculated_bmi_value<25\n",
+    "#     2 calculated_bmi_value>=25 & calculated_bmi_value<30\n",
+    "#     3 calculated_bmi_value>=30\n",
+    "\n",
+    "\n",
+    "# subjects = df_visit.subject_identifier.unique()\n",
+    "# df_screening = get_screening_df().query(\"consented==True and subject_identifier.isin(@subjects)\")\n",
+    "# df_screening[\"visit_code\"] = \"Enrol\"\n",
+    "# df_screening[\"fbg_value\"] = df_screening.apply(get_fbg_value, axis=1)\n",
+    "# df_screening[\"ogtt_value\"] = df_screening.apply(get_ogtt_value, axis=1)\n",
+    "# df_screening[\"site_id\"] = df_screening.site.astype(int)\n",
+    "# df_screening = df_screening.drop(columns=[\"site\"])\n",
+    "# df_screening[\"bmi\"] = pd.NA\n",
+    "# df_screening.loc[df_screening[\"calculated_bmi_value\"] < 25.0, \"bmi\"] = \"bmi<25\"\n",
+    "# df_screening.loc[(df_screening[\"calculated_bmi_value\"]>=25.0) & (df_screening[\"calculated_bmi_value\"] < 30.0), \"bmi\"] = \"25<=bmi<30\"\n",
+    "# df_screening.loc[df_screening[\"calculated_bmi_value\"] > 30.0, \"bmi\"] = \"bmi>30\"\n",
+    "#\n",
+    "# category_labels = [ \"bmi<25\", \"25<=bmi<30\", \"bmi>=30\", \"Total (n)\"]\n",
+    "# df_table3 = get_table_df(df_screening, month_label=\"enrol\", get_row_func=get_row_by_df, category_labels=category_labels)\n",
+    "# df_table3 = format_table_with_bmi_df(df_table3, category_labels=category_labels)\n",
+    "# df_table3 = df_table3.fillna(0.0)\n",
+    "# gt = df_as_great_table(df_table3, title=\"Table 3b: OGTT/FBG by BMI at Screening / Enrolment\")\n",
+    "# column_headers_enrol = {\"bmi<25_str\":\"bmi<25\", \"25<=bmi<30_str\":\"25<=bmi<30\", \"bmi>30_str\":\"bmi>30\", \"total_str\": \"total\"}\n",
+    "# gt = (\n",
+    "#     gt\n",
+    "#     .cols_label(column_headers_enrol)\n",
+    "#     .cols_align(align=\"center\", columns=[\"bmi<25_str\", \"25<=bmi<30_str\", \"bmi>30_str\", \"total_str\"])\n",
+    "#     .cols_align(align=\"left\", columns=[\"label\"])\n",
+    "#     .cols_width(cases={\"label\": \"35%\"})\n",
+    "#     .tab_source_note(source_note=\"Excluding patients eventually withdrawn for `late exclusion` criteria\")\n",
+    "# )\n",
+    "# html_data.append(gt.as_raw_html())\n",
+    "# gt.show()\n",
+    "\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "19",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "[col for col in df_screening.columns if \"bmi\" in col]"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "20",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Table 4: OGTT and FBG at 12-month visit\n",
+    "df_table3 = get_table_df(df_glucose, visit_codes=[1120.0])\n",
+    "df_table3 = format_table_df(df_table3)\n",
+    "df_table3 = df_table3.fillna(0.0)\n",
+    "gt = df_as_great_table(df_table3, title=\"Table 4: OGTT and FBG at 12-month visit\")\n",
+    "gt = (\n",
+    "    gt\n",
+    "    .cols_label(column_headers_with_str)\n",
+    "    .cols_align(align=\"center\", columns=[\"10_str\", \"20_str\", \"30_str\", \"40_str\", \"60_str\", \"total_str\"])\n",
+    "    .cols_align(align=\"left\", columns=[\"label\"])\n",
+    "    .cols_width(cases={\"label\": \"35%\"})\n",
+    ")\n",
+    "html_data.append(gt.as_raw_html())\n",
+    "gt.show()\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "21",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Table 5: OGTT and FBG at 24-month visit\n",
+    "df_table4 = get_table_df(df_glucose, 1240.0)\n",
+    "df_table4 = format_table_df(df_table4)\n",
+    "df_table4 = df_table4.fillna(0.0)\n",
+    "gt = df_as_great_table(df_table4, title=\"Table 5: OGTT and FBG at 24-month visit\")\n",
+    "gt = (\n",
+    "    gt\n",
+    "    .cols_label(column_headers_with_str)\n",
+    "    .cols_align(align=\"center\", columns=[\"10_str\", \"20_str\", \"30_str\", \"40_str\", \"60_str\", \"total_str\"])\n",
+    "    .cols_align(align=\"left\", columns=[\"label\"])\n",
+    "    .cols_width(cases={\"label\": \"35%\"})\n",
+    ")\n",
+    "html_data.append(gt.as_raw_html())\n",
+    "gt.show()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "22",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Table 6: OGTT and FBG at 36-month visit\n",
+    "df_table5 = get_table_df(df_glucose, 1360.0)\n",
+    "df_table5 = format_table_df(df_table5)\n",
+    "df_table5 = df_table5.fillna(0.0)\n",
+    "gt = df_as_great_table(df_table5, title=\"Table 6: OGTT and FBG at 36-month visit\")\n",
+    "gt = (\n",
+    "    gt\n",
+    "    .cols_label(column_headers_with_str)\n",
+    "    .cols_align(align=\"center\", columns=[\"10_str\", \"20_str\", \"30_str\", \"40_str\", \"60_str\", \"total_str\"])\n",
+    "    .cols_align(align=\"left\", columns=[\"label\"])\n",
+    "    .cols_width(cases={\"label\": \"35%\"})\n",
+    ")\n",
+    "html_data.append(gt.as_raw_html())\n",
+    "gt.show()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "23",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Table 7: Any OGTT>11.1 ever\n",
+    "row_df = df_glucose[df_glucose.ogtt_value>=11.1].copy()\n",
+    "table_df = get_row_df(row_df, \"Total (n)\")\n",
+    "df_table6 = format_table_df(table_df)\n",
+    "df_table = df_table6[:1].fillna(0.0).copy().reset_index(drop=True)\n",
+    "gt = df_as_great_table(df_table, title=\"Table 7: Any OGTT>11.1 ever\")\n",
+    "gt = (\n",
+    "    gt\n",
+    "    .cols_label(column_headers_with_str)\n",
+    "    .cols_align(align=\"center\", columns=[\"10_str\", \"20_str\", \"30_str\", \"40_str\", \"60_str\", \"total_str\"])\n",
+    "    .cols_align(align=\"left\", columns=[\"label\"])\n",
+    "    .cols_width(cases={\"label\": \"35%\"})\n",
+    ")\n",
+    "html_data.append(gt.as_raw_html())\n",
+    "gt.show()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "24",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# func for table 7\n",
+    "def get_table7_df(df_source:pd.DataFrame, visit_code:float)->pd.DataFrame:\n",
+    "    df_month = df_source[(df_source.visit_code>=visit_code) & (df_source.visit_code<=visit_code + 0.9)].copy()\n",
+    "\n",
+    "    row_df = df_month.copy()\n",
+    "    table_df = get_row_df(row_df, \"Total (n)\")\n",
+    "\n",
+    "    row_df = df_month[(df_month.fbg_value<6.1)].copy()\n",
+    "    table_df = pd.concat([table_df, get_row_df(row_df, \"FBG <6.1\")])\n",
+    "\n",
+    "    row_df = df_month[(df_month.fbg_value>=6.1) & (df_month.fbg_value<7.0)].copy()\n",
+    "    table_df = pd.concat([table_df, get_row_df(row_df, \"FBG >=6.1 <7.0\")])\n",
+    "\n",
+    "    row_df = df_month[(df_month.fbg_value>=7.0)].copy()\n",
+    "    table_df = pd.concat([table_df, get_row_df(row_df, \"FBG >=7.0\")])\n",
+    "    return table_df"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "25",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Table 8: Interim FBG results\n",
+    "df_table7 = get_table7_df(df_glucose, 1150.0)\n",
+    "df_table7 = format_table_df(df_table7, add_totals=False)\n",
+    "df_table7[\"visit_code\"] = MONTH15\n",
+    "\n",
+    "df_table71 = get_table7_df(df_glucose, 1180.0)\n",
+    "df_table71 = format_table_df(df_table71, add_totals=False)\n",
+    "df_table71[\"visit_code\"] = MONTH18\n",
+    "\n",
+    "df_table72 = get_table7_df(df_glucose, 1210.0)\n",
+    "df_table72 = format_table_df(df_table72, add_totals=False)\n",
+    "df_table72[\"visit_code\"] = MONTH21\n",
+    "\n",
+    "df_table73 = get_table7_df(df_glucose, 1270.0)\n",
+    "df_table73 = format_table_df(df_table73, add_totals=False)\n",
+    "df_table73[\"visit_code\"] = MONTH27\n",
+    "\n",
+    "df_table74 = get_table7_df(df_glucose, 1300.0)\n",
+    "df_table74 = format_table_df(df_table74, add_totals=False)\n",
+    "df_table74[\"visit_code\"] = MONTH30\n",
+    "\n",
+    "df_table75 = get_table7_df(df_glucose, 1330.0)\n",
+    "df_table75 = format_table_df(df_table75, add_totals=False)\n",
+    "df_table75[\"visit_code\"] = MONTH33\n",
+    "\n",
+    "df_table76 = get_table7_df(df_glucose, 1390.0)\n",
+    "df_table76 = format_table_df(df_table76, add_totals=False)\n",
+    "df_table76[\"visit_code\"] = MONTH39\n",
+    "\n",
+    "df_table = pd.concat([df_table7, df_table71, df_table72, df_table73, df_table74, df_table75, df_table76])\n",
+    "df_table = df_table.reset_index(drop=True)\n",
+    "df_table = df_table.fillna(0.0)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "26",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "column_headers_with_str = {\"visit_code\": \"Visit Code\", **column_headers_with_str}\n",
+    "gt = df_as_great_table2(df_table, title=\"Table 8: Interim FBG results\")\n",
+    "gt = (\n",
+    "    gt\n",
+    "    .cols_label(column_headers_with_str)\n",
+    "    .cols_move_to_start(columns=\"visit_code\")\n",
+    "    .cols_align(align=\"center\", columns=[\"10_str\", \"20_str\", \"30_str\", \"40_str\", \"60_str\", \"total_str\"])\n",
+    "    .cols_align(align=\"left\", columns=[\"visit_code\", \"label\"])\n",
+    "    .cols_width(cases={\"label\": \"15%\"})\n",
+    "    .tab_style(\n",
+    "        style=[\n",
+    "            style.text(color=\"black\", weight=\"bold\"),\n",
+    "            style.fill(color=\"lightgray\")\n",
+    "        ],\n",
+    "        locations=loc.row_groups()\n",
+    "    )\n",
+    ")\n",
+    "html_data.append(gt.as_raw_html())\n",
+    "gt.show()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "27",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Table 9: Primary Endpoint met\n",
+    "df_endpoint_grp = df_endpoint.groupby(by=[\"site_id\", \"endpoint_label\"]).size().to_frame().reset_index()\n",
+    "df_endpoint_grp.columns = [\"site_id\", \"label\", \"endpoints\"]\n",
+    "df_endpoint_pivot = df_endpoint_grp.pivot_table(index=\"label\", columns=\"site_id\", values=\"endpoints\").reset_index()\n",
+    "df_endpoint_pivot.columns.name = \"\"\n",
+    "df_endpoint_pivot.columns = ['label', \"10\", \"20\", \"30\", \"40\", \"60\"]\n",
+    "df_endpoint_pivot.loc[len(df_endpoint_pivot)] = df_endpoint_pivot[['10', '20', '30', '40', '60']].sum().to_dict()\n",
+    "df_endpoint_pivot.at[len(df_endpoint_pivot)-1, 'label'] = 'Total'\n",
+    "df_endpoint_pivot['total'] = df_endpoint_pivot[['10', '20', '30', '40', '60']].sum(axis=1)\n",
+    "df_endpoint_pivot = df_endpoint_pivot.fillna(0.0)\n",
+    "\n",
+    "gt = df_as_great_table(\n",
+    "    df_endpoint_pivot,\n",
+    "    title=\"Table 9a: Primary Endpoint met\"\n",
+    ")\n",
+    "gt = (\n",
+    "    gt\n",
+    "    .cols_label({k:v for k, v in column_headers.items() if k not in [\"visit_code\"]})\n",
+    "    .cols_align(align=\"center\", columns=[\"10\", \"20\", \"30\", \"40\", \"60\", \"total\"])\n",
+    "    .cols_align(align=\"left\", columns=[\"label\"])\n",
+    "    .cols_width(cases={\"label\": \"25%\"})\n",
+    ")\n",
+    "html_data.append(gt.as_raw_html())\n",
+    "gt.show()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "28",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "#read_frame(SubjectScheduleHistory.objects.filter(offschedule_model=\"meta_prn.offschedule\"), verbose=False).rename(columns={\"site\": \"site_id\"})"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "29",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from great_tables import html\n",
+    "\n",
+    "# Table 9b: Primary Endpoint no EOS or DM Referral\n",
+    "df_subjecthistory = read_frame(SubjectScheduleHistory.objects.filter(offschedule_model=\"meta_prn.offschedule\", offschedule_datetime__isnull=False), verbose=False).rename(columns={\"site\": \"site_id\"})\n",
+    "df_subjecthistory[\"site_id\"] = df_subjecthistory[\"site_id\"].astype(str)\n",
+    "df_endpoint_no_off = df_endpoint.merge(df_subjecthistory[[\"subject_identifier\", \"offschedule_datetime\"]], on=[\"subject_identifier\"], how=\"left\")\n",
+    "df_endpoint_grp = df_endpoint_no_off.query(\"offschedule_datetime.isna()\").groupby(by=[\"site_id\", \"endpoint_label\"]).size().to_frame().reset_index()\n",
+    "df_endpoint_grp.columns = [\"site_id\", \"label\", \"endpoints\"]\n",
+    "df_endpoint_pivot = df_endpoint_grp.pivot_table(index=\"label\", columns=\"site_id\", values=\"endpoints\").reset_index()\n",
+    "df_endpoint_pivot.columns.name = \"\"\n",
+    "df_endpoint_pivot.columns = ['label', *[str(col) for col in df_endpoint_pivot.columns if col != \"label\"]]\n",
+    "for col in [c for c in ['label', \"10\", \"20\", \"30\", \"40\", \"60\"] if str(c) not in df_endpoint_pivot.columns]:\n",
+    "    df_endpoint_pivot[str(col)] = np.nan\n",
+    "df_endpoint_pivot.columns = ['label', \"10\", \"20\", \"30\", \"40\", \"60\"]\n",
+    "df_endpoint_pivot.loc[len(df_endpoint_pivot)] = df_endpoint_pivot[['10', '20', '30', '40', '60']].sum().to_dict()\n",
+    "df_endpoint_pivot.at[len(df_endpoint_pivot)-1, 'label'] = 'Total'\n",
+    "df_endpoint_pivot['total'] = df_endpoint_pivot[['10', '20', '30', '40', '60']].sum(axis=1)\n",
+    "df_endpoint_pivot = df_endpoint_pivot.fillna(0.0)\n",
+    "subjects = df_endpoint_no_off.query(\"offschedule_datetime.isna()\").subject_identifier.to_list()\n",
+    "\n",
+    "gt = df_as_great_table(\n",
+    "    df_endpoint_pivot,\n",
+    "    title=\"Table 9b: Primary Endpoint met -- participant not referred\"\n",
+    ")\n",
+    "gt = (\n",
+    "    gt\n",
+    "    .cols_label({k:v for k, v in column_headers.items() if k not in [\"visit_code\"]})\n",
+    "    .cols_align(align=\"center\", columns=[\"10\", \"20\", \"30\", \"40\", \"60\", \"total\"])\n",
+    "    .cols_align(align=\"left\", columns=[\"label\"])\n",
+    "    .cols_width(cases={\"label\": \"25%\"})\n",
+    "    .tab_source_note(source_note=html(\"<BR>\".join(subjects)))\n",
+    ")\n",
+    "html_data.append(gt.as_raw_html())\n",
+    "gt.show()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "30",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "31",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Table 10: Incident Rate per 1000 person years\n",
+    "\n",
+    "def get_df_main(df_visit:pd.DataFrame, lower_days:float|None=None, upper_days:float|None=None):\n",
+    "    if not lower_days:\n",
+    "        lower_days = -1\n",
+    "    cutoff_datetime = df_visit.query(\"@lower_days<followup_days<=@upper_days\").visit_datetime.max()\n",
+    "    # exclude subjects for this reason\n",
+    "    offstudy_reasons = ['Patient fulfilled late exclusion criteria (due to abnormal blood values or raised blood pressure at enrolment']\n",
+    "\n",
+    "    df_eos = get_eos_df()\n",
+    "    df_eos_excluded = (\n",
+    "        df_eos\n",
+    "        .query(\"followup_days>@lower_days and followup_days<=@upper_days and offstudy_reason.isin(@offstudy_reasons)\")\n",
+    "        .copy()\n",
+    "        .reset_index()\n",
+    "    )\n",
+    "    df_visit_final = (\n",
+    "        df_visit.query(\"@lower_days<followup_days<=@upper_days and reason!='missed' and visit_code<2000.0\")\n",
+    "        .merge(df_eos_excluded[[\"subject_identifier\"]], on=\"subject_identifier\", how=\"left\", suffixes=(\"\", \"_y\"), indicator=True)\n",
+    "        .query(\"_merge=='left_only'\")\n",
+    "        .drop(columns=[\"_merge\"])\n",
+    "    )\n",
+    "    df_main = (\n",
+    "        df_visit_final\n",
+    "        .groupby(by=[\"subject_identifier\"])[[\"baseline_datetime\", \"visit_datetime\", \"followup_days\"]]\n",
+    "        .max()\n",
+    "        .reset_index()\n",
+    "    )\n",
+    "\n",
+    "    df_main = (\n",
+    "        df_main\n",
+    "        .merge(\n",
+    "            df_endpoint.query(\"days_to_endpoint>@lower_days\")[[\"subject_identifier\", \"endpoint_label\", \"endpoint_type\", \"days_to_endpoint\"]],\n",
+    "            how=\"left\",\n",
+    "            on=[\"subject_identifier\"])\n",
+    "        .reset_index(drop=True)\n",
+    "    )\n",
+    "    if lower_days>=365.25:\n",
+    "        df_main[\"followup_days\"] = df_main[\"followup_days\"] - lower_days\n",
+    "    df_main[\"followup_years\"] = df_main[\"followup_days\"]/365.25\n",
+    "    return df_main, len(df_main), len(df_main.query(\"@lower_days<days_to_endpoint<=@upper_days and endpoint_label.notna()\"))\n",
+    "\n",
+    "def get_rate_and_ci(events, person_years_total):\n",
+    "    lower_ci = (chi2.ppf(0.025, 2 * events) / (2 * person_years_total)) * 1000\n",
+    "    upper_ci = (chi2.ppf(0.975, 2 * (events + 1)) / (2 * person_years_total)) * 1000\n",
+    "    return events/person_years_total*1000, lower_ci, upper_ci\n",
+    "\n",
+    "def get_incidence_data(term:str, lower_days:float, upper_days:float):\n",
+    "    data = {}\n",
+    "    df_main, subjects, events = get_df_main(df_visit, lower_days=lower_days, upper_days=upper_days)\n",
+    "    person_years_total = df_main.followup_years.sum()\n",
+    "    data.update({term:[person_years_total, subjects, events, *get_rate_and_ci(events, person_years_total)]})\n",
+    "    return data"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "32",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "incidence_data = {}\n",
+    "incidence_data.update(get_incidence_data(\"total\", lower_days=-1, upper_days=10000))\n",
+    "incidence_data.update(get_incidence_data(\"0-1 years\", lower_days=-1, upper_days=365.25))\n",
+    "incidence_data.update(get_incidence_data(\"1-2 years\", lower_days=365.25, upper_days=2 * 365.25))\n",
+    "incidence_data.update(get_incidence_data(\"2-3 years\", lower_days=2 * 365.25, upper_days=3 * 365.25))\n",
+    "incidence_data.update(get_incidence_data(\"3+ years\", lower_days=3 * 365.25, upper_days=10 * 365.25))\n",
+    "data = dict(label=[], person_years=[], subjects=[], failures=[], rate=[], lower_ci=[], upper_ci=[])\n",
+    "for k in incidence_data:\n",
+    "    data[\"label\"].append(k)\n",
+    "\n",
+    "for v in incidence_data.values():\n",
+    "    data[\"person_years\"].append(v[0])\n",
+    "    data[\"subjects\"].append(v[1])\n",
+    "    data[\"failures\"].append(v[2])\n",
+    "    data[\"rate\"].append(v[3])\n",
+    "    data[\"lower_ci\"].append(v[4])\n",
+    "    data[\"upper_ci\"].append(v[5])\n",
+    "\n",
+    "df_table9 = pd.DataFrame(data={k:v for k,v in data.items() if k!=\"subjects\"})"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "33",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "gt = df_as_great_table(\n",
+    "    df_table9,\n",
+    "    title=\"Table 10: Incident Rate per 1000 person years\",\n",
+    "    subtitle=md(\"using randomisation to diabetes/last seen\"),\n",
+    ")\n",
+    "gt = gt.fmt_number(columns=[\"person_years\", \"failures\", \"rate\", \"lower_ci\", \"upper_ci\"], decimals=2)\n",
+    "gt = (gt\n",
+    "    .cols_label({\"label\": \"Label\", \"person_years\": \"Person years\", \"failures\": \"Failures\", \"rate\": \"Rate\", \"lower_ci\": \"Lower\", \"upper_ci\": \"Upper\"})\n",
+    "    .cols_align(align=\"left\", columns=[\"label\"])\n",
+    "    .cols_align(align=\"center\", columns=[\"person_years\", \"failures\", \"rate\", \"lower_ci\", \"upper_ci\"])\n",
+    "    .tab_spanner(\n",
+    "        label=\"95%CI\",\n",
+    "        columns=[\"lower_ci\", \"upper_ci\"],\n",
+    "    )\n",
+    "    .tab_source_note(source_note=\"Excluding patients withdrawn for `late exclusion` criteria\")\n",
+    ")\n",
+    "gt.show()\n",
+    "html_data.append(gt.as_raw_html())"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "34",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Table 11: Proportion meeting primary endpoint\n",
+    "df_table10 = pd.DataFrame(data=data)\n",
+    "df_table10[\"proportion\"] = df_table10[\"failures\"]/df_table10[\"subjects\"]*100\n",
+    "gt = df_as_great_table(\n",
+    "    df_table10[[\"label\", \"subjects\", 'failures', \"proportion\"]],\n",
+    "    title=\"Table 11: Proportion meeting primary endpoint\",\n",
+    ")\n",
+    "gt = (\n",
+    "    gt\n",
+    "    .fmt_number(columns=[\"failures\", \"proportion\"], decimals=2)\n",
+    "    .cols_label({\"label\": \"Label\", \"subjects\": \"Participants\", \"failures\": \"Failures\", \"proportion\": \"%\"})\n",
+    "    .cols_align(align=\"left\", columns=[\"label\"])\n",
+    "    .cols_align(align=\"center\", columns=[\"subjects\", \"failures\", \"proportion\"])\n",
+    "    .tab_source_note(source_note=\"Excluding patients withdrawn for `late exclusion` criteria\")\n",
+    ")\n",
+    "html_data.append(gt.as_raw_html())\n",
+    "gt.show()\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "35",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "36",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# TODO: EoS should be for all 1691, not 1631\n",
+    "\n",
+    "# Table 11a: End of Study Table (for those who have completed an end of study form)\n",
+    "df_eos = get_eos_df()\n",
+    "offstudy_reasons = {\n",
+    "    \"Delivered / Completed followup from pregnancy\": \"Pregnancy\",\n",
+    "    \"Patient completed 36 months of follow-up\": \"Completed 36m\",\n",
+    "    \"Patient developed diabetes\": \"Developed diabetes\",\n",
+    "    \"Other reason (specify below)\": \"Other\",\n",
+    "    \"Patient fulfilled late exclusion criteria (due to abnormal blood values or raised blood pressure at enrolment\": \"Late exclusion\",\n",
+    "    \"Patient has been transferred to another health centre\": \"Transferred out\",\n",
+    "    \"Patient is withdrawn on CLINICAL grounds ...\": \"Withdrawal: Clinical grounds\",\n",
+    "    \"Patient lost to follow-up\": \"LTFU\",\n",
+    "    \"Patient reported/known to have died\": \"Died\",\n",
+    "    \"Patient withdrew consent to participate further\": \"Withdrawal: Consent\",\n",
+    "}\n",
+    "df_eos[\"offstudy_reason\"] = df_eos[\"offstudy_reason\"].map(offstudy_reasons)\n",
+    "df_eos[\"offstudy_reason\"] = pd.Categorical(df_eos[\"offstudy_reason\"], categories=sorted(list(offstudy_reasons.values())), ordered=True)\n",
+    "df_eos[\"site_id\"] = df_eos[\"site_id\"].astype(str)\n",
+    "df_eos_pivot = (\n",
+    "    df_eos\n",
+    "    .groupby(by=[\"offstudy_reason\", \"site_id\"],observed=True)\n",
+    "    .size()\n",
+    "    .reset_index()\n",
+    "    .pivot_table(index=\"offstudy_reason\", columns=\"site_id\", values=0, observed=True)\n",
+    "    .fillna(0)\n",
+    "    .astype(int)\n",
+    "    .reset_index()\n",
+    ")\n",
+    "df_eos_pivot[\"total\"] = df_eos_pivot[[\"10\", \"20\",\"30\",\"40\",\"60\"]].sum(axis=1)\n",
+    "df_eos_pivot.columns.name=\"\"\n",
+    "sum_row = df_eos_pivot.select_dtypes(include='int64').sum()\n",
+    "sum_row['offstudy_reason'] = 'Total'\n",
+    "sum_row_df = pd.DataFrame(sum_row).T\n",
+    "enrolled_pivot[\"offstudy_reason\"] = \"Enrolled\"\n",
+    "enrolled_pivot = enrolled_pivot[[*df_eos_pivot.columns]]\n",
+    "df_eos_pivot = pd.concat([enrolled_pivot, df_eos_pivot, sum_row_df], ignore_index=True)\n",
+    "\n",
+    "gt = df_as_great_table(\n",
+    "    df_eos_pivot,\n",
+    "    title=\"Table 12a: End of study report\",\n",
+    "    subtitle=md(\"for those who have completed an End of study report\"),\n",
+    ")\n",
+    "gt = (\n",
+    "    gt\n",
+    "    .cols_label({\"offstudy_reason\": \"Reason\", **{k:v for k,v in column_headers.items() if k not in [\"visit_code\", \"label\"]}})\n",
+    "    .cols_align(align=\"left\", columns=[\"offstudy_reason\"])\n",
+    "    .cols_align(align=\"center\", columns=[\"10\", \"20\",\"30\",\"40\",\"60\", \"total\"])\n",
+    "    .tab_style(\n",
+    "        style=[style.fill(color=\"snow\"), style.text(color=\"black\")],\n",
+    "        locations=loc.body(\n",
+    "            columns=[0],\n",
+    "            rows=[len(df_eos_pivot)-1]),\n",
+    "        )\n",
+    "    .tab_style(\n",
+    "        style=[style.fill(color=\"lightblue\"), style.text(color=\"black\")],\n",
+    "        locations=loc.body(\n",
+    "            columns=[\"10\", \"20\", \"30\", \"40\", \"60\"],\n",
+    "            rows=[len(df_eos_pivot)-1],\n",
+    "        ),\n",
+    "    )\n",
+    "    .tab_style(\n",
+    "        style=[style.fill(color=\"lightgreen\"), style.text(color=\"black\")],\n",
+    "        locations=loc.body(\n",
+    "            columns=[\"total\"],\n",
+    "            rows=[len(df_eos_pivot)-1],\n",
+    "        ),\n",
+    "    )\n",
+    "    .tab_style(\n",
+    "        style=[style.fill(color=\"snow\"), style.text(color=\"black\")],\n",
+    "        locations=loc.body(\n",
+    "            columns=[\"offstudy_reason\"],\n",
+    "            rows=[0],\n",
+    "        ),\n",
+    "    )\n",
+    ")\n",
+    "html_data.append(gt.as_raw_html())\n",
+    "gt.show()\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "37",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "38",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Table 12b: Study status\n",
+    "def get_schedule_df(df_subjecthistory:pd.DataFrame, onschedule_model:str, offschedule_model:str, mode:str)->pd.DataFrame:\n",
+    "    columns = {k:f\"{k}_{mode}\" for k in [\"10\", \"20\", \"30\", \"40\", \"60\"]}\n",
+    "    df_schedule = (\n",
+    "        df_subjecthistory\n",
+    "        .query(f\"onschedule_model==@onschedule_model and offschedule_model==@offschedule_model and offschedule_datetime.{'isna' if mode=='on' else 'notna'}()\")\n",
+    "        .groupby(by=[\"onschedule_model\", \"site_id\"])\n",
+    "        .size()\n",
+    "        .reset_index()\n",
+    "        .pivot_table(index=\"onschedule_model\", columns=\"site_id\", values=0, observed=True)\n",
+    "        .reset_index()\n",
+    "        .rename(columns={\"onschedule_model\":\"schedule\", **columns})\n",
+    "        .fillna(0)\n",
+    "        .copy()\n",
+    "    )\n",
+    "    df_schedule.columns.name = \"\"\n",
+    "    return df_schedule\n",
+    "\n",
+    "df_subjecthistory = read_frame(SubjectScheduleHistory.objects.all(), verbose=False).rename(columns={\"site\": \"site_id\"})\n",
+    "df_subjecthistory[\"site_id\"] = df_subjecthistory[\"site_id\"].astype(str)\n",
+    "\n",
+    "df_on = pd.concat([\n",
+    "    get_schedule_df(df_subjecthistory, 'meta_prn.onschedule', 'meta_prn.offschedule', \"on\"),\n",
+    "    get_schedule_df(df_subjecthistory, 'meta_prn.onscheduledmreferral', 'meta_prn.offscheduledmreferral', \"on\"),\n",
+    "    get_schedule_df(df_subjecthistory, 'meta_prn.onschedulepregnancy', 'meta_prn.offschedulepregnancy', \"on\"),\n",
+    "])\n",
+    "\n",
+    "df_on = (\n",
+    "    df_on\n",
+    "    .fillna(0)\n",
+    "    .reset_index(drop=True)\n",
+    ")\n",
+    "\n",
+    "df_off = pd.concat([\n",
+    "    get_schedule_df(df_subjecthistory, 'meta_prn.onschedule', 'meta_prn.offschedule', \"off\"),\n",
+    "    get_schedule_df(df_subjecthistory, 'meta_prn.onscheduledmreferral', 'meta_prn.offscheduledmreferral', \"off\"),\n",
+    "    get_schedule_df(df_subjecthistory, 'meta_prn.onschedulepregnancy', 'meta_prn.offschedulepregnancy', \"off\"),\n",
+    "])\n",
+    "df_off = (\n",
+    "    df_off\n",
+    "    .fillna(0)\n",
+    "    .reset_index(drop=True)\n",
+    ")\n",
+    "\n",
+    "df_status = pd.merge(df_on, df_off, on=[\"schedule\"], how=\"outer\")\n",
+    "columns = []\n",
+    "for ele in [[f\"{x}_on\", f\"{x}_off\"] for x in [\"10\", \"20\", \"30\", \"40\", \"60\"]]:\n",
+    "    columns.extend(ele)\n",
+    "df_status = df_status[[\"schedule\", *columns]]\n",
+    "df_status[\"total_on\"] = df_status[[col for col in columns if \"on\" in col]].sum(axis=1)\n",
+    "df_status[\"total_off\"] = df_status[[col for col in columns if \"off\" in col]].sum(axis=1)\n",
+    "df_status[\"total\"] = df_status[columns].sum(axis=1)\n",
+    "df_status[\"schedule\"] = df_status.schedule.map({\"meta_prn.onschedule\": \"Main trial\", \"meta_prn.onscheduledmreferral\": \"Diabetes\", \"meta_prn.onschedulepregnancy\": \"Pregnancy\"})\n",
+    "\n",
+    "gt = df_as_great_table(\n",
+    "    df_status,\n",
+    "    title=\"Table 12b: Study status\",\n",
+    "    subtitle=md(\"Calculated from Offschedule form; not End of study report\"),\n",
+    ")\n",
+    "# gt = gt.fmt_number(columns=[\"person_years\", \"failures\", \"rate\", \"lower_ci\", \"upper_ci\"], decimals=0)\n",
+    "gt = (gt\n",
+    "    .tab_source_note(\n",
+    "        source_note=(\n",
+    "            \"Note: Offschedule form is always submitted before the End of study report. \"\n",
+    "            \"When the Offschedule form is submitted, future appointments for the schedule are removed and \"\n",
+    "            \"the site staff are actioned to submit the End of study report.\"\n",
+    "        )\n",
+    "    )\n",
+    "    .cols_label({\n",
+    "        \"10_on\": \"On\", \"10_off\": \"Off\",\n",
+    "        \"20_on\": \"On\", \"20_off\": \"Off\",\n",
+    "        \"30_on\": \"On\", \"30_off\": \"Off\",\n",
+    "        \"40_on\": \"On\", \"40_off\": \"Off\",\n",
+    "        \"60_on\": \"On\", \"60_off\": \"Off\",\n",
+    "        \"total_on\": \"On\", \"total_off\": \"Off\",\n",
+    "        \"schedule\": \"Schedule\", \"total\": \"Total\"})\n",
+    "    .cols_align(align=\"center\")\n",
+    "    .cols_align(align=\"left\", columns=[\"label\"])\n",
+    "    .tab_spanner(\n",
+    "        label=\"Hindu mandal\",\n",
+    "        columns=[\"10_on\", \"10_off\"],\n",
+    "    )\n",
+    "    .tab_spanner(\n",
+    "        label=\"Amana\",\n",
+    "        columns=[\"20_on\", \"20_off\"],\n",
+    "    )\n",
+    "    .tab_spanner(\n",
+    "        label=\"Temeke\",\n",
+    "        columns=[\"30_on\", \"30_off\"],\n",
+    "    )\n",
+    "    .tab_spanner(\n",
+    "        label=\"Mwananyamala\",\n",
+    "        columns=[\"40_on\", \"40_off\"],\n",
+    "    )\n",
+    "    .tab_spanner(\n",
+    "        label=\"Mnazi Moja\",\n",
+    "        columns=[\"60_on\", \"60_off\"],\n",
+    "    )\n",
+    "    .tab_spanner(\n",
+    "        label=\"Total\",\n",
+    "        columns=[\"total_on\", \"total_off\"],\n",
+    "    )\n",
+    "    .tab_style(\n",
+    "        style=[style.fill(color=\"lightblue\"), style.text(color=\"black\")],\n",
+    "        locations=loc.body(\n",
+    "            columns=[\"10_off\", \"20_off\", \"30_off\", \"40_off\", \"60_off\"],\n",
+    "            rows=list(range(0, 1)),\n",
+    "        ),\n",
+    "    )\n",
+    "    .tab_style(\n",
+    "        style=[style.fill(color=\"lightgreen\"), style.text(color=\"black\")],\n",
+    "        locations=loc.body(\n",
+    "            columns=[\"total_off\"],\n",
+    "            rows=list(range(0, 1)),\n",
+    "        ),\n",
+    "    )\n",
+    "    .fmt_number(columns=[*[c for c in df_status.columns if c not in [\"schedule\"]]], decimals=0)\n",
+    ")\n",
+    "html_data.append(gt.as_raw_html())\n",
+    "gt.show()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "39",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Table 13: Loss to Follow Up\n",
+    "df_ltfu = read_frame(LossToFollowup.objects.all(), verbose=False).rename(columns={\"site\": \"site_id\"})\n",
+    "df_ltfu_pivot = (\n",
+    "    df_ltfu\n",
+    "    .groupby(by=[\"loss_category\", \"site_id\"],observed=True,dropna=False)\n",
+    "    .size()\n",
+    "    .reset_index()\n",
+    "    .pivot_table(index=\"loss_category\", columns=\"site_id\", values=0, observed=True,dropna=False)\n",
+    "    .fillna(0)\n",
+    "    .astype(int)\n",
+    "    .reset_index()\n",
+    ")\n",
+    "df_ltfu_pivot[\"total\"] = df_eos_pivot[[\"10\", \"20\",\"30\",\"40\",\"60\"]].sum(axis=1)\n",
+    "df_ltfu_pivot.columns.name=\"\"\n",
+    "sum_row = df_ltfu_pivot.select_dtypes(include='int64').sum()\n",
+    "sum_row['loss_category'] = 'Total'\n",
+    "sum_row_df = pd.DataFrame(sum_row).T\n",
+    "df_ltfu_pivot = pd.concat([df_ltfu_pivot, sum_row_df], ignore_index=True)\n",
+    "df_ltfu_pivot\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "40",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Table 13c: End of study report not submitted\n",
+    "\n",
+    "df1 = (\n",
+    "    df_status\n",
+    "    .query(\"schedule=='Main trial'\")[[col for col in columns if \"off\" in col]]\n",
+    "    .rename(columns=dict(zip([col for col in columns if \"off\" in col], [\"10\", \"20\",\"30\",\"40\",\"60\"])))\n",
+    "    .reset_index(drop=True)\n",
+    ")\n",
+    "df2 = (\n",
+    "    df_eos_pivot\n",
+    "    .query(\"offstudy_reason=='Total'\")[[\"10\", \"20\",\"30\",\"40\",\"60\"]]\n",
+    "    .reset_index(drop=True)\n",
+    ")\n",
+    "\n",
+    "df_eos_not_reported = df1-df2\n",
+    "df_eos_not_reported[\"schedule\"] = 'Main trial'\n",
+    "df_eos_not_reported[\"total\"] = df_eos_not_reported[[\"10\", \"20\",\"30\",\"40\",\"60\"]].sum(axis=1)\n",
+    "df_eos_not_reported = df_eos_not_reported[[\"schedule\", \"10\", \"20\",\"30\",\"40\",\"60\", \"total\"]]\n",
+    "\n",
+    "gt = df_as_great_table(\n",
+    "    df_eos_not_reported,\n",
+    "    title=\"Table 13c: End of study report not submitted\",\n",
+    "    subtitle=md(\"End of study report expected based on Offschedule form\"),\n",
+    ")\n",
+    "gt = (\n",
+    "    gt\n",
+    "    .cols_label({\"schedule\": \"Schedule\", **{k:v for k,v in column_headers.items() if k not in [\"visit_code\", \"label\"]}})\n",
+    "    .cols_align(align=\"left\", columns=[\"schedule\"])\n",
+    "    .cols_align(align=\"center\", columns=[\"10\", \"20\",\"30\",\"40\",\"60\", \"total\"])\n",
+    "    .tab_style(\n",
+    "        style=[style.fill(color=\"snow\"), style.text(color=\"black\")],\n",
+    "        locations=loc.body(\n",
+    "            columns=[0],\n",
+    "            rows=[len(df_eos_pivot)-1]),\n",
+    "        )\n",
+    "    .tab_style(\n",
+    "        style=[style.fill(color=\"lightblue\"), style.text(color=\"black\")],\n",
+    "        locations=loc.body(\n",
+    "            columns=[\"10\", \"20\", \"30\", \"40\", \"60\"],\n",
+    "            rows=[len(df_eos_pivot)-1],\n",
+    "        ),\n",
+    "    )\n",
+    "    .tab_style(\n",
+    "        style=[style.fill(color=\"lightgreen\"), style.text(color=\"black\")],\n",
+    "        locations=loc.body(\n",
+    "            columns=[\"total\"],\n",
+    "            rows=[len(df_eos_pivot)-1],\n",
+    "        ),\n",
+    "    )\n",
+    ")\n",
+    "html_data.append(gt.as_raw_html())\n",
+    "gt.show()\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "41",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Table 14: Baseline Sample"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "42",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Table 15: Consented to extended followup\n",
+    "df_consented = (\n",
+    "    read_frame(SubjectConsentV1Ext.objects.all(), verbose=False)\n",
+    "    .query(\"agrees_to_extension==@YES\")\n",
+    "    .rename(columns={\"site\": \"site_id\"})\n",
+    ")\n",
+    "df_consented[\"site_id\"] = df_consented.site_id.astype(str)\n",
+    "df_consented[\"month\"] = df_consented.report_datetime.dt.strftime(\"%m\")\n",
+    "df_consented[\"year\"] = df_consented.report_datetime.dt.strftime(\"%Y\")\n",
+    "df_consented_grp = (\n",
+    "    df_consented.groupby(by=[\"site_id\", \"year\", \"month\"]).\n",
+    "    size()\n",
+    "    .reset_index()\n",
+    "    .sort_values(by=[\"site_id\", \"year\", \"month\"], ascending=True)\n",
+    "    .reset_index(drop=True)\n",
+    ")\n",
+    "df_consented_pivot = (\n",
+    "    df_consented_grp\n",
+    "    .pivot_table(index=[\"year\", \"month\"], columns=\"site_id\", values=0, aggfunc=\"sum\")\n",
+    "    .reset_index()\n",
+    "    .fillna(0)\n",
+    ")\n",
+    "if \"60\" not in df_consented_pivot.columns:\n",
+    "    df_consented_pivot[\"60\"] = 0.0 * len(df_consented_pivot)\n",
+    "df_consented_pivot.columns.name=\"\"\n",
+    "df_consented_pivot[\"year\"] = df_consented_pivot[\"year\"].astype(str)\n",
+    "df_consented_pivot[\"month\"] = df_consented_pivot[\"month\"].astype(str)\n",
+    "\n",
+    "sum_row = df_consented_pivot[[\"10\", \"20\",\"30\",\"40\",\"60\"]].sum()\n",
+    "sum_row['year'] = \"Total\"\n",
+    "sum_row['month'] = \"\"\n",
+    "df_consented_pivot = pd.concat([df_consented_pivot, sum_row.to_frame().T], ignore_index=True)\n",
+    "df_consented_pivot[\"total\"] = df_consented_pivot[[\"10\", \"20\",\"30\",\"40\",\"60\"]].sum(axis=1).astype(int)\n",
+    "df_consented_pivot[[\"10\", \"20\",\"30\",\"40\",\"60\"]] = df_consented_pivot[[\"10\", \"20\",\"30\",\"40\",\"60\"]].astype(int)\n",
+    "gt = df_as_great_table2(\n",
+    "    df_consented_pivot,\n",
+    "    title=\"Table 15: Consented to extended followup\",\n",
+    "    rowname_col=\"month\",\n",
+    "    groupname_col=\"year\",\n",
+    ")\n",
+    "gt = (\n",
+    "    gt\n",
+    "    .cols_label({\"year\": \"Year\", \"month\": \"Month\", **{k:v for k, v in column_headers.items() if k not in [\"visit_code\", \"label\"]}})\n",
+    "    .cols_align(align=\"center\")\n",
+    "    .fmt_number(columns=[\"10\", \"20\", \"30\", \"40\", \"60\", \"total\"], decimals=0)\n",
+    "    .tab_stubhead(label=\"Consented\")\n",
+    "    .tab_style(\n",
+    "        style=[\n",
+    "            style.text(color=\"black\", weight=\"bold\"),\n",
+    "            style.fill(color=\"lightgray\")\n",
+    "        ],\n",
+    "        locations=loc.row_groups()\n",
+    "    )\n",
+    ")\n",
+    "html_data.append(gt.as_raw_html())\n",
+    "gt.show()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "43",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "44",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# gather raw html\n",
+    "raw_html = [f'<div class=\"page-break\">{s}</div>' for s in html_data]\n",
+    "style_css = \"\"\"\n",
+    "<style>\n",
+    "  .page-break {\n",
+    "    page-break-inside: avoid; /* Always add page break before this element */\n",
+    "  }\n",
+    "  .table-header {\n",
+    "    font-weight: bold;\n",
+    "    font-size: 18px;\n",
+    "    text-align: center;\n",
+    "    border-bottom: None;\n",
+    "  }\n",
+    "</style>\n",
+    "\"\"\"\n",
+    "raw_html = ''.join(raw_html)\n",
+    "raw_html = f'<!DOCTYPE html>\\n<html lang=\"en\">\\n{style_css}\\n<head>\\n<meta charset=\"utf-8\"/>\\n</head>\\n<body>\\n' + document_title + raw_html + '\\n</body>\\n</html>\\n'"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "45",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# render html to PDF\n",
+    "pdfkit.from_string(raw_html, str(analysis_folder / pdf_filename),\n",
+    "options={\n",
+    "    'footer-center': 'Page [page] of [topage]',\n",
+    "    'footer-font-size': '8',\n",
+    "    'footer-spacing': '5',\n",
+    "    'encoding': \"UTF-8\",\n",
+    "    'margin-top':'10mm',\n",
+    "    'margin-right':'15mm',\n",
+    "    'margin-bottom':'15mm',\n",
+    "    'margin-left':'15mm',\n",
+    "    'header-center': study_title,\n",
+    "    'header-font-size': '6',\n",
+    "    'header-spacing': '0',\n",
+    "    'disable-javascript': None,\n",
+    "    'no-outline': None,\n",
+    "},\n",
+    "verbose=True)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "46",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.12.4"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}