meta-edc 1.0.7__py3-none-any.whl → 1.1.1__py3-none-any.whl

meta_analytics/notebooks/undiagnosed/meta_screening_table2.ipynb

@@ -0,0 +1,964 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "0",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "%%capture\n",
+    "import pandas as pd\n",
+    "import numpy as np\n",
+    "import math\n",
+    "import matplotlib.pyplot as plt\n",
+    "import scipy.stats as stats\n",
+    "\n",
+    "from dj_notebook import activate\n",
+    "\n",
+    "plus = activate(dotenv_file=\"/Users/erikvw/source/edc_source/meta-edc/.env\")\n",
+    "# output is suppressed ut normally would spew out all the edc loading messages\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "1",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# This notebook is incomplete / not working"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "2",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from edc_analytics.custom_tables import BpTable\n",
+    "from edc_analytics.table import Table\n",
+    "from meta_screening.models import SubjectScreening\n",
+    "from meta_subject.models import PhysicalExam, SubjectVisit\n",
+    "from django_pandas.io import read_frame"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "3",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "default_columns = [\"id\", \"subject_identifier\", \"report_datetime\", \"visit_code\"]\n",
+    "\n",
+    "title_row = []  # ???????????????"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "4",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# this step is slow, maybe because it is the first call to the DB\n",
+    "qs_screening = SubjectScreening.objects.all()\n",
+    "df = read_frame(qs_screening)\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "5",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# backup the df\n",
+    "df_screen = df.copy()\n",
+    "# df = df_screen.copy()\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "6",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# convert all to float\n",
+    "cols = [\"fbg_value\", \"fbg2_value\", \"ogtt_value\", \"ogtt2_value\", \"converted_fbg_value\", \n",
+    "        \"converted_fbg2_value\", \"converted_ogtt_value\", \"converted_ogtt2_value\",\n",
+    "       \"sys_blood_pressure_avg\", \"dia_blood_pressure_avg\",\n",
+    "       \"waist_circumference\"]\n",
+    "df[cols] = df[cols].apply(pd.to_numeric)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "7",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "\n",
+    "# condition to include any glucose test\n",
+    "cond_glu = (\n",
+    "    (df['fbg_value'].notna()) | \n",
+    "    (df['ogtt_value'].notna()) | \n",
+    "    (df['fbg2_value'].notna()) |\n",
+    "    (df['ogtt2_value'].notna())\n",
+    ")\n",
+    "\n",
+    "# conditions for Male/Female\n",
+    "male = (df[\"gender\"]==\"Male\")\n",
+    "female = (df[\"gender\"]==\"Female\")\n",
+    "\n",
+    "# condition for art stable\n",
+    "cond_art_stable = (df['on_rx_stable']==\"Yes\") & (df['vl_undetectable']==\"Yes\") & (df['art_six_months']==\"Yes\") \n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "8",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "9",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# lets fix some columns\n",
+    "# has_dm fillna with unk\n",
+    "df[\"has_dm\"] = df[\"has_dm\"].apply(lambda x: \"unk\" if not x else x)\n",
+    "# lets create a column that summarizes lives_nearby and staying_nearby_12\n",
+    "df[\"in_catchment\"] = (df[\"lives_nearby\"] == \"Yes\") & (df[\"staying_nearby_12\"] == \"Yes\")\n",
+    "\n",
+    "\n",
+    "# glucose\n",
+    "# are all glucose fields filled? YES\n",
+    "# for prefix in [\"fbg\", \"ogtt\", \"fbg2\", \"ogtt2\"]:\n",
+    "#     print(df[(df[f\"{prefix}_value\"].isna()) & (df[f\"converted_{prefix}_value\"].notna())][\"gender\"].count())\n",
+    "#     print(df[(df[f\"{prefix}_value\"].notna()) & (df[f\"converted_{prefix}_value\"].isna())][\"gender\"].count())\n",
+    "\n",
+    "# create fbg column\n",
+    "df[\"fbg\"] = df[\"converted_fbg_value\"]\n",
+    "df.loc[df[\"fbg\"].notna() & df[\"converted_fbg2_value\"].notna(), \"fbg\"] = df[\"converted_fbg2_value\"]\n",
+    "\n",
+    "# create ogtt column\n",
+    "df[\"ogtt\"] = df[\"converted_ogtt_value\"]\n",
+    "df.loc[df[\"ogtt\"].notna() & df[\"converted_ogtt2_value\"].notna(), \"ogtt\"] = df[\"converted_ogtt2_value\"]\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "10",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# subject SR9E8B4D has eligible part two == No but subject has a glucose value\n",
+    "df.loc[(df[\"screening_identifier\"]==\"SR9E8B4D\"), \"eligible_part_two\"] = \"Yes\"\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "11",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "\n",
+    "# condition where subject is eligible P1/P2 and has any type of glucose test\n",
+    "cond = ((df[\"eligible_part_one\"]==\"Yes\") & (df[\"eligible_part_two\"]==\"Yes\") & cond_glu)\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "12",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# filter dataframe\n",
+    "df = df[cond]\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "13",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "print(len(df))"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "14",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "wc_describe = df[\"waist_circumference\"].describe()\n",
+    "\n",
+    "# merge with physical exam to get waist circumference if taken at baseline\n",
+    "subject_identifiers = list(df[\"subject_identifier\"])\n",
+    "\n",
+    "qs_subject_visit = SubjectVisit.objects.filter(subject_identifier__in=subject_identifiers)\n",
+    "df_subject_visit = read_frame(qs_subject_visit)\n",
+    "df_subject_visit.rename(columns={\"id\": \"subject_visit\"}, inplace=True)\n",
+    "\n",
+    "qs_physical_exam = PhysicalExam.objects.filter(subject_visit__subject_identifier__in=subject_identifiers)\n",
+    "df_physical_exam = read_frame(qs_physical_exam)\n",
+    "\n",
+    "# merge w/ subject visit to get subject_identifier\n",
+    "df_physical_exam = pd.merge(df_physical_exam, df_subject_visit[[\"subject_visit\", \"subject_identifier\", \"visit_code\", \"visit_code_sequence\"]], on=\"subject_visit\", how=\"left\")\n",
+    "df_physical_exam = df_physical_exam[[\"subject_identifier\", \"visit_code\", \"visit_code_sequence\", \"waist_circumference\"]]\n",
+    "\n",
+    "df_physical_exam[[\"waist_circumference\"]] = df[[\"waist_circumference\"]].apply(pd.to_numeric)\n",
+    "\n",
+    "# rename column to waist_circumference_baseline\n",
+    "df_physical_exam[\"waist_circumference_baseline\"] = df_physical_exam[\"waist_circumference\"]\n",
+    "df_physical_exam.drop(columns=[\"waist_circumference\"])\n",
+    "\n",
+    "df_physical_exam[[\"waist_circumference_baseline\"]] = df_physical_exam[[\"waist_circumference_baseline\"]].apply(pd.to_numeric)\n",
+    "wc_baseline_describe = df_physical_exam[\"waist_circumference_baseline\"].describe()\n",
+    "\n",
+    "# merge on subject_identifier with main DF\n",
+    "df = pd.merge(df, df_physical_exam[[\"subject_identifier\", \"waist_circumference_baseline\"]], on=\"subject_identifier\", how=\"left\")\n",
+    "\n",
+    "# set waist_circumference=waist_circumference_baseline if `waist_circumference` is none and `waist_circumference_baseline` is not\n",
+    "df.loc[(df[\"waist_circumference\"].isna()) & (df[\"waist_circumference_baseline\"].notna()), \"waist_circumference\"] = df[\"waist_circumference_baseline\"]\n",
+    "\n",
+    "# drop waist_circumference_baseline\n",
+    "df.drop(columns=[\"waist_circumference_baseline\"], inplace=True)\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "15",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# gender\n",
+    "def cell(gender, all=None):\n",
+    "    cnt = df.loc[gender][\"gender\"].count()\n",
+    "    if not all:\n",
+    "        tot = df[\"gender\"].count()\n",
+    "        return f\"{cnt} ({round(cnt/tot *100, 1)}%)\"\n",
+    "    return f\"{cnt}\"\n",
+    "\n",
+    "df_gender = pd.DataFrame(columns=default_columns)\n",
+    "class GenderTable(Table):\n",
+    "    def build_table_df(self):\n",
+    "        pass\n",
+    "\n",
+    "tbl = Table(df, label=\"Gender\", columns=default_columns, show_ncol_perc=True)\n",
+    "# df_gender.loc[0] = [\"Gender\", \"n\", cell(female), cell(male), cell((male | female), all=True)]\n",
+    "tbl.table_df\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "16",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "17",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# age\n",
+    "agef = df.loc[female][\"age_in_years\"]\n",
+    "agem = df.loc[male][\"age_in_years\"]\n",
+    "age = df[\"age_in_years\"]\n",
+    "# bins\n",
+    "bin1 = (df[\"age_in_years\"]>=18) & (df[\"age_in_years\"]<35)\n",
+    "bin2 = (df[\"age_in_years\"]>=35) & (df[\"age_in_years\"]<50)\n",
+    "bin3 = (df[\"age_in_years\"]>=50) & (df[\"age_in_years\"]<65)\n",
+    "bin4 = (df[\"age_in_years\"]>=65)\n",
+    "\n",
+    "def cell(cond, gender, all=None):\n",
+    "    cnt = df.loc[gender & cond][\"age_in_years\"].count()\n",
+    "    if not all:\n",
+    "        tot = df.loc[cond][\"age_in_years\"].count()\n",
+    "        return f\"{cnt} ({round(cnt/tot *100, 1)}%)\"\n",
+    "    tot = df[\"age_in_years\"].count()\n",
+    "    return f\"{cnt} ({round(cnt/tot *100, 1)}%)\"\n",
+    "\n",
+    "df_age = pd.DataFrame(columns=title_row)\n",
+    "\n",
+    "df_age.loc[0] = [\"Age (years)\", \"n\", agef.count(), agem.count(), age.count()]\n",
+    "df_age.loc[1] = [\n",
+    "    \"\", \"Median (IQR)\",\n",
+    "    f\"{agef.quantile().astype(int)} ({agef.quantile(0.25).astype(int)}, {agef.quantile(0.75).astype(int)})\",\n",
+    "    f\"{agem.quantile().astype(int)} ({agem.quantile(0.25).astype(int)}, {agem.quantile(0.75).astype(int)})\",\n",
+    "    f\"{age.quantile().astype(int)} ({age.quantile(0.25).astype(int)}, {age.quantile(0.75).astype(int)})\",]    \n",
+    "df_age.loc[2] = [\"\", \"18-34\", cell(female, bin1), cell(male, bin1), cell(bin1, (male | female), all=True)]\n",
+    "df_age.loc[3] = [\"\", \"35-49\", cell(female, bin2), cell(male, bin2), cell(bin2, (male | female), all=True)]\n",
+    "df_age.loc[4] = [\"\", \"50-64\", cell(female, bin3), cell(male, bin3), cell(bin3, (male | female), all=True)]\n",
+    "df_age.loc[5] = [\"\", \"65 and older\", cell(female, bin4), cell(male, bin4), cell(bin4, (male | female), all=True)]\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "18",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "19",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# waist_circumference\n",
+    "desc = df[[\"waist_circumference\"]].describe()\n",
+    "descf = df[df[\"gender\"]==\"Female\"][[\"waist_circumference\"]].describe()\n",
+    "descm = df[df[\"gender\"]==\"Male\"][[\"waist_circumference\"]].describe()\n",
+    "\n",
+    "f = f\"{descf.loc[\"50%\"].values[0]} ({descf.loc[\"25%\"].values[0]}, {descf.loc[\"75%\"].values[0]})\"\n",
+    "m = f\"{descm.loc[\"50%\"].values[0]} ({descm.loc[\"25%\"].values[0]}, {descm.loc[\"75%\"].values[0]})\"\n",
+    "all = f\"{desc.loc[\"50%\"].values[0]} ({desc.loc[\"25%\"].values[0]}, {desc.loc[\"75%\"].values[0]})\"\n",
+    "\n",
+    "df_waist = pd.DataFrame(columns=title_row)\n",
+    "\n",
+    "df_waist.loc[0] = [\"Waist circumference (cm)\", \"n\", descf.loc[\"count\"].values[0].astype(\"int64\"), descm.loc[\"count\"].values[0].astype(\"int64\"), desc.loc[\"count\"].values[0].astype(\"int64\")]\n",
+    "df_waist.loc[1] = [\"\", \"Median (IQR)\", f, m, all]\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "20",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# waist_circumference (cont)\n",
+    "# Women 88 / Men 102\n",
+    "cond_lt_102 = ((df[\"waist_circumference\"]<102.0) & (df[\"gender\"]==\"Male\")) | ((df[\"waist_circumference\"]<88.0) & (df[\"gender\"]==\"Female\"))\n",
+    "cond_gte_102 = ((df[\"waist_circumference\"]>=102.0) & (df[\"gender\"]==\"Male\")) | ((df[\"waist_circumference\"]>=88.0) & (df[\"gender\"]==\"Female\"))\n",
+    "\n",
+    "tot = df[\"waist_circumference\"].count()\n",
+    "\n",
+    "f_cnt = df[cond_lt_102 & female][\"waist_circumference\"].count()\n",
+    "f = f\"{round(f_cnt, 0)} ({round(f_cnt/tot * 100, 1)}%)\"\n",
+    "m_cnt = df[cond_lt_102 & male][\"waist_circumference\"].count()\n",
+    "m = f\"{round(m_cnt, 0)} ({round(m_cnt / tot * 100, 1) }%)\"\n",
+    "value = f\"{round(df[cond_lt_102][\"waist_circumference\"].count(), 3)} ({round(df[cond_lt_102][\"waist_circumference\"].count() / df[\"waist_circumference\"].count(), 3) * 100}%)\"\n",
+    "\n",
+    "\n",
+    "df_waist.loc[2] = [\"\", \"Women<88 / Men<102\", f, m, value]\n",
+    "\n",
+    "\n",
+    "f_cnt = df[cond_gte_102 & female][\"waist_circumference\"].count()\n",
+    "f = f\"{round(f_cnt, 0)} ({round(f_cnt/tot * 100, 1)}%)\"\n",
+    "m_cnt = df[cond_gte_102 & male][\"waist_circumference\"].count()\n",
+    "m = f\"{round(m_cnt, 0)} ({round(m_cnt / tot * 100, 1) }%)\"\n",
+    "value = f\"{round(df[cond_gte_102][\"waist_circumference\"].count(), 3)} ({round(df[cond_gte_102][\"waist_circumference\"].count() / df[\"waist_circumference\"].count(), 3) * 100}%)\"\n",
+    "\n",
+    "df_waist.loc[3] = [\"\", \"Women>=88 / Men>=102\", f, m, value]\n",
+    "                   "
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "21",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# cond_art\n",
+    "\n",
+    "def cell(gender, all=None):\n",
+    "    cnt = df.loc[gender & cond_art_stable][\"gender\"].count()\n",
+    "    if not all:\n",
+    "        tot = df.loc[cond_art_stable][\"gender\"].count()\n",
+    "        return f\"{cnt} ({round(cnt/tot *100, 1)}%)\"\n",
+    "    tot = df[\"gender\"].count()\n",
+    "    return f\"{cnt} ({round(cnt/tot *100, 1)}%)\"\n",
+    "    \n",
+    "df_art = pd.DataFrame(columns=title_row)\n",
+    "df_art.loc[0] = [\"Stable on ART\", \"\", cell(female), cell(male), cell((male | female), all=True)]\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "22",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# blood pressure\n",
+    "# print(len(df[(df[\"sys_blood_pressure_one\"].notna()) & (df[\"dia_blood_pressure_one\"].notna())]))\n",
+    "# print(len(df[(df[\"sys_blood_pressure_two\"].notna()) & (df[\"dia_blood_pressure_two\"].notna())]))\n",
+    "# print(len(df[(df[\"sys_blood_pressure_avg\"].notna()) & (df[\"dia_blood_pressure_avg\"].notna())]))"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "23",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# blood pressure\n",
+    "\n",
+    "# df_tmp = df.copy()\n",
+    "# tot = len(df_tmp)\n",
+    "# print(f\"tot={tot}\")\n",
+    "# len(df_tmp[(df_tmp[\"sys_blood_pressure_avg\"].notna()) & (df_tmp[\"dia_blood_pressure_avg\"].notna())])\n",
+    "# severe_htn_cond = (df_tmp[\"sys_blood_pressure_avg\"]>=180) | (df_tmp[\"dia_blood_pressure_avg\"]>=110)\n",
+    "# severe_htn_df = df_tmp[severe_htn_cond]\n",
+    "# print(f\"severe_htn={len(severe_htn_df)}\")\n",
+    "# df_tmp.drop(severe_htn_df.index, inplace=True)\n",
+    "\n",
+    "# htn_cond = (df_tmp[\"sys_blood_pressure_avg\"]>=140) | (df_tmp[\"dia_blood_pressure_avg\"]>=90)\n",
+    "# htn_df = df_tmp[htn_cond]\n",
+    "# print(f\"htn={len(htn_df)}\")\n",
+    "# df_tmp.drop(htn_df.index, inplace=True)\n",
+    "\n",
+    "# pre_htn_cond = (df_tmp[\"sys_blood_pressure_avg\"]>=120) | (df_tmp[\"dia_blood_pressure_avg\"]>=80)\n",
+    "# pre_htn_df = df_tmp[pre_htn_cond]\n",
+    "# print(f\"pre_htn={len(pre_htn_df)}\")\n",
+    "# df_tmp.drop(pre_htn_df.index, inplace=True)\n",
+    "\n",
+    "# normal_cond = (df_tmp[\"sys_blood_pressure_avg\"]>=90) | (df_tmp[\"dia_blood_pressure_avg\"]>=60)\n",
+    "# normal_df = df_tmp[normal_cond]\n",
+    "# print(f\"normal={len(normal_df)}\")\n",
+    "# df_tmp.drop(normal_df.index, inplace=True)\n",
+    "\n",
+    "# low_cond = (df_tmp[\"sys_blood_pressure_avg\"]>=0) | (df_tmp[\"dia_blood_pressure_avg\"]>=0)\n",
+    "# low_df = df_tmp[low_cond]\n",
+    "# print(f\"low={len(low_df)}\")\n",
+    "# df_tmp.drop(low_df.index, inplace=True)\n",
+    "\n",
+    "    \n",
+    "# def cell(dfx, gender, all=None, perc=True):\n",
+    "#     cnt = dfx.loc[gender][\"gender\"].count()\n",
+    "#     if not all:\n",
+    "#         tot = df.loc[gender][\"gender\"].count()\n",
+    "#         return f\"{cnt}\" if not perc else f\"{cnt} ({round(cnt/tot *100, 1)}%)\" \n",
+    "#     tot = df[\"gender\"].count()\n",
+    "#     return f\"{cnt}\" if not perc else f\"{cnt} ({round(cnt/tot *100, 1)}%)\"\n",
+    "\n",
+    "# def quantile(gender, colname):\n",
+    "#     q50 = df.loc[gender][colname].quantile()\n",
+    "#     q25 = df.loc[gender][colname].quantile(0.25)\n",
+    "#     q75 = df.loc[gender][colname].quantile(0.75)\n",
+    "#     return f\"{q50} ({q25}, {q75})\"\n",
+    "\n",
+    "# df_bp = pd.DataFrame(columns=title_row)\n",
+    "# df_bp.loc[0] = [\"Blood pressure at baseline (mmHg)\", \"n\", cell(df_tmp, female), cell(df_tmp, male), cell(df_tmp, (male | female), all=True, perc=False)]\n",
+    "# df_bp.loc[1] = [\"\", \"Low (<90/60)\", cell(low_df, female), cell(low_df, male), cell(low_df, (male | female), all=True)]\n",
+    "# df_bp.loc[2] = [\"\", \"Normal (<120/80)\", cell(normal_df, female), cell(normal_df, male), cell(normal_df, (male | female), all=True)]\n",
+    "# df_bp.loc[3] = [\"\", \"Pre-hypertension (<140/90)\", cell(pre_htn_df, female), cell(pre_htn_df, male), cell(pre_htn_df, (male | female), all=True)]\n",
+    "# df_bp.loc[4] = [\"\", \"Hypertension (>=140/90)\", cell(htn_df, female), cell(htn_df, male), cell(htn_df, (male | female), all=True)]\n",
+    "# df_bp.loc[5] = [\"\", \"Severe hypertension (>=180/110)\", cell(severe_htn_df, female), cell(severe_htn_df, male), cell(severe_htn_df, (male | female), all=True)]\n",
+    "# df_bp.loc[6] = [\"\", \"Systolic - median (IQR)\", quantile(female, \"sys_blood_pressure_avg\"), quantile(male, \"sys_blood_pressure_avg\"), quantile((female | male), \"sys_blood_pressure_avg\")]\n",
+    "# df_bp.loc[7] = [\"\", \"Diastolic - median (IQR)\", quantile(female, \"dia_blood_pressure_avg\"), quantile(male, \"dia_blood_pressure_avg\"), quantile((female | male), \"dia_blood_pressure_avg\")]\n",
+    "\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "24",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "25",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# columns = [\n",
+    "#     'Characteristics', 'Statistics', \"F\", \"M\", 'All', \n",
+    "#     \"fnum\",\"f_prop\",\"fq25\",\"fq50\",\"fq75\",\n",
+    "#     \"mnum\",\"m_prop\",\"mq25\",\"mq50\",\"mq75\",\n",
+    "#      \"q25\",\"q50\",\"q75\",\"tot\"]\n",
+    "\n",
+    "# class SubjectRow:\n",
+    "#     def __init__(self, gender, dfx, main_df, iqr_col=None):\n",
+    "#         self.num = dfx.loc[gender][\"gender\"].count()\n",
+    "#         self.total = len(main_df.loc[gender])\n",
+    "#         self.perc = self.num/self.total\n",
+    "#         if iqr_col:\n",
+    "#             self.q25, self.q50, self.q75 = dfx.loc[gender][iqr_col].quantile([0.25, 0.50, 0.75])\n",
+    "#         else:\n",
+    "#             self.q25, self.q50, self.q75 = np.nan,np.nan,np.nan\n",
+    "\n",
+    "# class MaleRow(SubjectRow):\n",
+    "#     def __init__(self, dfx, main_df, iqr_col=None):\n",
+    "#         super().__init__(male, dfx, main_df, iqr_col)\n",
+    "\n",
+    "# class FemaleRow(SubjectRow):\n",
+    "#     def __init__(self, dfx, main_df, iqr_col=None):\n",
+    "#         super().__init__(female, dfx, main_df, iqr_col)\n",
+    "\n",
+    "# class Row:\n",
+    "#     def __init__(self, dfx, main_df, label=None, statistic=None, iqr_col=None, columns=None):\n",
+    "#         self.m = MaleRow(dfx, main_df, iqr_col)\n",
+    "#         self.f =FemaleRow(dfx, main_df, iqr_col)\n",
+    "#         self.total = len(main_df)\n",
+    "#         self.subtotal = len(dfx)\n",
+    "#         if iqr_col:\n",
+    "#             self.q25, self.q50, self.q75 = main_df[iqr_col].quantile([0.25, 0.50, 0.75])\n",
+    "#         else:\n",
+    "#             self.q25, self.q50, self.q75 = np.nan,np.nan,np.nan        \n",
+    "#         self.label = label or \"\"\n",
+    "#         self.statistic = statistic\n",
+    "#         self.df = pd.DataFrame(columns=columns)\n",
+    "\n",
+    "#     def with_perc(total=None):\n",
+    "#         if total:\n",
+    "#             return f\"{self.num} ({round(self.num/self.total *100, 1)}%)\"\n",
+    "#         return f\"{self.num} ({round(self.num/self.subtotal *100, 1)}%)\"\n",
+    "    \n",
+    "#     def values(self):\n",
+    "#         if self.statistic==\"n\":\n",
+    "#             return [\n",
+    "#                 self.label, self.statistic, \"\", \"\", \"\",\n",
+    "#                 self.f.num, self.f.perc, self.f.q25, self.f.q50, self.f.q75,\n",
+    "#                 self.m.num, self.m.perc, self.m.q25, self.m.q50, self.m.q75,\n",
+    "#                 self.q25, self.q50, self.q75, \n",
+    "#                 self.total]\n",
+    "#         return [\n",
+    "#             self.label, self.statistic, \"\", \"\", \"\", \n",
+    "#             self.f.num, self.f.perc, self.f.q25, self.f.q50, self.f.q75, \n",
+    "#             self.m.num, self.m.perc,self.m.q25, self.m.q50, self.m.q75, \n",
+    "#             self.q25, self.q50, self.q75, \n",
+    "#             self.subtotal]\n",
+    "\n",
+    "# class Table:\n",
+    "\n",
+    "#     statistic_col = \"Statistics\"\n",
+    "#     female_col = \"F\"\n",
+    "#     male_col = \"M\"\n",
+    "#     all_col = \"All\"\n",
+    "#     n_sublabel = \"n\"\n",
+    "#     grand_total_col = \"tot\"\n",
+    "    \n",
+    "#     def __init__(self, main_df, label=None, columns=None):\n",
+    "#         self.main_df = main_df\n",
+    "#         self.table_df = pd.DataFrame(columns=columns)\n",
+    "#         self.row_zero = Row(main_df, main_df, label=label, statistic=self.n_sublabel, columns=columns)\n",
+    "\n",
+    "#         self.build_table_df()\n",
+    "        \n",
+    "#         # format string cols\n",
+    "#         self.table_df[self.female_col] = self.table_df.apply(lambda x: self.format_f_col(x), axis=1)\n",
+    "#         self.table_df[self.male_col] = self.table_df.apply(lambda x: self.format_m_col(x), axis=1)\n",
+    "#         self.table_df[self.all_col] = self.table_df.apply(lambda x: self.format_all_col(x), axis=1)\n",
+    "\n",
+    "#     def build_table_df(self):\n",
+    "#         self.table_df.loc[0] = self.row_zero.values()\n",
+    "\n",
+    "#     @property\n",
+    "#     def formatted_df(self):\n",
+    "#         return self.table_df[['Characteristics', 'Statistics', \"F\", \"M\", 'All']]\n",
+    "\n",
+    "#     def format_f_col(self, x):\n",
+    "#         if x[self.statistic_col] == self.n_sublabel:\n",
+    "#             return f\"{x.fnum}\"\n",
+    "#         elif pd.notna(x.q25):\n",
+    "#             return f\"{x.fq50} ({x.fq25},{x.fq75})\"\n",
+    "#         return f\"{x.fnum} ({round(x.fnum/self.row_zero.f.total *100, 1)}%)\" \n",
+    "\n",
+    "#     def format_m_col(self, x):\n",
+    "#         if x[self.statistic_col] == self.n_sublabel:\n",
+    "#             return f\"{x.mnum}\"\n",
+    "#         elif pd.notna(x.q25):\n",
+    "#             return f\"{x.mq50} ({x.mq25},{x.mq75})\"\n",
+    "#         return f\"{x.mnum} ({round(x.mnum/self.row_zero.m.total *100, 1)}%)\" \n",
+    "\n",
+    "#     def format_all_col(self, x):\n",
+    "#         if x[self.statistic_col] == self.n_sublabel:\n",
+    "#             return f\"{x.tot}\"\n",
+    "#         elif pd.notna(x.q25):\n",
+    "#             return f\"{x.q50} ({x.q25},{x.q75})\"\n",
+    "#         return f\"{x.tot} ({round(x.tot/self.table_df.loc[0][self.grand_total_col] *100, 1)}%)\" \n",
+    "\n",
+    "# class BpTable(Table):\n",
+    "\n",
+    "#     sys_col = \"sys_blood_pressure_avg\"\n",
+    "#     dia_col = \"dia_blood_pressure_avg\"\n",
+    "\n",
+    "#     def build_table_df(self):\n",
+    "#         self.table_df.loc[0] = self.row_zero.values()\n",
+    "#         i = 1\n",
+    "#         for key, dfx in self.get_dfs(self.main_df).items():\n",
+    "#             self.table_df.loc[i] = Row(dfx, self.main_df, label=\"\", statistic=key, columns=columns).values()\n",
+    "#             i += 1\n",
+    "#         self.table_df.loc[i+1] = Row(self.main_df, self.main_df, label=\"\", statistic=\"Systolic - median (IQR)\", iqr_col=\"sys_blood_pressure_avg\", columns=columns).values()\n",
+    "#         self.table_df.loc[i+2] = Row(self.main_df, self.main_df, label=\"\", statistic=\"Diastolic - median (IQR)\", iqr_col=\"dia_blood_pressure_avg\", columns=columns).values()        \n",
+    "    \n",
+    "#     def get_dfs(self, main_df):\n",
+    "#         dfs = {}\n",
+    "#         df_tmp = main_df.copy()\n",
+    "#         tot = len(df_tmp)\n",
+    "#         severe_htn_cond = (df_tmp[\"sys_blood_pressure_avg\"]>=180) | (df_tmp[\"dia_blood_pressure_avg\"]>=110)\n",
+    "#         severe_htn_df = df_tmp[severe_htn_cond]\n",
+    "#         dfs.update({\"Severe hypertension (>=180/110)\": severe_htn_df})\n",
+    "#         df_tmp.drop(severe_htn_df.index, inplace=True)\n",
+    "        \n",
+    "#         htn_cond = (df_tmp[\"sys_blood_pressure_avg\"]>=140) | (df_tmp[\"dia_blood_pressure_avg\"]>=90)\n",
+    "#         htn_df = df_tmp[htn_cond]\n",
+    "#         dfs.update({\"Hypertension (>=140/90)\": htn_df})\n",
+    "#         df_tmp.drop(htn_df.index, inplace=True)\n",
+    "        \n",
+    "#         pre_htn_cond = (df_tmp[\"sys_blood_pressure_avg\"]>=120) | (df_tmp[\"dia_blood_pressure_avg\"]>=80)\n",
+    "#         pre_htn_df = df_tmp[pre_htn_cond]\n",
+    "#         dfs.update({\"Pre-hypertension (<140/90)\": pre_htn_df})\n",
+    "#         df_tmp.drop(pre_htn_df.index, inplace=True)\n",
+    "        \n",
+    "#         normal_cond = (df_tmp[\"sys_blood_pressure_avg\"]>=90) | (df_tmp[\"dia_blood_pressure_avg\"]>=60)\n",
+    "#         normal_df = df_tmp[normal_cond]\n",
+    "#         dfs.update({\"Normal (<120/80)\": normal_df})\n",
+    "#         df_tmp.drop(normal_df.index, inplace=True)\n",
+    "        \n",
+    "#         low_cond = (df_tmp[\"sys_blood_pressure_avg\"]>=0) | (df_tmp[\"dia_blood_pressure_avg\"]>=0)\n",
+    "#         low_df = df_tmp[low_cond]\n",
+    "#         dfs.update({\"Low (<90/60)\": low_df})\n",
+    "#         df_tmp.drop(low_df.index, inplace=True)\n",
+    "#         dfs = dict(reversed(list(dfs.items())))\n",
+    "#         return dfs\n",
+    "\n",
+    "    \n",
+    "\n",
+    "tbl = BpTable(df, label=\"Blood pressure at baseline (mmHg)\", columns=columns)\n",
+    "tbl.formatted_df\n",
+    "        "
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "26",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "tbl.table_df"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "27",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "df_bp2"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "28",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# fbg\n",
+    "\n",
+    "def cell(measure, gender, all=None):\n",
+    "    if measure == \"<6.1\":\n",
+    "        cond  = (df[\"fbg\"]<6.1)\n",
+    "    elif measure == \"6.1-6.9\":\n",
+    "        cond  = (df[\"fbg\"]>=6.1) & (df[\"fbg\"]<7.0)\n",
+    "    elif measure == \">=7.0\":\n",
+    "        cond  = (df[\"fbg\"]>=7.0)\n",
+    "    else:\n",
+    "        cond = (df[\"fbg\"].notna())\n",
+    "    cnt = df.loc[gender & cond][\"gender\"].count()\n",
+    "    if not all:\n",
+    "        tot = df.loc[gender][\"gender\"].count()\n",
+    "        return f\"{cnt}\" if measure==\"n\" else f\"{cnt} ({round(cnt/tot *100, 1)}%)\" \n",
+    "    tot = df[\"gender\"].count()\n",
+    "    return f\"{cnt}\" if measure==\"n\" else f\"{cnt} ({round(cnt/tot *100, 1)}%)\"\n",
+    "\n",
+    "df_fbg = pd.DataFrame(columns=title_row)\n",
+    "df_fbg.loc[0] = [\"FBG (mmol/L) categories\", \"n\", cell(\"n\", female), cell(\"n\", male), cell(\"n\", (male | female), all=True)]\n",
+    "df_fbg.loc[1] = [\"\", \"<6.1\", cell(\"<6.1\", female), cell(\"<6.1\", male), cell(\"<6.1\", (male | female), all=True)]\n",
+    "df_fbg.loc[2] = [\"\", \"6.1-6.9\", cell(\"6.1-6.9\", female), cell(\"6.1-6.9\", male), cell(\"6.1-6.9\", (male | female), all=True)]\n",
+    "df_fbg.loc[3] = [\"\", \"7.0 and above\", cell(\">=7.0\", female), cell(\">=7.0\", male), cell(\">=7.0\", (male | female), all=True)]\n",
+    "\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "29",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# ogtt\n",
+    "\n",
+    "def cell(measure, gender, all=None):\n",
+    "    if measure == \"<7.7\":\n",
+    "        cond  = (df[\"ogtt\"]<7.8)\n",
+    "    elif measure == \"7.8-11.1\":\n",
+    "        cond  = (df[\"ogtt\"]>=7.8) & (df[\"ogtt\"]<11.1)\n",
+    "    elif measure == \">=11.1\":\n",
+    "        cond  = (df[\"ogtt\"]>=11.1)\n",
+    "    elif measure == \"missing\":\n",
+    "        cond = (df[\"ogtt\"].isna())\n",
+    "    else:\n",
+    "        cond = (df[\"ogtt\"].notna() | df[\"ogtt\"].isna())\n",
+    "    cnt = df.loc[gender & cond][\"gender\"].count()\n",
+    "    if not all:\n",
+    "        tot = df.loc[gender][\"gender\"].count()\n",
+    "        return f\"{cnt}\" if measure==\"n\" else f\"{cnt} ({round(cnt/tot *100, 1)}%)\" \n",
+    "    tot = df[\"gender\"].count()\n",
+    "    return f\"{cnt}\" if measure==\"n\" else f\"{cnt} ({round(cnt/tot *100, 1)}%)\"\n",
+    "\n",
+    "df_ogtt = pd.DataFrame(columns=title_row)\n",
+    "df_ogtt.loc[0] = [\"OGTT (mmol/L) categories\", \"n\", cell(\"n\", female), cell(\"n\", male), cell(\"n\", (male | female), all=True)]\n",
+    "df_ogtt.loc[1] = [\"\", \"<7.7\", cell(\"<7.7\", female), cell(\"<7.7\", male), cell(\"<7.7\", (male | female), all=True)]\n",
+    "df_ogtt.loc[2] = [\"\", \"7.8-11.1\", cell(\"7.8-11.1\", female), cell(\"7.8-11.1\", male), cell(\"7.8-11.1\", (male | female), all=True)]\n",
+    "df_ogtt.loc[3] = [\"\", \"11.1 and above\", cell(\">=11.1\", female), cell(\">=11.1\", male), cell(\">=11.1\", (male | female), all=True)]\n",
+    "df_ogtt.loc[4] = [\"\", \"not done\", cell(\"missing\", female), cell(\"missing\", male), cell(\"missing\", (male | female), all=True)]\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "30",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# fbg and ogtt\n",
+    "\n",
+    "def cell(measure, gender, all=None):\n",
+    "    if measure == \"dm1\":\n",
+    "        cond  = (df[\"ogtt\"]>=11.1) | (df[\"fbg\"]>=7.0) & (df[\"ogtt\"].notna())\n",
+    "    elif measure == \"other\":\n",
+    "        cond  = ~((df[\"ogtt\"]>=11.1) | (df[\"fbg\"]>=7.0)) & (df[\"ogtt\"].notna())\n",
+    "    elif measure == \"ogtt\":\n",
+    "        cond = (df[\"fbg\"].notna()) & (df[\"ogtt\"].isna())\n",
+    "    else:\n",
+    "        cond = (df[\"fbg\"].notna())\n",
+    "    cnt = df.loc[gender & cond][\"gender\"].count()\n",
+    "    if not all:\n",
+    "        tot = df.loc[gender][\"gender\"].count()\n",
+    "        return f\"{cnt}\" if measure==\"n\" else f\"{cnt} ({round(cnt/tot *100, 1)}%)\" \n",
+    "    tot = df[\"gender\"].count()\n",
+    "    return f\"{cnt}\" if measure==\"n\" else f\"{cnt} ({round(cnt/tot *100, 1)}%)\"\n",
+    "\n",
+    "df_fbg_ogtt = pd.DataFrame(columns=title_row)\n",
+    "df_fbg_ogtt.loc[0] = [\"OGTT & FBG (mmol/L) categories\", \"n\", cell(\"n\", female), cell(\"n\", male), cell(\"n\", (male | female), all=True)]\n",
+    "df_fbg_ogtt.loc[1] = [\"\", \"OGTT>=11.1 or FBG>=7.0\", cell(\"dm1\", female), cell(\"dm1\", male), cell(\"dm1\", (male | female), all=True)]\n",
+    "df_fbg_ogtt.loc[2] = [\"\", \"other\", cell(\"other\", female), cell(\"other\", male), cell(\"other\", (male | female), all=True)]\n",
+    "df_fbg_ogtt.loc[3] = [\"\", \"OGTT not done\", cell(\"ogtt\", female), cell(\"ogtt\", male), cell(\"ogtt\", (male | female), all=True)]\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "31",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "df_table2 = pd.concat([df_gender, df_age, df_waist, df_art, df_bp, df_fbg, df_ogtt, df_fbg_ogtt], ignore_index=True)\n",
+    "df_table2"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "32",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# blood pressure\n",
+    "# Blood pressure interested in IQR25, IQR50(median), IQR75\n",
+    "df[[\"sys_blood_pressure_avg\", \"dia_blood_pressure_avg\"]].describe()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "33",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "df_table"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "34",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# blood pressure\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "35",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "36",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "37",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "38",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "39",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "40",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import matplotlib.pyplot as plt\n",
+    "import numpy as np\n",
+    "import scipy.stats as stats\n",
+    "import math\n",
+    "import seaborn as sns\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "41",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "sns.boxplot(x=\"age_in_years\",y=\"gender\", data=df)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "42",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "sns.boxplot(x=\"fbg\",y=\"gender\", data=df)\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "43",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "sns.boxplot(x=\"ogtt\",y=\"gender\", data=df)\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "44",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "df[[\"age_in_years\", \"fbg\", \"ogtt\"]].hist()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "45",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "sns.pairplot(df[[\"calculated_bmi_value\", \"fbg\"]])"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "46",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "cond = (df[\"fbg\"]>=7.0) & (df[\"fbg\"]<=10.0)\n",
+    "sns.displot(df[cond], x=\"fbg\", hue=\"gender\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "47",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "sns.displot(df, x=\"sys_\", hue=\"gender\")"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.12.4"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}