meta-edc 1.0.7__py3-none-any.whl → 1.1.1__py3-none-any.whl

meta_analytics/notebooks/undiagnosed/meta3_screening_consort_chart.ipynb

@@ -0,0 +1,1176 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "0",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "%%capture\n",
+    "# output is suppressed but normally would spew out all the edc loading messages\n",
+    "\n",
+    "import os\n",
+    "from pathlib import Path\n",
+    "from datetime import datetime\n",
+    "import pandas as pd\n",
+    "import numpy as np\n",
+    "import math\n",
+    "# import matplotlxib.pyplot as plt\n",
+    "# import seaborn as sns\n",
+    "import scipy.stats as stats\n",
+    "\n",
+    "from dj_notebook import activate\n",
+    "\n",
+    "env_file = os.environ[\"META_ENV\"]\n",
+    "documents_folder = os.environ[\"META_DOCUMENTS_FOLDER\"]\n",
+    "report_folder = Path(documents_folder)\n",
+    "\n",
+    "plus = activate(dotenv_file=env_file)\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "1",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from meta_screening.models import SubjectScreening\n",
+    "from django_pandas.io import read_frame\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "2",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "cols = [\n",
+    "    f.name\n",
+    "    for f in SubjectScreening._meta.get_fields()\n",
+    "    if f.name\n",
+    "    not in [\n",
+    "        \"contact_number\",\n",
+    "        \"initials\",\n",
+    "        \"hospital_identifier\",\n",
+    "        \"modified\",\n",
+    "        \"user_created\",\n",
+    "        \"user_modified\",\n",
+    "        \"hostname_created\",\n",
+    "        \"hostname_modified\",\n",
+    "        \"device_created\",\n",
+    "        \"device_modified\",\n",
+    "        \"locale_created\",\n",
+    "        \"locale_modified\",\n",
+    "        \"slug\",\n",
+    "    ]\n",
+    "]\n",
+    "qs_screening = SubjectScreening.objects.values(*cols).all()\n",
+    "df = read_frame(qs_screening)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "3",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "df.count()\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "4",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from edc_constants.constants import NO, YES\n",
+    "df.count()\n",
+    "df.hiv_pos.value_counts()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "5",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "df = df.drop(df[df[\"hiv_pos\"] == \"No\"].index)\n",
+    "# df = df.drop(df[df.art_six_months==NO].index)\n",
+    "# df = df.drop(df[df.on_rx_stable==NO].index)\n",
+    "df.count()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "6",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# check for duplicate subjects / there are none\n",
+    "# df[df.duplicated([\"hospital_identifier\"], keep=False)]\n",
+    "# len(df)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "7",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "8",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "len(df)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "9",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "df_tmp = df.gender.value_counts().to_frame().reset_index()\n",
+    "df_tmp[\"total\"] = df_tmp[\"count\"].sum()\n",
+    "df_tmp[\"prop\"] = df_tmp[\"count\"] / df_tmp[\"total\"]\n",
+    "df_tmp"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "10",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "df_tmp = df.has_dm.value_counts(dropna=False).to_frame().reset_index()\n",
+    "df_tmp[\"total\"] = df_tmp[\"count\"].sum()\n",
+    "df_tmp[\"prop\"] = df_tmp[\"count\"] / df_tmp[\"total\"]\n",
+    "df_tmp"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "11",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "df_tmp = df[df.has_dm.isna()].gender.value_counts(dropna=False).to_frame().reset_index()\n",
+    "df_tmp[\"total\"] = df_tmp[\"count\"].sum()\n",
+    "df_tmp[\"prop\"] = df_tmp[\"count\"] / df_tmp[\"total\"]\n",
+    "df_tmp"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "12",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "cond1 = (df.has_dm==NO) & (df.on_dm_medication==NO)\n",
+    "df_tmp = df[cond1].gender.value_counts(dropna=False).to_frame().reset_index()\n",
+    "df_tmp[\"total\"] = df_tmp[\"count\"].sum()\n",
+    "df_tmp[\"prop\"] = df_tmp[\"count\"] / df_tmp[\"total\"]\n",
+    "df_tmp"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "13",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "cond2 = (cond1 & (df.on_rx_stable==YES) & (df.art_six_months==YES) & (df.vl_undetectable==YES))\n",
+    "df_tmp = df[cond2].gender.value_counts(dropna=False).to_frame().reset_index()\n",
+    "df_tmp[\"total\"] = df_tmp[\"count\"].sum()\n",
+    "df_tmp[\"prop\"] = df_tmp[\"count\"] / df_tmp[\"total\"]\n",
+    "df_tmp"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "14",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "cond3 = (cond2 & (df.staying_nearby_12==YES) & (df.lives_nearby==YES))\n",
+    "df_tmp = df[cond3].gender.value_counts(dropna=False).to_frame().reset_index()\n",
+    "df_tmp[\"total\"] = df_tmp[\"count\"].sum()\n",
+    "df_tmp[\"prop\"] = df_tmp[\"count\"] / df_tmp[\"total\"]\n",
+    "df_tmp"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "15",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "cond4 = (cond3 & ~(df.pregnant==YES))\n",
+    "df_tmp = df[cond4].gender.value_counts(dropna=False).to_frame().reset_index()\n",
+    "df_tmp[\"total\"] = df_tmp[\"count\"].sum()\n",
+    "df_tmp[\"prop\"] = df_tmp[\"count\"] / df_tmp[\"total\"]\n",
+    "df_tmp"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "16",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "cond5 = (cond4 & (df.congestive_heart_failure==NO) & (df.liver_disease==NO) & (df.alcoholism==NO) & (df.acute_metabolic_acidosis==NO) & (df.renal_function_condition==NO) & (df.tissue_hypoxia_condition==NO) & (df.acute_condition==NO) & (df.metformin_sensitivity==NO))\n",
+    "\n",
+    "df_tmp = df[cond5].gender.value_counts(dropna=False).to_frame().reset_index()\n",
+    "df_tmp[\"total\"] = df_tmp[\"count\"].sum()\n",
+    "df_tmp[\"prop\"] = df_tmp[\"count\"] / df_tmp[\"total\"]\n",
+    "df_tmp"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "17",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "df[cond5].eligible_part_one.value_counts()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "18",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "df[cond5].eligible_part_two.value_counts()\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "19",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "cond6 = (cond5 & (df.meta_phase_two==NO))\n",
+    "df_tmp = df[cond6].gender.value_counts(dropna=False).to_frame().reset_index()\n",
+    "df_tmp[\"total\"] = df_tmp[\"count\"].sum()\n",
+    "df_tmp[\"prop\"] = df_tmp[\"count\"] / df_tmp[\"total\"]\n",
+    "df_tmp"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "20",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "cond7 = (cond6 & (df[\"eligible_part_one\"]==\"Yes\") & (df[\"eligible_part_two\"]==\"Yes\") & (df.agree_to_p3==YES))\n",
+    "df_tmp = df[cond7].gender.value_counts(dropna=False).to_frame().reset_index()\n",
+    "df_tmp[\"total\"] = df_tmp[\"count\"].sum()\n",
+    "df_tmp[\"prop\"] = df_tmp[\"count\"] / df_tmp[\"total\"]\n",
+    "df_tmp"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "21",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "22",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "cond8 = (cond7 & (df.already_fasted==YES))\n",
+    "df_tmp = df[cond8].gender.value_counts(dropna=False).to_frame().reset_index()\n",
+    "df_tmp[\"total\"] = df_tmp[\"count\"].sum()\n",
+    "df_tmp[\"prop\"] = df_tmp[\"count\"] / df_tmp[\"total\"]\n",
+    "df_tmp"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "23",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "cond9 = (cond7 & (df.already_fasted==NO))\n",
+    "df_tmp = df[cond9].gender.value_counts(dropna=False).to_frame().reset_index()\n",
+    "df_tmp[\"total\"] = df_tmp[\"count\"].sum()\n",
+    "df_tmp[\"prop\"] = df_tmp[\"count\"] / df_tmp[\"total\"]\n",
+    "df_tmp"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "24",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from edc_model.utils import duration_hm_to_timedelta\n",
+    "\n",
+    "# gen fasted variable\n",
+    "def get_duration_dh_to_timedelta(s):\n",
+    "    if not pd.isna(s[\"fasting_duration_str\"]):\n",
+    "        return duration_hm_to_timedelta(s[\"fasting_duration_str\"])\n",
+    "    return s[\"fasting_duration_str\"]\n",
+    "\n",
+    "def get_fasted(s):\n",
+    "    if pd.isna(s[\"fasted_duration_delta\"]) and not has_glucose_value(s):\n",
+    "        return None\n",
+    "    elif pd.isna(s[\"fasted_duration_delta\"]) and has_glucose_value(s):\n",
+    "        return has_glucose_value(s)\n",
+    "    if s[\"fasted_duration_delta\"] <= pd.Timedelta(hours=8):\n",
+    "        return NO\n",
+    "    return YES\n",
+    "\n",
+    "def has_glucose_value(s):\n",
+    "    if not pd.isna(s[\"fbg_value\"]):\n",
+    "        return \"FBG only\"\n",
+    "    if not pd.isna(s[\"ogtt_value\"]) and not pd.isna(s[\"fbg_value\"]):\n",
+    "        return \"FBG-OGTT\"\n",
+    "    elif pd.isna(s[\"ogtt_value\"]) and pd.isna(s[\"ogtt2_value\"]) and pd.isna(s[\"fbg_value\"]) and pd.isna(s[\"fbg2_value\"]):\n",
+    "        return False\n",
+    "    return True\n",
+    "\n",
+    "df[\"fasted_duration_delta\"] = df.apply(get_duration_dh_to_timedelta, axis=1)\n",
+    "df[\"fasted\"] = df.apply(get_fasted, axis=1)\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "25",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "df[(df.subject_identifier.notna()) & (df.subject_identifier.str.len() < 20)].eligible.value_counts()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "26",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "df_tmp = df[cond9 & (df.fasted==YES)].gender.value_counts(dropna=False).to_frame().reset_index()\n",
+    "df_tmp[\"total\"] = df_tmp[\"count\"].sum()\n",
+    "df_tmp[\"prop\"] = df_tmp[\"count\"] / df_tmp[\"total\"]\n",
+    "df_tmp"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "27",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "df_tmp = df[cond9 & ~(df.fasted==YES)].gender.value_counts(dropna=False).to_frame().reset_index()\n",
+    "df_tmp[\"total\"] = df_tmp[\"count\"].sum()\n",
+    "df_tmp[\"prop\"] = df_tmp[\"count\"] / df_tmp[\"total\"]\n",
+    "df_tmp"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "28",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "df[cond9 & ~(df.fasted==YES) & (df.subject_identifier.str.len() < 20)][[\"screening_identifier\", \"subject_identifier\"]]"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "29",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# never returned or not evaluated\n",
+    "# note some have part three started and 1 even has a fasting duration\n",
+    "cond10 = (cond9 & (df.eligible_part_three==\"To be determined\"))\n",
+    "# df[(df.fasted==YES) & cond_eligible].eligible_part_one.value_counts(dropna=False)\n",
+    "df_tmp = df[cond10].gender.value_counts(dropna=False).to_frame().reset_index()\n",
+    "df_tmp[\"total\"] = df_tmp[\"count\"].sum()\n",
+    "df_tmp[\"prop\"] = df_tmp[\"count\"] / df_tmp[\"total\"]\n",
+    "df_tmp"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "30",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "cond11 = (cond7 & (df.fasted==YES))\n",
+    "df_tmp = df[cond11].gender.value_counts(dropna=False).to_frame().reset_index()\n",
+    "df_tmp[\"total\"] = df_tmp[\"count\"].sum()\n",
+    "df_tmp[\"prop\"] = df_tmp[\"count\"] / df_tmp[\"total\"]\n",
+    "df_tmp"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "31",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "cond11 = (cond9 & ~(df.eligible_part_three==\"To be determined\") & (df.fasted==NO))\n",
+    "df_tmp = df[cond11].gender.value_counts(dropna=False).to_frame().reset_index()\n",
+    "df_tmp[\"total\"] = df_tmp[\"count\"].sum()\n",
+    "df_tmp[\"prop\"] = df_tmp[\"count\"] / df_tmp[\"total\"]\n",
+    "df_tmp"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "32",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "cond11 = (cond9 & ~(df.eligible_part_three==\"To be determined\") & ~(df.fbg_value.isna()))\n",
+    "df_tmp = df[cond11].gender.value_counts(dropna=False).to_frame().reset_index()\n",
+    "df_tmp[\"total\"] = df_tmp[\"count\"].sum()\n",
+    "df_tmp[\"prop\"] = df_tmp[\"count\"] / df_tmp[\"total\"]\n",
+    "df_tmp"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "33",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "df_tmp = df.eligible_part_three.value_counts(dropna=False).to_frame().reset_index()\n",
+    "df_tmp[\"total\"] = df_tmp[\"count\"].sum()\n",
+    "df_tmp[\"prop\"] = df_tmp[\"count\"] / df_tmp[\"total\"]\n",
+    "df_tmp"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "34",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "df_tmp = df[all_conds & (df.agree_to_p3==YES) & (df.already_fasted==NO) & ~(df.fasted==YES) & cond_eligible].gender.value_counts(dropna=False).to_frame().reset_index()\n",
+    "df_tmp[\"total\"] = df_tmp[\"count\"].sum()\n",
+    "df_tmp[\"prop\"] = df_tmp[\"count\"] / df_tmp[\"total\"]\n",
+    "df_tmp\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "35",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "36",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "37",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "all_conds = (all_conds & (df.agree_to_p3==YES) & (df.already_fasted==NO))\n",
+    "cond_eligible = (df[\"eligible_part_one\"]==\"Yes\") & (df[\"eligible_part_two\"]==\"Yes\")\n",
+    "df[all_conds & cond_eligible & (df.eligible_part_three.isin([YES, NO]))].gender.value_counts(dropna=False)\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "38",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "39",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "print(len(df[(df[\"ogtt_base_datetime\"].notna()) | (df[\"ogtt2_base_datetime\"].notna())]))\n",
+    "print(len(df[(df[\"ogtt_datetime\"].notna()) | (df[\"ogtt2_datetime\"].notna())]))\n",
+    "print(len(df[(df[\"ogtt_value\"].notna()) | (df[\"ogtt2_value\"].notna())]))\n",
+    "# len(df[(df[\"fbg_value\"].notna()) | (df[\"fbg2_value\"].notna())])\n",
+    "df[((df[\"ogtt_value\"].notna()) | (df[\"ogtt2_value\"].notna()) | (df[\"fbg_value\"].notna()) | (df[\"fbg2_value\"].notna())) & (df[\"has_dm\"]==\"No\")]\n",
+    "                                                                                                                        "
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "40",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# counts by site - row, column\n",
+    "gender_by_site = pd.crosstab(df['site'], df['gender'], margins=True)\n",
+    "gender_by_site.columns = [\"F (%)\", \"M (%)\", \"(%)\"]\n",
+    "gender_by_site.index = [\"amana\", \"hindu-mandal\", \"mnazi-moja\", \"mwananyamala\", \"temeke\", \"total (%)\"]\n",
+    "gender_by_site"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "41",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "round(gender_by_site/len(df) , 3) * 100\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "42",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "\n",
+    "round(gender_by_site.div(gender_by_site[\"(%)\"], axis=0) , 3) * 100\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "43",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# has_dm fillna with unk\n",
+    "df[\"has_dm\"] = df[\"has_dm\"].apply(lambda x: \"unk\" if not x else x)\n",
+    "\n",
+    "# in_catchment =\n",
+    "df[\"in_catchment\"] = (df[\"lives_nearby\"] == \"Yes\") & (df[\"staying_nearby_12\"] == \"Yes\")\n",
+    "\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "44",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# run crosstabs"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "45",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# crosstab by has_dm, gender\n",
+    "df_crosstab = pd.crosstab(df['has_dm'], df['gender'], margins=True, dropna=False)\n",
+    "# has_dm_by_gender.columns = [\"female\", \"male\", \"rowtotal\"]\n",
+    "# has_dm_by_gender.index = [\"no\", \"yes\", \"unknown\", \"coltotal\"]\n",
+    "df_crosstab"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "46",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# crosstab by has_dm == Yes by on_dm_medication, gender\n",
+    "cond = (df[\"has_dm\"]==\"Yes\")\n",
+    "df_crosstab = pd.crosstab(df[cond]['on_dm_medication'], df[cond]['gender'], margins=True, dropna=False)\n",
+    "# has_dm_by_gender.columns = [\"female\", \"male\", \"rowtotal\"]\n",
+    "# has_dm_by_gender.index = [\"no\", \"yes\", \"unknown\", \"coltotal\"]\n",
+    "df_crosstab"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "47",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# crosstab by has_dm == No by on_dm_medication, gender\n",
+    "cond = (df[\"has_dm\"]==\"No\")\n",
+    "df_crosstab = pd.crosstab(df[cond]['on_dm_medication'], df[cond]['gender'], margins=True, dropna=False)\n",
+    "# has_dm_by_gender.columns = [\"female\", \"male\", \"rowtotal\"]\n",
+    "# has_dm_by_gender.index = [\"no\", \"yes\", \"unknown\", \"coltotal\"]\n",
+    "df_crosstab"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "48",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# crosstab by has_dm == No & on_dm_medication==No by stable on ART for 6m, gender\n",
+    "cond = (df[\"has_dm\"]==\"No\") & (df['on_dm_medication']==\"No\") & (df['on_rx_stable']==\"Yes\") & (df['vl_undetectable']==\"Yes\") & (df['art_six_months']==\"Yes\")\n",
+    "neg_cond = (df[\"has_dm\"]==\"No\") & (df['on_dm_medication']==\"No\") & ((df['on_rx_stable']!=\"Yes\") | (df['vl_undetectable']!=\"Yes\") | (df['art_six_months']!=\"Yes\"))\n",
+    "df_crosstab = pd.crosstab(df[neg_cond]['art_six_months'], df[neg_cond]['gender'], margins=True, dropna=False)\n",
+    "# has_dm_by_gender.columns = [\"female\", \"male\", \"rowtotal\"]\n",
+    "# has_dm_by_gender.index = [\"no\", \"yes\", \"unknown\", \"coltotal\"]\n",
+    "df_crosstab"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "49",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# \"lives_nearby\",\n",
+    "# \"staying_nearby_12\",\n",
+    "# crosstab by has_dm == No & on_dm_medication==No by stable on ART for 6m, gender\n",
+    "cond = (df[\"has_dm\"]==\"No\") & (df['on_dm_medication']==\"No\") & (df['on_rx_stable']==\"Yes\") & (df['vl_undetectable']==\"Yes\") & (df['art_six_months']==\"Yes\")\n",
+    "\n",
+    "df_crosstab = pd.crosstab(df[cond]['in_catchment'], df[cond]['gender'], margins=True, dropna=False)\n",
+    "# has_dm_by_gender.columns = [\"female\", \"male\", \"rowtotal\"]\n",
+    "# has_dm_by_gender.index = [\"no\", \"yes\", \"unknown\", \"coltotal\"]\n",
+    "df_crosstab"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "50",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# crosstab pregnant, gender\n",
+    "cond = (df[\"has_dm\"]==\"No\") & (df['on_dm_medication']==\"No\") & (df['on_rx_stable']==\"Yes\") & (df['vl_undetectable']==\"Yes\") & (df['art_six_months']==\"Yes\") & (df['in_catchment']==True) & (df[\"in_catchment\"]==True)\n",
+    "df_crosstab = pd.crosstab(df[cond]['pregnant'], df[cond]['gender'], margins=True, dropna=False)\n",
+    "# has_dm_by_gender.columns = [\"female\", \"male\", \"rowtotal\"]\n",
+    "# has_dm_by_gender.index = [\"no\", \"yes\", \"unknown\", \"coltotal\"]\n",
+    "df_crosstab\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "51",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# crosstab on conditions (part two)\n",
+    "# \"congestive_heart_failure\",\n",
+    "# \"liver_disease\",\n",
+    "# \"alcoholism\",\n",
+    "# \"acute_metabolic_acidosis\",\n",
+    "# \"renal_function_condition\",\n",
+    "# \"tissue_hypoxia_condition\",\n",
+    "# \"acute_condition\",\n",
+    "# \"metformin_sensitivity\","
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "52",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# crosstab (use for any single condition)\n",
+    "cond = (df[\"has_dm\"]==\"No\") & (df['on_dm_medication']==\"No\") & (df['on_rx_stable']==\"Yes\") & (df['vl_undetectable']==\"Yes\") & (df['art_six_months']==\"Yes\") & (df['in_catchment']==True) & (df[\"in_catchment\"]==True) & (df[\"pregnant\"]!=\"Yes\")\n",
+    "df_crosstab = pd.crosstab(df[cond]['metformin_sensitivity'], df[cond]['gender'], margins=True, dropna=False)\n",
+    "# has_dm_by_gender.columns = [\"female\", \"male\", \"rowtotal\"]\n",
+    "# has_dm_by_gender.index = [\"no\", \"yes\", \"unknown\", \"coltotal\"]\n",
+    "df_crosstab\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "53",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# crosstab meta_phase_two\n",
+    "cond = ((df[\"has_dm\"]==\"No\") & (df['on_dm_medication']==\"No\")\n",
+    "        & (df['on_rx_stable']==\"Yes\") & (df['vl_undetectable']==\"Yes\") & (df['art_six_months']==\"Yes\") \n",
+    "        & (df['in_catchment']==True) \n",
+    "        & (df[\"pregnant\"]!=\"Yes\")\n",
+    "        & (df[\"congestive_heart_failure\"]!=\"Yes\")\n",
+    "        & (df[\"liver_disease\"]!=\"Yes\")\n",
+    "        & (df[\"alcoholism\"]!=\"Yes\")\n",
+    "        & (df[\"acute_metabolic_acidosis\"]!=\"Yes\")\n",
+    "        & (df[\"renal_function_condition\"]!=\"Yes\")\n",
+    "        & (df[\"tissue_hypoxia_condition\"]!=\"Yes\")\n",
+    "        & (df[\"acute_condition\"]!=\"Yes\")\n",
+    "        & (df[\"metformin_sensitivity\"]!=\"Yes\")\n",
+    "       )\n",
+    "df_crosstab = pd.crosstab(df[cond]['meta_phase_two'], df[cond]['gender'], margins=True, dropna=False)\n",
+    "# has_dm_by_gender.columns = [\"female\", \"male\", \"rowtotal\"]\n",
+    "# has_dm_by_gender.index = [\"no\", \"yes\", \"unknown\", \"coltotal\"]\n",
+    "df_crosstab\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "54",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# crosstab (use for any single condition)\n",
+    "cond = ((df[\"has_dm\"]==\"No\") & (df['on_dm_medication']==\"No\")\n",
+    "        & (df['on_rx_stable']==\"Yes\") & (df['vl_undetectable']==\"Yes\") & (df['art_six_months']==\"Yes\") \n",
+    "        & (df['in_catchment']==True) \n",
+    "        & (df[\"pregnant\"]!=\"Yes\")\n",
+    "        & (df[\"congestive_heart_failure\"]!=\"Yes\")\n",
+    "        & (df[\"liver_disease\"]!=\"Yes\")\n",
+    "        & (df[\"alcoholism\"]!=\"Yes\")\n",
+    "        & (df[\"acute_metabolic_acidosis\"]!=\"Yes\")\n",
+    "        & (df[\"renal_function_condition\"]!=\"Yes\")\n",
+    "        & (df[\"tissue_hypoxia_condition\"]!=\"Yes\")\n",
+    "        & (df[\"acute_condition\"]!=\"Yes\")\n",
+    "        & (df[\"metformin_sensitivity\"]!=\"Yes\")\n",
+    "        & (df[\"meta_phase_two\"]!=\"Yes\")\n",
+    "       )\n",
+    "len(df[cond])"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "55",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# check against eligible_part_one and two\n",
+    "\n",
+    "cond_eligible = (df[\"eligible_part_one\"]==\"Yes\") & (df[\"eligible_part_two\"]==\"Yes\")\n",
+    "print([len(df[cond_eligible]), len(df[cond])])\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "56",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "cond = (df[\"eligible_part_one\"]==\"Yes\") & (df[\"eligible_part_two\"]==\"Yes\")\n",
+    "df_crosstab = pd.crosstab(df[cond]['agree_to_p3'], df[cond]['gender'], margins=True, dropna=False)\n",
+    "df_crosstab\n",
+    "\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "57",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "cond = (df[\"eligible_part_one\"]==\"Yes\") & (df[\"eligible_part_two\"]==\"Yes\")\n",
+    "df_crosstab = pd.crosstab(df[cond]['already_fasted'], df[cond]['gender'], margins=True, dropna=False)\n",
+    "df_crosstab\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "58",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# who returned and had an FBG performed\n",
+    "cond = (df[\"eligible_part_one\"]==\"Yes\") & (df[\"eligible_part_two\"]==\"Yes\")\n",
+    "df_crosstab = pd.crosstab(df[cond]['fbg_value'].notna(), df[cond]['gender'], margins=True, dropna=False)\n",
+    "df_crosstab\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "59",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# df_crosstab / len(df[cond & cond2])"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "60",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# of 5616 look at FBG and OGTT counts. Run lines for \n",
+    "# glucose: fbg_value,fbg2_value,ogtt_value,ogtt2_value,\n",
+    "# BP: sys_blood_pressure_one, sys_blood_pressure_two,dia_blood_pressure_one, dia_blood_pressure_two  \n",
+    "cond = (df[\"eligible_part_one\"]==\"Yes\") & (df[\"eligible_part_two\"]==\"Yes\") & (df[\"fbg_value\"].notna())\n",
+    "\n",
+    "df_crosstab = pd.crosstab(df[cond]['hba1c_value'].notna(), df[cond]['gender'], margins=True, dropna=False)\n",
+    "df_crosstab\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "61",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "62",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# let's look at screening glucose and BP measurements"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "63",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "cond = ((df[\"eligible_part_one\"]==\"Yes\") \n",
+    "        & (df[\"eligible_part_two\"]==\"Yes\")\n",
+    "        & (df[\"fasted\"]==\"Yes\")\n",
+    "        & ((df['fbg_value'].notna()) | (df['ogtt_value'].notna()) | (df['fbg2_value'].notna()) | (df['ogtt2_value'].notna()))\n",
+    "          )\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "64",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "cond = ((df[\"eligible_part_one\"]==\"Yes\") \n",
+    "        & (df[\"eligible_part_two\"]==\"Yes\")\n",
+    "        & (df[\"fasted\"]==\"Yes\")\n",
+    "        & (df['fbg_value'].notna())\n",
+    "          )\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "65",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "len(df[cond])"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "66",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "cond = cond & (df[\"ogtt_value\"].notna())\n",
+    "df_crosstab = pd.crosstab(df[cond]['fbg_value'].notna(), df[cond]['gender'], margins=True, dropna=False)\n",
+    "df_crosstab\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "67",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "df2 = df[cond]\n",
+    "df2[\"fbg\"] = df2[\"fbg_value\"]\n",
+    "df2.loc[df[\"fbg_value\"].notna() & df2[\"fbg2_value\"].notna(), \"fbg\"] = df2[\"fbg2_value\"]\n",
+    "df2[\"fbg\"] = pd.to_numeric(df2[\"fbg\"])"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "68",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "df2[df2[\"fbg2_value\"].notna()][[\"fbg\", \"fbg_value\",\"fbg2_value\"]]"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "69",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "df2['fbg'].describe()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "70",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "71",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "72",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# PART TWO\n",
+    "# \"congestive_heart_failure\",\n",
+    "# \"liver_disease\",\n",
+    "# \"alcoholism\",\n",
+    "# \"acute_metabolic_acidosis\",\n",
+    "# \"renal_function_condition\",\n",
+    "# \"tissue_hypoxia_condition\",\n",
+    "# \"acute_condition\",\n",
+    "# \"metformin_sensitivity\","
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "73",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# part one variables\n",
+    "\n",
+    "# \"meta_phase_two\",\n",
+    "# \"hiv_pos\",\n",
+    "# \"art_six_months\",\n",
+    "# \"on_rx_stable\",\n",
+    "# \"vl_undetectable\",\n",
+    "# \"lives_nearby\",\n",
+    "# \"staying_nearby_12\",\n",
+    "# \"pregnant\",\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "74",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "75",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# only fasted for 7h\n",
+    "df[df.subject_identifier==\"105-30-0164-8\"].to_dict()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "76",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "df[~(df.subject_identifier.isna())][[\"fasted\", \"fasted_duration_delta\"]]\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "77",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "78",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "79",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from meta_prn.models import OnSchedule, OffSchedule, OnScheduleDmReferral, OffScheduleDmReferral\n",
+    "df_on_meta = read_frame(OnSchedule.objects.values(\"subject_identifier\", \"onschedule_datetime\").all())\n",
+    "df_off_meta = read_frame(OffSchedule.objects.values(\"subject_identifier\", \"offschedule_datetime\").all())\n",
+    "df_on = read_frame(OnScheduleDmReferral.objects.values(\"subject_identifier\", \"onschedule_datetime\").all())\n",
+    "df_off = read_frame(OffScheduleDmReferral.objects.values(\"subject_identifier\", \"offschedule_datetime\").all())\n"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "80",
+   "metadata": {},
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "81",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def get_meta_duration(s):\n",
+    "    meta_off = get_utcnow() if pd.isna(s[\"meta_offschedule_datetime\"]) else s[\"meta_offschedule_datetime\"]\n",
+    "    return meta_off - s[\"meta_onschedule_datetime\"]  \n",
+    "\n",
+    "def get_dm_duration(s):\n",
+    "    dm_off = get_utcnow() if pd.isna(s[\"dm_offschedule_datetime\"]) else s[\"dm_offschedule_datetime\"]\n",
+    "    return dm_off - s[\"dm_onschedule_datetime\"]  \n",
+    "\n",
+    "df_status = pd.merge(df_on_meta, df_off_meta, on=\"subject_identifier\", how=\"left\") \n",
+    "df_status.columns = [\"subject_identifier\", \"meta_onschedule_datetime\", \"meta_offschedule_datetime\"]\n",
+    "df_status = df_status.merge(df_on, on=\"subject_identifier\", how=\"left\")\n",
+    "df_status = df_status.merge(df_off, on=\"subject_identifier\", how=\"left\")\n",
+    "df_status.columns = [\"subject_identifier\", \"meta_onschedule_datetime\", \"meta_offschedule_datetime\", \"dm_onschedule_datetime\", \"dm_offschedule_datetime\"]\n",
+    "df_status[\"meta_duration\"] = df_status.apply(get_meta_duration, axis=1)\n",
+    "df_status[\"meta_duration_days\"] = df_status[\"meta_duration\"].dt.days\n",
+    "df_status[\"dm_duration\"] = df_status.apply(get_dm_duration, axis=1)\n",
+    "df_status[\"dm_duration_days\"] = df_status[\"dm_duration\"].dt.days\n",
+    "df_status.to_csv(report_folder / \"meta_schedule_status.csv\", index=False)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "82",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "df_on = df_on.merge(df_off, on=\"subject_identifier\", how=\"left\")\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "83",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from edc_utils import get_utcnow\n",
+    "\n",
+    "now = get_utcnow()\n",
+    "df_on[\"duration\"] = now - df_on[\"onschedule_datetime\"] "
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "84",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "df_on[df_on.duration >= pd.Timedelta(days=182)].to_stata\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "85",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.12.4"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}