meta-edc 1.0.7__py3-none-any.whl → 1.1.1__py3-none-any.whl

meta_analytics/notebooks/endpoints/meta_endpoints_by_date.ipynb

@@ -0,0 +1,664 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "0",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "%%capture\n",
+    "import os\n",
+    "from pathlib import Path\n",
+    "import pandas as pd\n",
+    "from dj_notebook import activate\n",
+    "env_file = os.environ[\"META_ENV\"]\n",
+    "reports_folder = Path(os.environ[\"META_REPORTS_FOLDER\"])\n",
+    "analysis_folder = Path(os.environ[\"META_ANALYSIS_FOLDER\"])\n",
+    "pharmacy_folder = Path(os.environ[\"META_PHARMACY_FOLDER\"])\n",
+    "plus = activate(dotenv_file=env_file)\n",
+    "pd.set_option('future.no_silent_downcasting', True)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "1",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import numpy as np\n",
+    "import io\n",
+    "import msoffcrypto\n",
+    "import mempass\n",
+    "\n",
+    "from datetime import datetime\n",
+    "from edc_appointment.constants import ONTIME_APPT, NEW_APPT, CANCELLED_APPT, MISSED_APPT\n",
+    "from edc_pdutils.dataframes import get_crf, get_subject_visit\n",
+    "from tabulate import tabulate\n",
+    "from meta_analytics.dataframes import get_glucose_fbg_ogtt_df, get_glucose_fbg_df\n",
+    "from meta_analytics.dataframes import GlucoseEndpointsByDate\n",
+    "from meta_analytics.dataframes import get_glucose_df\n",
+    "from meta_analytics.dataframes import EndpointByDate"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "2",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "cls = GlucoseEndpointsByDate()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "3",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "cls.run()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "4",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "cls.endpoint_only_df.endpoint_label.value_counts()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "5",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "cls.endpoint_only_df"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "6",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "df_glu = get_glucose_df()\n",
+    "df_glu.query(\"subject_identifier=='105-40-0379-1'\")\n",
+    "\n",
+    "ep = EndpointByDate(subject_df=df_glu.query(\"subject_identifier=='105-40-0379-1'\").copy().sort_values(by=[\"visit_code\"]).reset_index(drop=True), fbg_threshhold=7.0, ogtt_threshhold=11.1)\n",
+    "ep.evaluate()\n",
+    "ep.subject_df"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "7",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "df_katie = pd.read_csv(analysis_folder / \"katie_endpoint_subjects.csv\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "8",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "cls.endpoint_only_df[~cls.endpoint_only_df.subject_identifier.isin(df_katie.subject_identifier)]"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "9",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "df_katie[~df_katie.subject_identifier.isin(cls.endpoint_only_df.subject_identifier)]\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "10",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "df_katie"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "11",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "df_glu = get_glucose_df()\n",
+    "df_glu.query(\"subject_identifier=='105-40-0370-0'\")\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "12",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "df_visit = get_subject_visit(model=\"meta_subject.subjectvisit\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "13",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Table 1 Visits completed to date\n",
+    "df_tbl1 = df_visit[(df_visit.visit_code_sequence==0) & (df_visit.appt_timing==ONTIME_APPT) & ~(df_visit.appt_status.isin([NEW_APPT, CANCELLED_APPT]))].groupby(by=[\"visit_code\", \"site_id\"]).size().to_frame().reset_index()\n",
+    "df_tbl1.columns = [\"visit_code\", \"site_id\", \"visits\"]\n",
+    "df1 = df_tbl1.pivot(index=\"visit_code\", columns=\"site_id\", values=\"visits\").reset_index()\n",
+    "df1.columns.name = None\n",
+    "df1.columns = ['visit_code', \"10\", \"20\", \"30\", \"40\", \"60\"]\n",
+    "df1['total'] = df1[['10', '20', '30', '40', '60']].sum(axis=1)\n",
+    "df1.fillna(0, inplace=True)\n",
+    "df_attended = df1.copy()\n",
+    "df_attended"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "14",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Table 2 Visits Missed to Date as % of Visits Attended + Visits Missed\n",
+    "df_tbl12 = df_visit[(df_visit.visit_code_sequence==0) & (df_visit.appt_timing==MISSED_APPT) & ~(df_visit.appt_status.isin([NEW_APPT, CANCELLED_APPT]))].groupby(by=[\"visit_code\", \"site_id\"]).size().to_frame().reset_index()\n",
+    "df_tbl12.columns = [\"visit_code\", \"site_id\", \"visits\"]\n",
+    "df1 = df_tbl12.pivot(index=\"visit_code\", columns=\"site_id\", values=\"visits\").reset_index()\n",
+    "df1.columns.name = None\n",
+    "df1.columns = ['visit_code', \"10\", \"20\", \"30\", \"40\", \"60\"]\n",
+    "df1['total'] = df1[['10', '20', '30', '40', '60']].sum(axis=1)\n",
+    "df1.fillna(0, inplace=True)\n",
+    "df_missed = df1.copy()\n",
+    "\n",
+    "df_attended.set_index([\"visit_code\"], inplace=True)\n",
+    "df_missed.set_index([\"visit_code\"], inplace=True)\n",
+    "\n",
+    "attended_and_missed = df_attended + df_missed\n",
+    "attended_and_missed.fillna(0, inplace=True)\n",
+    "attended_and_missed.reset_index(inplace=True)\n",
+    "attended_and_missed.set_index([\"visit_code\"], inplace=True)\n",
+    "attended_and_missed_perc = df_missed/attended_and_missed\n",
+    "attended_and_missed_perc.fillna(0, inplace=True)\n",
+    "attended_and_missed_perc.reset_index(inplace=True)\n",
+    "attended_and_missed_perc.set_index([\"visit_code\"], inplace=True)\n",
+    "\n",
+    "df_result = df_missed.merge(attended_and_missed_perc, on=[\"visit_code\"], suffixes=(\"\", \"_perc\"))\n",
+    "for col in  [\"10\", \"20\", \"30\", \"40\", \"60\", \"total\"]:\n",
+    "    col_perc = f\"{col}_perc\"\n",
+    "    df_result[col] = df_result.apply(lambda x: f\"{x[col]} ({x[col_perc]*100:.2f})\", axis=1)\n",
+    "df_result.reset_index(inplace=True)\n",
+    "df_result.sort_values(by=[\"visit_code\"], ascending=True, inplace=True)\n",
+    "df_result[[\"visit_code\", \"10\", \"20\", \"30\", \"40\", \"60\", \"total\"]]"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "15",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "\n",
+    "# Table 3: OGTT and FBG at 12-month visit"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "16",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def get_row_df(row_df:pd.DataFrame, label:str)->pd.DataFrame:\n",
+    "    row_df = row_df.groupby(by=[\"site_id\"]).site_id.count().to_frame(name=\"n\")\n",
+    "    row_df[\"label\"] = label\n",
+    "    row_df = row_df.reset_index()\n",
+    "    row_df = row_df.pivot(index=\"label\", values=\"n\", columns=\"site_id\").reset_index()\n",
+    "    row_df.columns.name = \"\"\n",
+    "    all_sites = [10, 20, 30, 40, 60]\n",
+    "    for site in all_sites:\n",
+    "        if site not in row_df.columns:\n",
+    "            row_df[site] = None\n",
+    "    row_df = row_df.reset_index(drop=True)\n",
+    "    return row_df\n",
+    "\n",
+    "\n",
+    "def get_table_df(df_source:pd.DataFrame, visit_code:float)->pd.DataFrame:\n",
+    "    df_month = df_source[df_source.visit_code==visit_code].copy()\n",
+    "    \n",
+    "    row_df = df_month.copy()\n",
+    "    table_df = get_row_df(row_df, \"Total (n)\")\n",
+    "    \n",
+    "    row_df = df_month[(df_month.ogtt_value<7.8) & (df_month.fbg_value<6.1)].copy()\n",
+    "    table_df = pd.concat([table_df, get_row_df(row_df, \"2-hour OGTT <7.8; FBG <6.1\")])\n",
+    "    \n",
+    "    row_df = df_month[(df_month.ogtt_value<7.8) & (df_month.fbg_value>=6.1) & (df_month.fbg_value<7.0)].copy()\n",
+    "    table_df = pd.concat([table_df, get_row_df(row_df, \"2-hour OGTT <7.8; FBG >=6.1 <7.0\")])\n",
+    "    \n",
+    "    row_df = df_month[(df_month.ogtt_value<7.8) & (df_month.fbg_value>=7.0)].copy()\n",
+    "    table_df = pd.concat([table_df, get_row_df(row_df, \"2-hour OGTT <7.8; FBG >=7.0\")])\n",
+    "    \n",
+    "    row_df = df_month[(df_month.ogtt_value>=7.8) & (df_month.ogtt_value<11.1) & (df_month.fbg_value<6.1)].copy()\n",
+    "    table_df = pd.concat([table_df, get_row_df(row_df, \"2-hour OGTT ≥7.8 to <11.1; FBG <6.1\")])\n",
+    "    \n",
+    "    row_df = df_month[(df_month.ogtt_value>=7.8) & (df_month.ogtt_value<11.1) & (df_month.fbg_value>=6.1) & (df_month.fbg_value<7.0)].copy()\n",
+    "    table_df = pd.concat([table_df, get_row_df(row_df, \"2-hour OGTT ≥7.8 to <11.1; FBG >=6.1 <7.0\")])\n",
+    "    \n",
+    "    row_df = df_month[(df_month.ogtt_value>=7.8) & (df_month.ogtt_value<11.1) & (df_month.fbg_value>=7.0)].copy()\n",
+    "    table_df = pd.concat([table_df, get_row_df(row_df, \"2-hour OGTT ≥7.8 to <11.1; FBG >=7.0\")])\n",
+    "    \n",
+    "    row_df = df_month[(df_month.ogtt_value>=11.1) & (df_month.fbg_value<6.1)].copy()\n",
+    "    table_df = pd.concat([table_df, get_row_df(row_df, \"2-hour OGTT ≥11.1; FBG <6.1\")])\n",
+    "    \n",
+    "    row_df = df_month[(df_month.ogtt_value>=11.1) & (df_month.fbg_value>=6.1) & (df_month.fbg_value<7.0)].copy()\n",
+    "    table_df = pd.concat([table_df, get_row_df(row_df, \"2-hour OGTT ≥11.1; FBG >=6.1 <7.0\")])\n",
+    "    \n",
+    "    row_df = df_month[(df_month.ogtt_value>=11.1) & (df_month.fbg_value>=7.0)].copy()\n",
+    "    table_df = pd.concat([table_df, get_row_df(row_df, \"2-hour OGTT ≥11.1; FBG >=7.0\")])\n",
+    "\n",
+    "    row_df = df_month[(df_month.ogtt_value.isna())].copy()\n",
+    "    table_df = pd.concat([table_df, get_row_df(row_df, \"Missing OGTT\")])\n",
+    "    return table_df\n",
+    "\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "17",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def format_table_df(tbl_df, rename_columns:bool|None=None, add_totals:bool|None=None):\n",
+    "    add_totals = True if add_totals is None else add_totals\n",
+    "    tbl_df = tbl_df.fillna(0.0)\n",
+    "    tbl_df[\"total\"] = tbl_df.iloc[:,1:].sum(axis=1)\n",
+    "    tbl_df = tbl_df.reset_index(drop=True)\n",
+    "\n",
+    "    if add_totals:\n",
+    "        df_last = tbl_df[1:].sum().to_frame()\n",
+    "        df_last.loc[\"label\"] = np.nan\n",
+    "        df_last = df_last.reset_index()\n",
+    "        df_last.columns = [\"label\", \"value\"]\n",
+    "        df_last = df_last.pivot_table(columns=\"label\",  values=\"value\").reset_index(drop=True)\n",
+    "        df_last.columns.name = \"\"\n",
+    "        df_last[\"label\"] = \"totals\"\n",
+    "\n",
+    "        tbl_df = pd.concat([tbl_df, df_last])\n",
+    "        tbl_df = tbl_df.reset_index(drop=True)\n",
+    "\n",
+    "    tbl_df.columns = [\"label\", \"10\", \"20\", \"30\", \"40\", \"60\", \"Total\"]\n",
+    "\n",
+    "    for site in [\"10\", \"20\", \"30\", \"40\", \"60\", \"Total\"]:\n",
+    "        tbl_df[f\"{site}_perc\"] = (tbl_df[site]/tbl_df.iloc[0][site]) * 100 if tbl_df.iloc[0][site]>0 else 0\n",
+    "        tbl_df[f\"{site}_perc_str\"] = tbl_df[f\"{site}_perc\"].map('{:.1f}'.format)\n",
+    "\n",
+    "\n",
+    "    for site in [\"10\", \"20\", \"30\", \"40\", \"60\", \"Total\"]:\n",
+    "        tbl_df[f\"{site}_str\"] = tbl_df[[f\"{site}\", f\"{site}_perc_str\"]].apply(lambda x: ' ('.join(x.astype(str)), axis=1)\n",
+    "        tbl_df[f\"{site}_str\"] = tbl_df[f\"{site}_str\"] + \")\"\n",
+    "\n",
+    "    cols = [\"label\", *[f\"{site}_str\" for site in [\"10\", \"20\", \"30\", \"40\", \"60\", \"Total\"]]]\n",
+    "    tbl_df1 = tbl_df[cols]\n",
+    "    tbl_df1.loc[tbl_df.label==\"Total (n)\"] = tbl_df.iloc[0][[\"label\", \"10\", \"20\", \"30\", \"40\", \"60\", \"Total\"]].to_list()\n",
+    "    if rename_columns:\n",
+    "        tbl_df1 = tbl_df1.rename(columns={\"10_str\": \"Hindu Mandal\", \"20_str\": \"Amana\", \"30_str\": \"Temeke\", \"40_str\": \"Mwananyamala\", \"60_str\": \"Mnazi Moja\", \"Total_str\": \"Total\"})\n",
+    "    return tbl_df1\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "18",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "df_glucose = get_glucose_fbg_ogtt_df()\n",
+    "df_glucose_fbg = get_glucose_fbg_df()\n",
+    "df_glucose = pd.concat([df_glucose, df_glucose_fbg])"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "19",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "df_table3 = get_table_df(df_glucose, 1120.0)\n",
+    "df_table3 = format_table_df(df_table3, rename_columns=True)\n",
+    "df_table3"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "20",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "df_table4 = get_table_df(df_glucose, 1240.0)\n",
+    "df_table4 = format_table_df(df_table4, rename_columns=True)\n",
+    "df_table4"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "21",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "df_table5 = get_table_df(df_glucose, 1360.0)\n",
+    "df_table5 = format_table_df(df_table5, rename_columns=True)\n",
+    "df_table5\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "22",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# df_table6 = get_table_df(df_glucose, 1480.0, rename_columns=True)\n",
+    "# df_table6"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "23",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "row_df = df_glucose[df_glucose.ogtt_value>=11.1].copy()\n",
+    "table_df = get_row_df(row_df, \"Total (n)\")\n",
+    "df_table6 = format_table_df(table_df, rename_columns=True)\n",
+    "df_table6[:1]"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "24",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def get_table7_df(df_source:pd.DataFrame, visit_code:float)->pd.DataFrame:\n",
+    "    df_month = df_source[(df_source.visit_code>=visit_code) & (df_source.visit_code<=visit_code + 0.9)].copy()\n",
+    "\n",
+    "    row_df = df_month.copy()\n",
+    "    table_df = get_row_df(row_df, \"Total (n)\")\n",
+    "\n",
+    "    row_df = df_month[(df_month.fbg_value<6.1)].copy()\n",
+    "    table_df = pd.concat([table_df, get_row_df(row_df, \"FBG <6.1\")])\n",
+    "\n",
+    "    row_df = df_month[(df_month.fbg_value>=6.1) & (df_month.fbg_value<7.0)].copy()\n",
+    "    table_df = pd.concat([table_df, get_row_df(row_df, \"FBG >=6.1 <7.0\")])\n",
+    "\n",
+    "    row_df = df_month[(df_month.fbg_value>=7.0)].copy()\n",
+    "    table_df = pd.concat([table_df, get_row_df(row_df, \"FBG >=7.0\")])\n",
+    "    return table_df\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "25",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from meta_visit_schedule.constants import MONTH15, MONTH18, MONTH21, MONTH27, MONTH30, MONTH33, MONTH39\n",
+    "\n",
+    "df_table7 = get_table7_df(df_glucose, 1150.0)\n",
+    "df_table7 = format_table_df(df_table7, rename_columns=True, add_totals=False)\n",
+    "df_table7[\"visit_code\"] = MONTH15\n",
+    "\n",
+    "df_table71 = get_table7_df(df_glucose, 1180.0)\n",
+    "df_table71 = format_table_df(df_table71, rename_columns=True, add_totals=False)\n",
+    "df_table71[\"visit_code\"] = MONTH18\n",
+    "\n",
+    "df_table72 = get_table7_df(df_glucose, 1210.0)\n",
+    "df_table72 = format_table_df(df_table72, rename_columns=True, add_totals=False)\n",
+    "df_table72[\"visit_code\"] = MONTH21\n",
+    "\n",
+    "df_table73 = get_table7_df(df_glucose, 1270.0)\n",
+    "df_table73 = format_table_df(df_table73, rename_columns=True, add_totals=False)\n",
+    "df_table73[\"visit_code\"] = MONTH27\n",
+    "\n",
+    "df_table74 = get_table7_df(df_glucose, 1300.0)\n",
+    "df_table74 = format_table_df(df_table74, rename_columns=True, add_totals=False)\n",
+    "df_table74[\"visit_code\"] = MONTH30\n",
+    "\n",
+    "df_table75 = get_table7_df(df_glucose, 1330.0)\n",
+    "df_table75 = format_table_df(df_table75, rename_columns=True, add_totals=False)\n",
+    "df_table75[\"visit_code\"] = MONTH33\n",
+    "\n",
+    "df_table76 = get_table7_df(df_glucose, 1390.0)\n",
+    "df_table76 = format_table_df(df_table76, rename_columns=True, add_totals=False)\n",
+    "df_table76[\"visit_code\"] = MONTH39\n",
+    "\n",
+    "df_table = pd.concat([df_table7, df_table71, df_table72, df_table73, df_table74, df_table75, df_table76])\n",
+    "df_table[[\"visit_code\", \"label\", \"Hindu Mandal\", \"Amana\", \"Temeke\", \"Mwananyamala\", \"Mnazi Moja\", \"Total\"]]\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "26",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "cls = GlucoseEndpointsByDate()\n",
+    "cls.run()\n",
+    "# cls.endpoint_only_df.endpoint_type.value_counts()\n",
+    "# cls.endpoint_only_df.endpoint_label.value_counts(dropna=False)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "27",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "df = cls.endpoint_only_df.groupby(by=[\"site_id\", \"endpoint_label\"]).size().to_frame().reset_index()\n",
+    "df.columns = [\"site_id\", \"label\", \"endpoints\"]\n",
+    "df = df.pivot_table(index=\"label\", columns=\"site_id\", values=\"endpoints\").reset_index()\n",
+    "df.columns.name = \"\"\n",
+    "df.columns = ['label', \"10\", \"20\", \"30\", \"40\", \"60\"]\n",
+    "df.loc[len(df)] = df[['10', '20', '30', '40', '60']].sum().to_dict()\n",
+    "df.at[len(df)-1, 'label'] = 'Total'\n",
+    "df['Total'] = df[['10', '20', '30', '40', '60']].sum(axis=1)\n",
+    "df.fillna(0, inplace=True)\n",
+    "df\n",
+    "print(tabulate(df[['label', '10', '20', '30', '40', '60', 'Total']], showindex=False, headers=\"keys\", tablefmt=\"simple_grid\"))\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "28",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "df"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "29",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "len(cls.endpoint_df[\"subject_identifier\"].unique())"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "30",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "cls.endpoint_df[(cls.endpoint_df[\"endpoint\"]==1)][\"interval_in_days\"].describe()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "31",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "cls.endpoint_only_df[(cls.endpoint_only_df[\"endpoint\"]==1)][\"days_to_endpoint\"].describe()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "32",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "len(cls.endpoint_df)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "33",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "len(cls.endpoint_only_df)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "34",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "cls.endpoint_only_df[\"subject_identifier\"].nunique()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "35",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "\n",
+    "fname = \"cross_check_end_fbgdate_pivot.csv\"\n",
+    "df_pivot = cls.endpoint_df.sort_values(by=[\"subject_identifier\"]).set_index(\"subject_identifier\").pivot_table(columns=[\"visit_code\"], values=[\"fbg_value\",\"ogtt_value\"], index=[\"subject_identifier\"])\n",
+    "df_pivot.sort_values(('visit_code'), axis=1).sort_values(\"subject_identifier\").to_csv(analysis_folder / fname, sep=\"|\", encoding=\"utf8\", index=True)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "36",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "\n",
+    "fname = f\"glucose-{datetime.now().strftime(\"%Y-%m-%d-%H%M\")}.csv\"\n",
+    "get_crf(\"meta_subject.glucose\", subject_visit_model=\"meta_subject.subjectvisit\", drop_columns=[\"consent_model\"]).to_csv(analysis_folder / fname, sep=\"|\", encoding=\"utf8\", index=False)\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "37",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "fname = f\"glucosefbg-{datetime.now().strftime(\"%Y-%m-%d-%H%M\")}.csv\"\n",
+    "get_crf(\"meta_subject.glucosefbg\", subject_visit_model=\"meta_subject.subjectvisit\", drop_columns=[\"consent_model\"]).to_csv(analysis_folder / fname, sep=\"|\", encoding=\"utf8\", index=False)\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "38",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "\n",
+    "fname = f\"glucose-merged-{datetime.now().strftime(\"%Y-%m-%d-%H%M\")}.csv\"\n",
+    "cls.df.to_csv(analysis_folder / fname, sep=\"|\", encoding=\"utf8\", index=False)\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "39",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "passwd = mempass.mkpassword(2)\n",
+    "fname = \"KBs_latest_enders_26072024.xlsx\"\n",
+    "decrypted_workbook = io.BytesIO()\n",
+    "with open(analysis_folder / fname, 'rb') as file:\n",
+    "    office_file = msoffcrypto.OfficeFile(file)\n",
+    "    office_file.load_key(password=passwd)\n",
+    "    office_file.decrypt(decrypted_workbook)\n",
+    "    \n",
+    "df_katie2 = pd.read_excel(decrypted_workbook, index_col=0)\n",
+    "df_katie2 = df_katie2.copy()\n",
+    "df_katie2 = df_katie2.reset_index()\n",
+    "print(passwd)"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.12.4"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}