meta-edc 1.0.7__py3-none-any.whl → 1.1.1__py3-none-any.whl

meta_analytics/notebooks/cleaning/consent_v1_ext.ipynb

@@ -0,0 +1,227 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "0",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "1",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "%%capture\n",
+    "import pandas as pd\n",
+    "from django_pandas.io import read_frame\n",
+    "from pathlib import Path\n",
+    "from dj_notebook import activate\n",
+    "\n",
+    "plus = activate(dotenv_file=\"/Users/erikvw/source/edc_source/meta-edc/.env\")\n",
+    "report_folder = Path(\"/Users/erikvw/Documents/ucl/protocols/meta3/reports/\")\n",
+    "# output is suppressed -- normally would spew out all the edc loading messages\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "2",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from edc_registration.models import RegisteredSubject\n",
+    "from edc_appointment.analytics import get_appointment_df\n",
+    "from meta_prn.models import OnSchedule, OffSchedule, OffSchedulePregnancy, OffSchedulePostnatal, OnScheduleDmReferral, \\\n",
+    "    EndOfStudy\n",
+    "\n",
+    "df_onschedule = read_frame(OnSchedule.objects.all(), verbose=True)\n",
+    "df_offschedule = read_frame(OffSchedule.objects.all(), verbose=True)\n",
+    "df_onschedule_preg = read_frame(OffSchedulePregnancy.objects.all(), verbose=True)\n",
+    "df_onschedule_postnatal = read_frame(OffSchedulePostnatal.objects.all(), verbose=True)\n",
+    "df_onschedule_dm = read_frame(OnScheduleDmReferral.objects.all(), verbose=True)\n",
+    "df_eos = read_frame(EndOfStudy.objects.all(), verbose=True)\n",
+    "df_appt = get_appointment_df()\n",
+    "df_rs = read_frame(RegisteredSubject.objects.values(\"subject_identifier\", \"registration_datetime\").all(), verbose=True)\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "3",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "df_appt[df_appt.visit_code > 1360][[\"visit_code\", \"schedule_name\"]].schedule_name.value_counts()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "4",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "df_appt[(df_appt.visit_code > 1360) & (df_appt.schedule_name == \"schedule\")][[\"visit_code\", \"schedule_name\"]].visit_code.value_counts()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "5",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "df_appt[(df_appt.visit_code > 1360) & (df_appt.schedule_name == \"schedule\")][[\"subject_identifier\", \"appt_datetime\", \"visit_code\", \"appt_status\"]].sort_values(\"subject_identifier\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "6",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "df_appt = get_appointment_df()\n",
+    "\n",
+    "df_appt = df_appt[(df_appt.visit_code >= 1360) & (df_appt.schedule_name == \"schedule\")][[\"subject_identifier\", \"appt_datetime\", \"visit_code\", \"appt_status\"]]"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "7",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "df_magreth = pd.read_csv(Path(\"/Users/erikvw/Documents/ucl/protocols/meta3/reports/\") / \"consented_v1_ext_magreth.csv\")\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "8",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "df_magreth"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "9",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "df_main = pd.merge(df_appt[[\"subject_identifier\", \"appt_datetime\", \"visit_code\", \"appt_status\"]], df_magreth, on=\"subject_identifier\", how=\"right\").groupby([\"subject_identifier\", \"agreed\"]).agg({\"visit_code\": \"min\", \"date_reconsented\": \"max\"})"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "10",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "df_main = df_main.merge(df_appt[[\"subject_identifier\", \"visit_code\", \"appt_status\", \"appt_datetime\"]], on=[\"subject_identifier\", \"visit_code\"], how=\"left\")\n",
+    "df_main\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "11",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "df_rs"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "12",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from meta_consent.models import SubjectConsentV1Ext\n",
+    "df_v1ext = read_frame(SubjectConsentV1Ext.objects.values(\"subject_identifier\", \"report_datetime\", \"agrees_to_extension\").all(), verbose=True)\n",
+    "df_v1ext"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "13",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "df_main = df_main.merge(df_rs, on=\"subject_identifier\", how=\"left\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "14",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "df_main = df_main.merge(df_v1ext, on=\"subject_identifier\", how=\"outer\")\n",
+    "df_main.rename(columns={\"report_datetime\": \"v1_ext_datetime\", \"agrees_to_extension\": \"agreed\", \"visit_code\": \"last_visit_code\"}, inplace=True)\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "15",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "df_main = df_main.merge(df_eos[[\"subject_identifier\", \"offstudy_datetime\"]], on=\"subject_identifier\", how=\"left\")\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "16",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "df_main[[\"subject_identifier\",\"registration_datetime\", \"last_visit_code\", \"appt_status\", \"appt_datetime\", \"date_reconsented\", \"v1_ext_datetime\", \"agreed\"]]"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "17",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 2
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython2",
+   "version": "2.7.6"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

meta_analytics/notebooks/cleaning/offschedule_eos.ipynb

@@ -0,0 +1,353 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "0",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "%%capture\n",
+    "import pandas as pd\n",
+    "from django_pandas.io import read_frame\n",
+    "from pathlib import Path\n",
+    "from dj_notebook import activate\n",
+    "\n",
+    "plus = activate(dotenv_file=\"/Users/erikvw/source/edc_source/meta-edc/.env\")\n",
+    "report_folder = Path(\"/Users/erikvw/Documents/ucl/protocols/meta3/reports/\")\n",
+    "# output is suppressed -- normally would spew out all the edc loading messages\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "1",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from meta_prn.models import OnSchedule, OffSchedule, EndOfStudy, OffSchedulePostnatal, OffSchedulePregnancy, \\\n",
+    "    OnScheduleDmReferral, OffStudyMedication\n",
+    "from meta_lists.models import OffstudyReasons\n",
+    "from meta_analytics.dataframes import GlucoseEndpointsByDate"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "2",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "df_onschedule = read_frame(OnSchedule.objects.all(), verbose=True)\n",
+    "df_offschedule = read_frame(OffSchedule.objects.all(), verbose=True)\n",
+    "df_onschedule_preg = read_frame(OffSchedulePregnancy.objects.all(), verbose=True)\n",
+    "df_onschedule_postnatal = read_frame(OffSchedulePostnatal.objects.all(), verbose=True)\n",
+    "df_onschedule_dm = read_frame(OnScheduleDmReferral.objects.all(), verbose=True)\n",
+    "df_offstudy_medication = read_frame(OffStudyMedication.objects.all(), verbose=True)\n",
+    "\n",
+    "df_eos = read_frame(EndOfStudy.objects.values(\"id\", \"subject_identifier\",  \"offstudy_datetime\",\"offstudy_reason\", \"other_offstudy_reason\", \"action_item__action_type__name\", \"parent_action_item__action_type__name\").all(), verbose=False)\n",
+    "df_eos.rename(columns={\"offstudy_reason\": \"offstudy_reason_id\", \"action_item__action_type__name\": \"action_item_name\", \"parent_action_item__action_type__name\": \"parent_action_item_name\"}, inplace=True)\n",
+    "\n",
+    "df_offstudy_reasons = read_frame(OffstudyReasons.objects.all(), verbose=False)\n",
+    "df_offstudy_reasons.rename(columns={\"id\": \"offstudy_reason_id\", \"name\": \"offstudy_reason\", \"display_name\": \"offstudy_reason_display\"}, inplace=True)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "3",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# merge df_eos with offstudyreasons\n",
+    "df_eos = df_eos.merge(df_offstudy_reasons[[\"offstudy_reason_id\", \"offstudy_reason\", \"offstudy_reason_display\"]], on=\"offstudy_reason_id\")\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "4",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# summarize offstudy\n",
+    "df_eos_summary = df_eos[[\"offstudy_reason_id\", \"offstudy_reason\"]].value_counts().to_frame().reset_index().sort_values(\"offstudy_reason_id\", ascending=True).reset_index(drop=True)\n",
+    "df_eos_summary.rename(columns={\"count\": \"subtotal\"}, inplace=True)\n",
+    "df_eos_summary[\"cumfreq\"] = df_eos_summary.subtotal.cumsum()\n",
+    "df_eos_summary\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "5",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "markdown",
+   "id": "6",
+   "metadata": {},
+   "source": [
+    "* Need to validate that all subjects off study went off study medication  first.\n",
+    "* Action items are linked to this flow and block the EndOfStudy.save() if  EndOfStudy was submitted before this workflow was implemented.\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "7",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# df_eos[df_eos.offstudy_reason_id==11][[\"subject_identifier\", \"offstudy_reason_id\", \"other_offstudy_reason\"]]\n",
+    "pd.merge(df_eos, df_offstudy_medication[[\"subject_identifier\", \"stop_date\"]], on=\"subject_identifier\", how=\"left\")\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "8",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "9",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "df_onschedule_preg"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "10",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "df_dm = df_onschedule_dm[[\"subject_identifier\"]].copy()\n",
+    "df_dm.reset_index(drop=True, inplace=True)\n",
+    "df_dm[\"dm\"] =1\n",
+    "\n",
+    "df_preg = df_onschedule_preg[[\"subject_identifier\"]].copy()\n",
+    "df_preg[\"preg\"] = 1\n",
+    "df_preg.reset_index(drop=True, inplace=True)\n",
+    "#\n",
+    "df_eos = df_eos[[\"subject_identifier\", \"offstudy_datetime\", \"offstudy_reason\", \"other_offstudy_reason\"]].copy()\n",
+    "df_eos[\"eos\"] = 1\n",
+    "df_eos.reset_index(drop=True, inplace=True)\n",
+    "\n",
+    "df = df_onschedule[[\"subject_identifier\", \"onschedule_datetime\"]].copy()\n",
+    "df[\"onschedule\"] = 1\n",
+    "df.reset_index(drop=True, inplace=True)\n",
+    "\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "11",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# df[df.offstudy_reason.isna()]"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "12",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "df = df.merge(df_dm, on=[\"subject_identifier\"], how=\"left\", suffixes=(\"\", \"_y\"))\n",
+    "df = df.merge(df_preg, on=[\"subject_identifier\"], how=\"left\", suffixes=(\"\", \"_y\"))\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "13",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "df_offschedule = df_offschedule[[\"subject_identifier\", \"offschedule_datetime\"]].copy()\n",
+    "df_offschedule[\"offschedule\"] = 1\n",
+    "df_offschedule.reset_index(drop=True, inplace=True)\n",
+    "df = df.merge(df_offschedule, on=[\"subject_identifier\"], how=\"left\", suffixes=(\"\", \"_y\"))\n",
+    "df = df.merge(df_eos, on=[\"subject_identifier\"], how=\"left\", suffixes=(\"\", \"_y\"))"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "14",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "cls = GlucoseEndpointsByDate()\n",
+    "cls.run()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "15",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "df_endpoint = cls.df\n",
+    "df_endpoint = df_endpoint[(df_endpoint.endpoint==1)][[\"subject_identifier\", \"endpoint\"]]\n",
+    "df = df.merge(df_endpoint, on=[\"subject_identifier\"], how=\"left\", suffixes=(\"\", \"_y\"))"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "16",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "df.fillna({\"dm\": 0, \"preg\":0, \"offschedule\":0, \"eos\": 0, \"endpoint\": 0}, inplace=True)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "17",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "df.dm.value_counts()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "18",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "df.preg.value_counts()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "19",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "df_export = df[(df.offschedule_datetime.notna()) & (df.eos==0) & ((df.dm==0) & (df.preg==0))][[\"subject_identifier\", \"onschedule_datetime\", \"offschedule_datetime\", \"onschedule\", \"offschedule\", \"dm\", \"preg\", \"endpoint\", \"eos\"]].copy()\n",
+    "df_export.reset_index(drop=True, inplace=True)\n",
+    "df_export[\"offschedule_datetime\"] = df_export.offschedule_datetime.dt.tz_localize(None)\n",
+    "df_export[\"offschedule_datetime\"] = df_export.offschedule_datetime.dt.normalize()\n",
+    "df_export[\"onschedule_datetime\"] = df_export.onschedule_datetime.dt.tz_localize(None)\n",
+    "df_export[\"onschedule_datetime\"] = df_export.onschedule_datetime.dt.normalize()\n",
+    "df_export.sort_values(by=[\"offschedule_datetime\"], inplace=True)\n",
+    "df_export.to_csv(report_folder / \"offschedule_eos.csv\", index=False)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "20",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "df"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "21",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "df[df.endpoint==1]"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "22",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# DM Referrals onstudy/off study\n",
+    "df.groupby(by=[\"dm\", \"eos\"]).size().reset_index(name=\"counts\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "23",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "df.groupby(by=[\"preg\", \"eos\"]).size().reset_index(name=\"counts\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "24",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "df_eos[(df_eos.other_offstudy_reason != \"\")][[\"subject_identifier\", \"offstudy_reason\", \"other_offstudy_reason\"]]"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "25",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "df_eos.offstudy_reason.value_counts()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "26",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "df_eos = read_frame(EndOfStudy.objects.all(), verbose=False)\n",
+    "df_offstudy_reasons = read_frame(OffstudyReasons.objects.all(), verbose=False)\n",
+    "df_offstudy_reasons.rename(columns={\"id\": \"offstudy_reason_id\", \"name\": \"offstudy_reason\"}, inplace=True)\n",
+    "df_eos = df_eos.merge(df_offstudy_reasons[\"offstudy_reason_id\", \"offstudy_reason\"], on=\"offstudy_reason_id\")\n",
+    "df_eos.offstudy_reason.value_counts()"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 2
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython2",
+   "version": "2.7.6"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}