meta-edc 1.0.6__py3-none-any.whl → 1.1.0__py3-none-any.whl

meta_analytics/notebooks/appointment_planning.ipynb

@@ -0,0 +1,329 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "0",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "%%capture\n",
+    "import os\n",
+    "from pathlib import Path\n",
+    "import pandas as pd\n",
+    "from dj_notebook import activate\n",
+    "import numpy as np\n",
+    "from django_pandas.io import read_frame\n",
+    "\n",
+    "env_file = os.environ[\"META_ENV\"]\n",
+    "reports_folder = Path(os.environ[\"META_REPORTS_FOLDER\"])\n",
+    "analysis_folder = Path(os.environ[\"META_ANALYSIS_FOLDER\"])\n",
+    "pharmacy_folder = Path(os.environ[\"META_PHARMACY_FOLDER\"])\n",
+    "plus = activate(dotenv_file=env_file)\n",
+    "pd.set_option('future.no_silent_downcasting', True)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "1",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import pdfkit\n",
+    "from datetime import date\n",
+    "from edc_pdutils.dataframes import get_subject_visit\n",
+    "from meta_analytics.dataframes import get_glucose_fbg_ogtt_df, get_glucose_fbg_df\n",
+    "from meta_visit_schedule.constants import MONTH15, MONTH18, MONTH21, MONTH27, MONTH30, MONTH33, MONTH39\n",
+    "from meta_analytics.dataframes import GlucoseEndpointsByDate\n",
+    "from scipy.stats import chi2\n",
+    "from great_tables import loc, style, md\n",
+    "from meta_analytics.dataframes import get_eos_df\n",
+    "from meta_analytics.utils import df_as_great_table, df_as_great_table2\n",
+    "from meta_prn.models import LossToFollowup\n",
+    "from edc_visit_schedule.models import SubjectScheduleHistory\n",
+    "from edc_appointment.analytics import get_appointment_df\n",
+    "from edc_appointment.constants import NEW_APPT, CANCELLED_APPT, ONTIME_APPT, MISSED_APPT, SCHEDULED_APPT, COMPLETE_APPT, INCOMPLETE_APPT, IN_PROGRESS_APPT, UNSCHEDULED_APPT\n",
+    "from edc_constants.constants import YES\n",
+    "from meta_consent.models import SubjectConsentV1Ext"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "2",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "html_data = []\n",
+    "cutoff_date = date(2025,3, 31)\n",
+    "end_of_trial_date= date(2026,3, 1)\n",
+    "document_title = f\"<h2>Monitoring Report: {cutoff_date.strftime('%B %Y')}</h2><h5>Data Download: {cutoff_date.strftime('%d %B %Y')}</h5>\"\n",
+    "study_title = 'META3 - Metformin treatment for diabetes prevention in Africa'\n",
+    "pdf_filename = f\"monitoring_report_{cutoff_date.strftime('%Y%m%d')}.pdf\"\n",
+    "\n",
+    "column_headers = {\"appt_datetime\": \"Appointment\", \"year\": \"Year\", \"month\": \"Month\", \"10\": \"Hindu Mandal\", \"20\": \"Amana\", \"30\": \"Temeke\", \"40\": \"Mwananyamala\", \"60\": \"Mnazi Moja\", \"total\": \"Total\"}\n",
+    "\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "3",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "df_visit = get_subject_visit(\"meta_subject.subjectvisit\")\n",
+    "df_visit = df_visit[df_visit.appt_datetime.dt.date<=cutoff_date]\n",
+    "df_appointments = get_appointment_df()\n",
+    "df_appointments[\"site_id\"] = df_appointments.site_id.astype(str)\n",
+    "cls = GlucoseEndpointsByDate()\n",
+    "cls.run()\n",
+    "df_endpoint = cls.endpoint_only_df.copy()\n",
+    "df_glucose = get_glucose_fbg_ogtt_df()\n",
+    "df_glucose_fbg = get_glucose_fbg_df()\n",
+    "df_glucose = pd.concat([df_glucose, df_glucose_fbg])\n",
+    "\n",
+    "enrolled = df_visit.copy()\n",
+    "enrolled[\"site_id\"] = enrolled[\"site_id\"].astype(str)\n",
+    "enrolled_pivot = (\n",
+    "    enrolled\n",
+    "    .query(\"visit_code==1000.0\").groupby([\"site_id\"])\n",
+    "    .size()\n",
+    "    .reset_index()\n",
+    "    .pivot_table(columns=\"site_id\", values=0, observed=True)\n",
+    ")\n",
+    "enrolled_pivot.columns.name=\"\"\n",
+    "enrolled_pivot[\"total\"] = enrolled_pivot[[\"10\", \"20\",\"30\",\"40\",\"60\"]].sum(axis=1)\n",
+    "\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "4",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Table 1f Future scheduled appointments per month\n",
+    "df_appt_pivot = (\n",
+    "    # df_appointments.query(\"appt_datetime<=@cutoff_date and appt_reason==@SCHEDULED_APPT and appt_timing==@ONTIME_APPT and ~appt_status.isin([@NEW_APPT])\")\n",
+    "    df_appointments.query(\"@cutoff_date<=appt_datetime<=@end_of_trial_date and appt_reason==@SCHEDULED_APPT and appt_timing==@ONTIME_APPT and appt_status.isin([@NEW_APPT])\")\n",
+    "    .set_index(\"appt_datetime\")\n",
+    "    .groupby(by=[\"site_id\", pd.Grouper(freq=\"ME\")])\n",
+    "    .size()\n",
+    "    .to_frame()\n",
+    "    .reset_index()\n",
+    "    .rename(columns={0:\"patients\"})\n",
+    "    .pivot(index=\"appt_datetime\", columns=\"site_id\", values=\"patients\")\n",
+    "    .reset_index()\n",
+    "    .fillna(0)\n",
+    ")\n",
+    "\n",
+    "df_appt_pivot.columns.name = None\n",
+    "df_appt_pivot[\"total\"] = df_appt_pivot.iloc[:,1:].sum(axis=1)\n",
+    "df_appt_pivot[\"appt_datetime\"] = df_appt_pivot.appt_datetime.dt.strftime(\"%Y-%m\")\n",
+    "sum_row = df_appt_pivot.select_dtypes(include='float64').sum()\n",
+    "sum_row['appt_datetime'] = 'Total-'\n",
+    "sum_row_df = pd.DataFrame(sum_row).T\n",
+    "df_appt_pivot = pd.concat([df_appt_pivot, sum_row_df], axis=0)\n",
+    "df_appt_pivot[[\"year\", \"month\"]] = df_appt_pivot[\"appt_datetime\"].str.split(\"-\", expand=True)\n",
+    "\n",
+    "df_appt_pivot2 = (\n",
+    "    # df_appointments.query(\"appt_datetime<=@cutoff_date and appt_reason==@SCHEDULED_APPT and appt_timing==@ONTIME_APPT and ~appt_status.isin([@NEW_APPT])\")\n",
+    "    df_appointments.query(\"@cutoff_date<=appt_datetime<=@end_of_trial_date and appt_reason==@SCHEDULED_APPT and appt_timing==@ONTIME_APPT and appt_status.isin([@NEW_APPT])\")\n",
+    "    .set_index(\"visit_code\")\n",
+    "    .groupby(by=[\"site_id\", \"visit_code\"])\n",
+    "    .agg([\"last\"])\n",
+    "    .size()\n",
+    "    .to_frame()\n",
+    "    .reset_index()\n",
+    "    .rename(columns={0:\"patients\"})\n",
+    "    .pivot(index=\"visit_code\", columns=\"site_id\", values=\"patients\")\n",
+    "    .reset_index()\n",
+    "    .fillna(0)\n",
+    ")\n",
+    "\n",
+    "df_appt_pivot2.columns.name = None\n",
+    "df_appt_pivot2[\"total\"] = df_appt_pivot2.iloc[:,1:].sum(axis=1)\n",
+    "df_appt_pivot2[\"visit_code\"] = df_appt_pivot2.visit_code.astype(str)\n",
+    "sum_row = df_appt_pivot2.select_dtypes(include='float64').sum()\n",
+    "sum_row['visit_code'] = 'Total-'\n",
+    "sum_row_df = pd.DataFrame(sum_row).T\n",
+    "df_appt_pivot2 = pd.concat([df_appt_pivot2, sum_row_df], axis=0)\n",
+    "\n",
+    "# df_appt_pivot2[[\"year\", \"month\"]] = df_appt_pivot2[\"appt_datetime\"].str.split(\"-\", expand=True)\n",
+    "\n",
+    "\n",
+    "df_appt_pivot2"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "5",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def get_df_appt(criteria:str):\n",
+    "    df_appt = (\n",
+    "        df_appointments.query(\"@cutoff_date<=appt_datetime<=@end_of_trial_date and appt_reason==@SCHEDULED_APPT and appt_timing==@ONTIME_APPT and appt_status.isin([@NEW_APPT]) and visit_code<2000.0\")\n",
+    "        .groupby([\"site_id\",  \"appt_datetime\"])\n",
+    "        .agg(\"last\")\n",
+    "        .reset_index()\n",
+    "        .query(criteria)\n",
+    "        .set_index(\"appt_datetime\")\n",
+    "        .groupby(by=[\"site_id\", pd.Grouper(freq=\"ME\")])\n",
+    "        .size()\n",
+    "        .to_frame()\n",
+    "        .reset_index()\n",
+    "        .rename(columns={0:\"patients\"})\n",
+    "        .pivot(index=\"appt_datetime\", columns=\"site_id\", values=\"patients\")\n",
+    "        .reset_index()\n",
+    "        .fillna(0)\n",
+    "    )\n",
+    "    df_appt.columns.name = None\n",
+    "    df_appt[\"total\"] = df_appt.iloc[:,1:].sum(axis=1)\n",
+    "    sum_row = df_appt.select_dtypes(include='float64').sum()\n",
+    "    sum_row_df = pd.DataFrame(sum_row).T\n",
+    "    df_appt = pd.concat([df_appt, sum_row_df], axis=0)\n",
+    "    df_appt[\"appt_datetime\"] = df_appt.appt_datetime.dt.strftime(\"%Y-%m\")\n",
+    "    df_appt[[\"year\", \"month\"]] = df_appt[\"appt_datetime\"].str.split(\"-\", expand=True)\n",
+    "    df_appt[\"year\"] = df_appt[\"year\"].fillna(\"Total\")\n",
+    "    return df_appt\n",
+    "\n",
+    "\n",
+    "gt = df_as_great_table2(\n",
+    "    get_df_appt(criteria=\"visit_code.isin([1360.0, 1480.0])\"),\n",
+    "    title=\"Table 1f: Participants who will complete followup on 1360 or 1480 before 2026-03-01\",\n",
+    "    # subtitle=\"Visit codes 1360 or 1480 only\",\n",
+    "    rowname_col=\"month\",\n",
+    "    groupname_col=\"year\",\n",
+    ")\n",
+    "gt = (\n",
+    "    gt\n",
+    "    .cols_label({k:v for k, v in column_headers.items() if k!=\"label\"})\n",
+    "    .cols_align(align=\"center\", columns=[\"appt_datetime\", \"10\", \"20\", \"30\", \"40\", \"60\", \"total\"])\n",
+    "    .cols_align(align=\"left\", columns=[\"month\", \"year\"])\n",
+    "    .fmt_number(columns=[\"10\", \"20\", \"30\", \"40\", \"60\", \"total\"], decimals=0)\n",
+    "    .tab_source_note(source_note=f\"Scheduled appointment date is on or after {cutoff_date.strftime('%d %B %Y')} and before {end_of_trial_date.strftime('%d %B %Y')}.\")\n",
+    "    .tab_style(\n",
+    "        style=[\n",
+    "            style.text(color=\"black\", weight=\"bold\"),\n",
+    "            style.fill(color=\"lightgray\")\n",
+    "        ],\n",
+    "        locations=loc.row_groups()\n",
+    "    )\n",
+    ")\n",
+    "html_data.append(gt.as_raw_html())\n",
+    "gt.show()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "6",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "\n",
+    "gt = df_as_great_table2(\n",
+    "    get_df_appt(criteria=\"~visit_code.isin([1360.0, 1480.0])\"),\n",
+    "    title=\"Table 1f: Participants who will NOT complete followup on 1360 or 1480 before 2026-03-01\",\n",
+    "    rowname_col=\"month\",\n",
+    "    groupname_col=\"year\",\n",
+    ")\n",
+    "gt = (\n",
+    "    gt\n",
+    "    .cols_label({k:v for k, v in column_headers.items() if k!=\"label\"})\n",
+    "    .cols_align(align=\"center\", columns=[\"appt_datetime\", \"10\", \"20\", \"30\", \"40\", \"60\", \"total\"])\n",
+    "    .cols_align(align=\"left\", columns=[\"month\", \"year\"])\n",
+    "    .fmt_number(columns=[\"10\", \"20\", \"30\", \"40\", \"60\", \"total\"], decimals=0)\n",
+    "    .tab_source_note(source_note=f\"Scheduled appointment date is on or after {cutoff_date.strftime('%d %B %Y')} and before {end_of_trial_date.strftime('%d %B %Y')}.\")\n",
+    "    .tab_style(\n",
+    "        style=[\n",
+    "            style.text(color=\"black\", weight=\"bold\"),\n",
+    "            style.fill(color=\"lightgray\")\n",
+    "        ],\n",
+    "        locations=loc.row_groups()\n",
+    "    )\n",
+    ")\n",
+    "html_data.append(gt.as_raw_html())\n",
+    "gt.show()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "7",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# gather raw html\n",
+    "raw_html = [f'<div class=\"page-break\">{s}</div>' for s in html_data]\n",
+    "style_css = \"\"\"\n",
+    "<style>\n",
+    "  .page-break {\n",
+    "    page-break-inside: avoid; /* Always add page break before this element */\n",
+    "  }\n",
+    "  .table-header {\n",
+    "    font-weight: bold;\n",
+    "    font-size: 18px;\n",
+    "    text-align: center;\n",
+    "    border-bottom: None;\n",
+    "  }\n",
+    "</style>\n",
+    "\"\"\"\n",
+    "raw_html = ''.join(raw_html)\n",
+    "raw_html = f'<!DOCTYPE html>\\n<html lang=\"en\">\\n{style_css}\\n<head>\\n<meta charset=\"utf-8\"/>\\n</head>\\n<body>\\n' + document_title + raw_html + '\\n</body>\\n</html>\\n'"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "8",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "pdfkit.from_string(raw_html, str(analysis_folder / pdf_filename),\n",
+    "options={\n",
+    "    'footer-center': 'Page [page] of [topage]',\n",
+    "    'footer-font-size': '8',\n",
+    "    'footer-spacing': '5',\n",
+    "    'encoding': \"UTF-8\",\n",
+    "    'margin-top':'10mm',\n",
+    "    'margin-right':'15mm',\n",
+    "    'margin-bottom':'15mm',\n",
+    "    'margin-left':'15mm',\n",
+    "    'header-center': study_title,\n",
+    "    'header-font-size': '6',\n",
+    "    'header-spacing': '0',\n",
+    "    'disable-javascript': None,\n",
+    "    'no-outline': None,\n",
+    "},\n",
+    "verbose=True)"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 2
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython2",
+   "version": "2.7.6"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

meta_analytics/notebooks/arvs.ipynb

@@ -0,0 +1,103 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "0",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "1",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "%%capture\n",
+    "import os\n",
+    "import pandas as pd\n",
+    "import numpy as np\n",
+    "from dj_notebook import activate\n",
+    "from pathlib import Path\n",
+    "\n",
+    "env_file = os.environ[\"META_ENV\"]\n",
+    "analysis_folder = Path(os.environ[\"META_ANALYSIS_FOLDER\"])\n",
+    "reports_folder = Path(os.environ[\"META_ANALYSIS_FOLDER\"])\n",
+    "plus = activate(dotenv_file=env_file)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "2",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from edc_pdutils.dataframes import get_crf"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "3",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "df_patient_history = get_crf(\"meta_subject.patienthistory\", subject_visit_model=\"meta_subject.subjectvisit\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "4",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "df_patient_history[['subject_identifier', \"current_arv_regimen\"]].groupby(by=[\"current_arv_regimen\"])[\"current_arv_regimen\"].value_counts()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "5",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "df_patient_history[['subject_identifier', \"current_arv_regimen\", \"other_current_arv_regimen\"]][[ \"other_current_arv_regimen\"]].value_counts()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "6",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "df_patient_history[ \"other_current_arv_regimen\"] = df_patient_history[ \"other_current_arv_regimen\"].apply(lambda x : x.split(\"-\")[0])"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 2
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython2",
+   "version": "2.7.6"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

meta_analytics/notebooks/cleaning/consent_v1_ext.ipynb

@@ -0,0 +1,227 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "0",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "1",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "%%capture\n",
+    "import pandas as pd\n",
+    "from django_pandas.io import read_frame\n",
+    "from pathlib import Path\n",
+    "from dj_notebook import activate\n",
+    "\n",
+    "plus = activate(dotenv_file=\"/Users/erikvw/source/edc_source/meta-edc/.env\")\n",
+    "report_folder = Path(\"/Users/erikvw/Documents/ucl/protocols/meta3/reports/\")\n",
+    "# output is suppressed -- normally would spew out all the edc loading messages\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "2",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from edc_registration.models import RegisteredSubject\n",
+    "from edc_appointment.analytics import get_appointment_df\n",
+    "from meta_prn.models import OnSchedule, OffSchedule, OffSchedulePregnancy, OffSchedulePostnatal, OnScheduleDmReferral, \\\n",
+    "    EndOfStudy\n",
+    "\n",
+    "df_onschedule = read_frame(OnSchedule.objects.all(), verbose=True)\n",
+    "df_offschedule = read_frame(OffSchedule.objects.all(), verbose=True)\n",
+    "df_onschedule_preg = read_frame(OffSchedulePregnancy.objects.all(), verbose=True)\n",
+    "df_onschedule_postnatal = read_frame(OffSchedulePostnatal.objects.all(), verbose=True)\n",
+    "df_onschedule_dm = read_frame(OnScheduleDmReferral.objects.all(), verbose=True)\n",
+    "df_eos = read_frame(EndOfStudy.objects.all(), verbose=True)\n",
+    "df_appt = get_appointment_df()\n",
+    "df_rs = read_frame(RegisteredSubject.objects.values(\"subject_identifier\", \"registration_datetime\").all(), verbose=True)\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "3",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "df_appt[df_appt.visit_code > 1360][[\"visit_code\", \"schedule_name\"]].schedule_name.value_counts()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "4",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "df_appt[(df_appt.visit_code > 1360) & (df_appt.schedule_name == \"schedule\")][[\"visit_code\", \"schedule_name\"]].visit_code.value_counts()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "5",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "df_appt[(df_appt.visit_code > 1360) & (df_appt.schedule_name == \"schedule\")][[\"subject_identifier\", \"appt_datetime\", \"visit_code\", \"appt_status\"]].sort_values(\"subject_identifier\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "6",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "df_appt = get_appointment_df()\n",
+    "\n",
+    "df_appt = df_appt[(df_appt.visit_code >= 1360) & (df_appt.schedule_name == \"schedule\")][[\"subject_identifier\", \"appt_datetime\", \"visit_code\", \"appt_status\"]]"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "7",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "df_magreth = pd.read_csv(Path(\"/Users/erikvw/Documents/ucl/protocols/meta3/reports/\") / \"consented_v1_ext_magreth.csv\")\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "8",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "df_magreth"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "9",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "df_main = pd.merge(df_appt[[\"subject_identifier\", \"appt_datetime\", \"visit_code\", \"appt_status\"]], df_magreth, on=\"subject_identifier\", how=\"right\").groupby([\"subject_identifier\", \"agreed\"]).agg({\"visit_code\": \"min\", \"date_reconsented\": \"max\"})"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "10",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "df_main = df_main.merge(df_appt[[\"subject_identifier\", \"visit_code\", \"appt_status\", \"appt_datetime\"]], on=[\"subject_identifier\", \"visit_code\"], how=\"left\")\n",
+    "df_main\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "11",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "df_rs"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "12",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from meta_consent.models import SubjectConsentV1Ext\n",
+    "df_v1ext = read_frame(SubjectConsentV1Ext.objects.values(\"subject_identifier\", \"report_datetime\", \"agrees_to_extension\").all(), verbose=True)\n",
+    "df_v1ext"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "13",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "df_main = df_main.merge(df_rs, on=\"subject_identifier\", how=\"left\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "14",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "df_main = df_main.merge(df_v1ext, on=\"subject_identifier\", how=\"outer\")\n",
+    "df_main.rename(columns={\"report_datetime\": \"v1_ext_datetime\", \"agrees_to_extension\": \"agreed\", \"visit_code\": \"last_visit_code\"}, inplace=True)\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "15",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "df_main = df_main.merge(df_eos[[\"subject_identifier\", \"offstudy_datetime\"]], on=\"subject_identifier\", how=\"left\")\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "16",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "df_main[[\"subject_identifier\",\"registration_datetime\", \"last_visit_code\", \"appt_status\", \"appt_datetime\", \"date_reconsented\", \"v1_ext_datetime\", \"agreed\"]]"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "17",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 2
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython2",
+   "version": "2.7.6"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}