meta-edc 0.3.36__py3-none-any.whl → 0.3.38__py3-none-any.whl

meta_analytics/__init__.py

@@ -1,2 +1,2 @@
-from .dataframes import get_glucose_tested_only_df, get_screening_df
+from .dataframes import get_eos_df, get_glucose_tested_only_df, get_screening_df
 from .get_tables import get_tables

meta_analytics/dataframes/__init__.py

@@ -7,6 +7,7 @@ from .constants import (
     endpoint_cases,
     endpoint_columns,
 )
+from .get_eos_df import get_eos_df
 from .glucose_endpoints import EndpointByDate, GlucoseEndpointsByDate
 from .screening import get_glucose_tested_only_df, get_screening_df
 from .utils import (

meta_analytics/dataframes/constants.py

@@ -8,7 +8,7 @@ OGTT_THRESHOLD_MET = "OGTT >= 11.1"
 
 endpoint_columns = [
     "subject_identifier",
-    "site",
+    "site_id",
     "baseline_datetime",
     "visit_datetime",
     "interval_in_days",

meta_analytics/dataframes/get_eos_df.py

@@ -0,0 +1,26 @@
+import pandas as pd
+from edc_pdutils.dataframes import get_eos, get_subject_visit
+
+
+def get_eos_df() -> pd.DataFrame:
+    """
+    df = get_eos_df()
+
+    # look at transfers and last attended visit
+    df[(df.transfer_reason.notna())]
+
+    """
+    df_eos = get_eos("meta_prn.endofstudy")
+    df_visit = get_subject_visit("meta_subject.subjectvisit")
+    df_last_visit = (
+        df_visit.groupby(["subject_identifier", "site"])
+        .agg({"last_visit_code": "max", "last_visit_datetime": "max"})
+        .reset_index()
+    )
+    df_last_visit = df_last_visit.rename(columns={"site": "site_id"})
+
+    df_eos = df_eos.merge(
+        df_last_visit, on="subject_identifier", how="left", suffixes=("", "_y")
+    )
+    df_eos = df_eos.drop(columns=["site_id_y"])
+    return df_eos

meta_analytics/dataframes/glucose_endpoints/glucose_endpoints_by_date.py

@@ -28,26 +28,34 @@ from ..utils import (
 from .endpoint_by_date import EndpointByDate
 
 
-def normalize_date_columns(df: pd.DataFrame, cols: list[str] = None) -> pd.DataFrame:
-    """Normalize date columns by flooring"""
-    for col in cols:
-        if not df[col].empty:
-            df[col] = df[col].dt.floor("d")
-        else:
-            df[col] = pd.NaT
-    return df
-
-
 def calculate_fasting_hrs(df: pd.DataFrame):
     df.loc[(df["fasting"] == NO), "fasting_duration_delta"] = pd.NaT
     if df.empty:
         df["fasting_hrs"] = np.nan
     else:
-        df["fasting_hrs"] = df["fasting_duration_delta"].dt.total_seconds() / 3600
+        df["fasting_hrs"] = df["fasting_duration_delta"].apply(
+            lambda s: np.nan if pd.isna(s) else s.total_seconds() / 3600
+        )
     return df
 
 
 class GlucoseEndpointsByDate:
+    """
+    Usage:
+        cls = GlucoseEndpointsByDate()
+        cls.run()
+
+        # subjects who reached endpoint
+        cls.endpoint_only_df.endpoint_type.value_counts()
+        cls.endpoint_only_df.endpoint_label.value_counts()
+
+        # subjects who reached endpoint with total
+        result_df = cls.endpoint_only_df.endpoint_label.value_counts().to_frame().reset_index()
+        result_df.columns = ["endpoint_label", "total"]
+        result_df.loc[-1] = ["total", result_df.total.sum()]
+        result_df = result_df.reset_index(drop=True)
+        result_df
+    """
 
     fbg_threshhold = 7.0
     ogtt_threshhold = 11.1
@@ -67,7 +75,7 @@ class GlucoseEndpointsByDate:
         "subject_identifier",
         "visit_code",
         "visit_datetime",
-        "site",
+        "site_id",
         "baseline_datetime",
     ]
 
@@ -88,40 +96,75 @@ class GlucoseEndpointsByDate:
         self.endpoint_cases = {k: v for k, v in endpoint_cases.items() if k in self.case_list}
 
         # merge two model DFs
-        self.df = self.glucose_fbg_ogtt_df.merge(
-            self.glucose_fbg_df,
-            on=["subject_visit_id"],
-            how="outer",
-            indicator=True,
-            suffixes=("", "_y"),
+        if self.glucose_fbg_ogtt_df.empty:
+            self.df = self.glucose_fbg_df.copy()
+            self.df[["ogtt_value", "ogtt_units"]] = np.nan
+            self.df[["ogtt_datetime"]] = pd.NaT
+        elif self.glucose_fbg_df.empty:
+            self.df = self.glucose_fbg_ogtt_df.copy()
+        else:
+            self.df = self.glucose_fbg_ogtt_df.merge(
+                self.glucose_fbg_df,
+                on=["subject_visit_id"],
+                how="outer",
+                indicator=True,
+                suffixes=("", "_y"),
+            )
+        # cast as ...
+        for col in ["fasting_hrs", "fbg_value"]:
+            self.df[col] = self.df[col].astype("float64")
+            if f"{col}_y" in self.df.columns:
+                self.df[f"{col}_y"] = self.df[f"{col}_y"].astype("float64")
+        for col in ["fasting", "fbg_units", "source"]:
+            self.df[col] = self.df[col].astype("object")
+            if f"{col}_y" in self.df.columns:
+                self.df[f"{col}_y"] = self.df[f"{col}_y"].astype("object")
+        self.df = self.df.drop(
+            columns=[col for col in self.df.columns if col.endswith("_y") or col == "_merge"]
+        )
+        self.df = self.df.reset_index(drop=True)
+
+        # merge w/ subject_visit
+        subject_visit_df = get_subject_visit(
+            "meta_subject.subjectvisit", subject_identifiers=self.subject_identifiers
+        )
+        self.df = subject_visit_df.merge(
+            self.df, on=["subject_visit_id"], how="left", suffixes=("", "_y")
         )
+        self.df = self.df[[col for col in self.keep_cols]]
         self.df = self.df.reset_index(drop=True)
 
         # pivot right_only cols
-        cols = {
-            "fasting": None,
-            "fasting_hrs": np.nan,
-            "fbg_value": None,
-            "fbg_units": None,
-            "fbg_datetime": None,
-            "source": None,
-            "report_datetime": pd.NaT,
-        }
-        for col, null_value in cols.items():
-            self.df.loc[
-                (self.df["_merge"].isin(["both", "right_only"])) & (self.df[col].isna()), col
-            ] = self.df[f"{col}_y"]
-        # if fbg_datetime still null, use visit datetime to sort order
-        self.df.loc[(self.df["fbg_datetime"].isna()), "fbg_datetime"] = self.df[
-            "visit_datetime"
+        cols = [
+            "fasting",
+            "fasting_hrs",
+            "fbg_value",
+            "fbg_units",
+            "fbg_datetime",
+            "source",
+            "report_datetime",
         ]
-        cols = [col for col in self.df.columns if col.endswith("_y")]
-        cols.append("_merge")
-        self.df = self.df.drop(columns=cols)
+        for col in cols:
+            if f"{col}_y" in self.df.columns and not self.df[f"{col}_y"].isnull().all():
+                self.df.loc[
+                    (self.df["_merge"].isin(["both", "right_only"])) & (self.df[col].isna()),
+                    col,
+                ] = self.df[f"{col}_y"]
+        # if fbg_datetime still null, use visit datetime
+        if self.df["fbg_datetime"].isnull().all():
+            self["fbg_datetime"] = self.df["visit_datetime"]
+        else:
+            self.df.loc[(self.df["fbg_datetime"].isna()), "fbg_datetime"] = self.df[
+                "visit_datetime"
+            ]
+        self.df = self.df.drop(
+            columns=[col for col in self.df.columns if col.endswith("_y") or col == "_merge"]
+        )
         self.df = self.df.reset_index(drop=True)
 
         self.merge_with_consent()
         self.merge_with_eos()
+
         self.add_calculated_days_from_baseline_to_event_columns()
 
         # label rows by type of glu tests (ones with value)
@@ -173,23 +216,15 @@ class GlucoseEndpointsByDate:
             df = get_crf(
                 model="meta_subject.glucosefbg",
                 subject_identifiers=self.subject_identifiers,
-                subject_visit_model="meta_subject.subjectvisit",
-            )
-            # merge w/ subject_visit
-            subject_visit_df = get_subject_visit("meta_subject.subjectvisit")
-            df = subject_visit_df.merge(
-                df, on=["subject_visit_id"], how="left", suffixes=("", "_y")
+                # subject_visit_model="meta_subject.subjectvisit",
             )
             df["source"] = "meta_subject.glucosefbg"
             df.rename(columns={"fbg_fasting": "fasting"}, inplace=True)
             df.loc[(df["fasting"] == "fasting"), "fasting"] = YES
             df.loc[(df["fasting"] == "non_fasting"), "fasting"] = NO
             df = calculate_fasting_hrs(df)
-            df = df[[col for col in self.keep_cols if not col.startswith("ogtt")]]
+            # df = df[[col for col in self.keep_cols if not col.startswith("ogtt")]]
             df = df.reset_index(drop=True)
-            df = normalize_date_columns(
-                df, cols=["fbg_datetime", "report_datetime", "visit_datetime"]
-            )
             self._glucose_fbg_df = df
         return self._glucose_fbg_df
 
@@ -203,27 +238,23 @@ class GlucoseEndpointsByDate:
             df = get_crf(
                 model="meta_subject.glucose",
                 subject_identifiers=self.subject_identifiers,
-                subject_visit_model="meta_subject.subjectvisit",
+                # subject_visit_model="meta_subject.subjectvisit",
             )
             df["source"] = "meta_subject.glucose"
-            # merge w/ subject_visit
-            subject_visit_df = get_subject_visit("meta_subject.subjectvisit")
-            df = subject_visit_df.merge(
-                df, on=["subject_visit_id"], how="left", suffixes=("", "_y")
-            )
             df = calculate_fasting_hrs(df)
-            df = df[self.keep_cols]
+            # df = df[self.keep_cols]
             df = df.reset_index(drop=True)
-            df = normalize_date_columns(
-                df, cols=["fbg_datetime", "ogtt_datetime", "report_datetime", "visit_datetime"]
-            )
             self._glucose_fbg_ogtt_df = df
         return self._glucose_fbg_ogtt_df
 
     def merge_with_consent(self):
         """Merge in consent DF."""
-        df_consent = get_subject_consent("meta_consent.subjectconsent")
-        self.df = pd.merge(self.df, df_consent, on="subject_identifier", how="left")
+        df_consent = get_subject_consent(
+            "meta_consent.subjectconsent", subject_identifiers=self.subject_identifiers
+        )
+        self.df = pd.merge(
+            self.df, df_consent, on="subject_identifier", how="left", suffixes=("", "_y")
+        )
         self.df = self.df.sort_values(by=["subject_identifier", "fbg_datetime"])
         self.df = self.df.reset_index(drop=True)
 
@@ -232,7 +263,7 @@ class GlucoseEndpointsByDate:
 
         Drops patients who were taken off study by late exclusion.
         """
-        df_eos = get_eos("meta_prn.endofstudy")
+        df_eos = get_eos("meta_prn.endofstudy", subject_identifiers=self.subject_identifiers)
         df_eos = df_eos[
             df_eos["offstudy_reason"]
             != (
@@ -240,7 +271,9 @@ class GlucoseEndpointsByDate:
                 "values or raised blood pressure at enrolment"
             )
         ]
-        self.df = pd.merge(self.df, df_eos, on="subject_identifier", how="left")
+        self.df = pd.merge(
+            self.df, df_eos, on="subject_identifier", how="left", suffixes=("", "_y")
+        )
         self.df = self.df.sort_values(by=["subject_identifier", "fbg_datetime"])
         self.df = self.df.reset_index(drop=True)
 
@@ -255,11 +288,11 @@ class GlucoseEndpointsByDate:
         self.df["visit_days"] = (
             self.df["visit_datetime"] - self.df["baseline_datetime"]
         ).dt.days
-        if not self.df["fbg_datetime"].empty:
+        if not self.df["fbg_datetime"].isnull().all():
             self.df["fbg_days"] = (
                 self.df["fbg_datetime"] - self.df["baseline_datetime"]
             ).dt.days
-        if not self.df["ogtt_datetime"].empty:
+        if not self.df["ogtt_datetime"].isnull().all():
             self.df["ogtt_days"] = (
                 self.df["ogtt_datetime"] - self.df["baseline_datetime"]
             ).dt.days
@@ -375,16 +408,6 @@ class GlucoseEndpointsByDate:
         subject_df = subject_df[endpoint_columns]
         subject_df = subject_df.sort_values(["subject_identifier", "fbg_datetime"])
         subject_df = subject_df.reset_index(drop=True)
-        # subject_df = subject_df.merge(
-        #     self.visit_codes_df,
-        #     on="visit_code",
-        #     how="outer",
-        #     indicator=False,
-        #     suffixes=["", "2"],
-        # )
-        # subject_df["subject_identifier"] = subject_identifier
-        # subject_df = subject_df.drop(columns=["count"])
-        # subject_df = subject_df.reset_index(drop=True)
         return subject_df
 
     def check_endpoint_by_fbg_for_subject(
@@ -476,24 +499,24 @@ class GlucoseEndpointsByDate:
             model_cls.objects.all().delete()
         created = 0
         if not df.empty:
+            df["fbg_datetime"] = df["fbg_datetime"].dt.tz_localize("UTC")
+            df["baseline_datetime"] = df["baseline_datetime"].dt.tz_localize("UTC")
             data = [
                 model_cls(
                     subject_identifier=row["subject_identifier"],
-                    site_id=row["site"],
-                    baseline_datetime=(
+                    site_id=row["site_id"],
+                    baseline_date=(
                         None if pd.isna(row["baseline_datetime"]) else row["baseline_datetime"]
                     ),
                     visit_code=None if pd.isna(row["visit_code"]) else row["visit_code"],
                     fbg_value=(None if pd.isna(row["fbg_value"]) else row["fbg_value"]),
                     ogtt_value=None if pd.isna(row["ogtt_value"]) else row["ogtt_value"],
-                    fbg_datetime=(
-                        None if pd.isna(row["fbg_datetime"]) else row["fbg_datetime"]
-                    ),
+                    fbg_date=(None if pd.isna(row["fbg_datetime"]) else row["fbg_datetime"]),
                     fasting=(None if pd.isna(row["fasting"]) else row["fasting"]),
                     endpoint_label=(
                         None if pd.isna(row["endpoint_label"]) else row["endpoint_label"]
                     ),
-                    offstudy_datetime=(
+                    offstudy_date=(
                         None if pd.isna(row["offstudy_datetime"]) else row["offstudy_datetime"]
                     ),
                     offstudy_reason=(

meta_analytics/dataframes/screening/get_screening_df.py

@@ -9,13 +9,33 @@ from meta_subject.models import PhysicalExam, SubjectVisit
 def get_screening_df(df: pd.DataFrame | None = None) -> pd.DataFrame:
     df = pd.DataFrame() if not hasattr(df, "empty") else df
     if df.empty:
-        qs_screening = SubjectScreening.objects.all()
-        df = read_frame(qs_screening)
-        df = df.drop(df[df["hiv_pos"] == "No"].index)
-        # df = df.drop(df[df["meta_phase_two"] == "Yes"].index)
+        cols = [
+            f.name
+            for f in SubjectScreening._meta.get_fields()
+            if f.name
+            not in [
+                "contact_number",
+                "initials",
+                "hospital_identifier",
+                "modified",
+                "user_created",
+                "user_modified",
+                "hostname_created",
+                "hostname_modified",
+                "device_created",
+                "device_modified",
+                "locale_created",
+                "locale_modified",
+                "slug",
+            ]
+        ]
+        qs_screening = SubjectScreening.objects.values(*cols).all()
+        df = read_frame(qs_screening, verbose=False)
+        df = df.drop(df[df["hiv_pos"] == "No"].index)  # removes 2 rows
+        df = df.reset_index(drop=True)
 
     # convert all to float
-    cols = [
+    num_cols = [
         "age_in_years",
         "calculated_bmi_value",
         "converted_fbg2_value",
@@ -31,7 +51,23 @@ def get_screening_df(df: pd.DataFrame | None = None) -> pd.DataFrame:
         "sys_blood_pressure_avg",
         "waist_circumference",
     ]
-    df[cols] = df[cols].apply(pd.to_numeric)
+    df[num_cols] = df[num_cols].apply(pd.to_numeric)
+
+    df["reasons_ineligible_part_one"] = df["reasons_ineligible_part_one"].apply(
+        lambda x: None if x == "" else x
+    )
+    df["reasons_ineligible_part_two"] = df["reasons_ineligible_part_two"].apply(
+        lambda x: None if x == "" else x
+    )
+    df["reasons_ineligible_part_two"] = df["reasons_ineligible_part_two"].str.replace(
+        "Has Dm", "Diabetes"
+    )
+    df["reasons_ineligible_part_two"] = df["reasons_ineligible_part_two"].str.replace(
+        "On Dm Medication", "taking anti-diabetic medications"
+    )
+    df["reasons_ineligible_part_three"] = df["reasons_ineligible_part_three"].apply(
+        lambda x: None if x == "" else x
+    )
 
     # condition to include any glucose test
 

meta_analytics/get_tables.py

@@ -1,6 +1,3 @@
-from datetime import datetime
-from pathlib import Path
-
 import pandas as pd
 from edc_analytics.custom_tables import (
     AgeTable,
@@ -15,31 +12,10 @@ from edc_analytics.custom_tables import (
     OgttTable,
     WaistCircumferenceTable,
 )
+from edc_analytics.data import Data
 
-from meta_analytics.dataframes import get_glucose_tested_only_df, get_screening_df
-from meta_analytics.tables import EligibleP12Table, HasDmTable
-
-
-class Data:
-    def __init__(self, label: str, table_df: pd.DataFrame, data_df: pd.DataFrame):
-        self.label = label
-        self.table_df = table_df
-        self.data_df = data_df
-
-    def __repr__(self):
-        return f"{self.label} obs={len(self.data_df)}"
-
-    def to_csv(
-        self, folder: str | None = None, filename: str | None = None, cols: int | None = None
-    ):
-        folder = folder or "/Users/erikvw/Documents/ucl/protocols/meta3/reports/"
-        cols = cols or 5
-        datestamp = datetime.now().strftime("%Y%m%d%H%M")
-        filename = filename or f"meta3_table_{self.label}_{datestamp}.csv"
-        path = Path(folder) / filename
-        self.table_df.iloc[:, :cols].to_csv(
-            path_or_buf=path, encoding="utf-8", index=0, sep="|"
-        )
+from .dataframes import get_glucose_tested_only_df, get_screening_df
+from .tables import EligibleP12Table, HasDmTable
 
 
 def get_tables() -> dict[str, Data]:
@@ -101,5 +77,5 @@ def get_tables() -> dict[str, Data]:
         ]:
             tbl = tbl_cls(main_df=df)
             dfs.append(tbl.table_df)
-        results.update({key: Data(key, pd.concat(list(dfs)), df)})
+        results.update({key: Data(key, pd.concat(list(dfs)), df, "meta3")})
     return results

meta_edc/settings/debug.py

@@ -7,8 +7,8 @@ from .defaults import *  # noqa
 print(f"Settings file {__file__}")
 
 # TZ Sites:
-# SITE_ID = SiteID(default=20)  # Amana
-SITE_ID = SiteID(default=10)  # Hindu Mandal
+SITE_ID = SiteID(default=20)  # Amana
+# SITE_ID = SiteID(default=10)  # Hindu Mandal
 # SITE_ID = SiteID(default=40)  # Mwananyamala
 # SITE_ID = SiteID(default=50)  # Mbagala
 # SITE_ID = SiteID(default=60)  # Mnazi-Moja