meta-edc 0.3.18__tar.gz → 0.3.19__tar.gz

{meta_edc-0.3.18 → meta_edc-0.3.19}/.github/workflows/build.yml

@@ -5,15 +5,20 @@ on: [push, pull_request]
 
 jobs:
   build:
-    name: ${{ matrix.codebase }} / Dj ${{ matrix.django-version }} / Py ${{ matrix.python-version }}
+    name: >
+      ${{ matrix.codebase }}-
+      Py${{ matrix.python-version }}-
+      Dj${{ matrix.django-version }}-
+      ${{ matrix.database-engine }}
     runs-on: ubuntu-latest
 
     strategy:
       fail-fast: false
       matrix:
         codebase: ['Prod', 'Dev']
-        django-version: ['4.2', '5.0', 'dev']
+        django-version: ['4.2', '5.1', 'dev']
         python-version: ['3.12']
+        database-engine: ["mysql", "postgres"]
         exclude:
           - codebase: 'Prod'
             django-version: 'dev'
@@ -27,7 +32,17 @@ jobs:
         ports:
           - 3306:3306
         options: --health-cmd="mysqladmin ping" --health-interval=10s --health-timeout=5s --health-retries=3
-
+      postgres:
+        image: postgres:latest
+        env:
+          POSTGRES_PASSWORD: postgres
+        options: >-
+          --health-cmd pg_isready
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+        ports:
+          - 5432:5432
     steps:
       - name: Install pycups and words dependency
         run: |

{meta_edc-0.3.18 → meta_edc-0.3.19}/.pre-commit-config.yaml

@@ -13,7 +13,7 @@ repos:
     rev: 24.4.2
     hooks:
       - id: black
-        language_version: python3.11
+        language_version: python3.12
 
   - repo: https://github.com/pycqa/flake8
     rev: 7.0.0

{meta_edc-0.3.18 → meta_edc-0.3.19}/CHANGES

@@ -1,6 +1,15 @@
 Changes
 -------
 
+0.3.19
+------
+- update for changes in edc-qareports, use django_db_views for
+  unmanaged SQL views
+- add Endpoints (DM) report
+- fix error in screening dataframe prep column waist circumference
+- bump edc==0.5.99
+- Django 5.1
+
 0.3.18
 ------
 - add edc-qareports as a dependency and use in meta_reports

{meta_edc-0.3.18 → meta_edc-0.3.19}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: meta-edc
-Version: 0.3.18
+Version: 0.3.19
 Summary: META Trial EDC (http://www.isrctn.com/ISRCTN76157257)
 Home-page: https://github.com/meta-trial/meta-edc
 Author: Erik van Widenfelt
@@ -19,10 +19,11 @@ Requires-Python: >=3.12
 Description-Content-Type: text/x-rst
 License-File: LICENSE
 License-File: AUTHORS
-Requires-Dist: edc==0.5.93
+Requires-Dist: edc==0.5.99
 Requires-Dist: edc-microscopy
 Requires-Dist: beautifulsoup4
-Requires-Dist: Django==4.2.11
+Requires-Dist: edc-analytics
+Requires-Dist: Django==5.1
 
 |pypi| |actions| |codecov| |downloads|
 

{meta_edc-0.3.18 → meta_edc-0.3.19}/meta_ae/migrations/0017_auto_20221130_2257.py

@@ -1,4 +1,5 @@
 # Generated by Django 4.1.2 on 2022-11-30 19:57
+from django.core.exceptions import ObjectDoesNotExist
 from django.db import migrations
 from edc_constants.constants import NOT_APPLICABLE
 from tqdm import tqdm
@@ -7,13 +8,17 @@ from tqdm import tqdm
 def update_investigator_ae_classification(apps, schema_editor):
     ae_tmg_cls = apps.get_model("meta_ae", "aetmg")
     ae_classification_cls = apps.get_model("edc_adverse_event", "aeclassification")
-    ae_classification = ae_classification_cls.objects.get(name=NOT_APPLICABLE)
-    total = ae_tmg_cls.objects.filter(investigator_ae_classification__isnull=True).count()
-    for obj in tqdm(
-        ae_tmg_cls.objects.filter(investigator_ae_classification__isnull=True), total=total
-    ):
-        obj.investigator_ae_classification = ae_classification
-        obj.save_base(update_fields=["investigator_ae_classification"])
+    try:
+        ae_classification = ae_classification_cls.objects.get(name=NOT_APPLICABLE)
+    except ObjectDoesNotExist:
+        pass
+    else:
+        total = ae_tmg_cls.objects.filter(investigator_ae_classification__isnull=True).count()
+        for obj in tqdm(
+            ae_tmg_cls.objects.filter(investigator_ae_classification__isnull=True), total=total
+        ):
+            obj.investigator_ae_classification = ae_classification
+            obj.save_base(update_fields=["investigator_ae_classification"])
 
 
 class Migration(migrations.Migration):

meta_edc-0.3.19/meta_analytics/README.rst

@@ -0,0 +1,17 @@
+
+
+
+import pandas as pd
+import numpy as np
+import matplotlib.pyplot as plt
+import seaborn as sns
+from django_pandas.io import read_frame
+from dj_notebook import activate
+
+from meta_screening.models import SubjectScreening
+
+plus = activate(dotenv_file="/Users/erikvw/source/edc_source/meta-edc/.env")
+
+qs = SubjectScreening.objects.all()
+
+df = read_frame(qs)

meta_edc-0.3.19/meta_analytics/__init__.py

@@ -0,0 +1,2 @@
+from .dataframes import get_glucose_tested_only_df, get_screening_df
+from .get_tables import get_tables

meta_edc-0.3.19/meta_analytics/dataframes/__init__.py

@@ -0,0 +1,2 @@
+from .glucose_endpoints import EndpointByDate, GlucoseEndpointsByDate
+from .screening import get_glucose_tested_only_df, get_screening_df

meta_edc-0.3.19/meta_analytics/dataframes/enrolled/__init__.py

@@ -0,0 +1 @@
+from .get_glucose_df import get_glucose_df

meta_edc-0.3.19/meta_analytics/dataframes/enrolled/get_glucose_df.py

@@ -0,0 +1,122 @@
+import numpy as np
+import pandas as pd
+from django_pandas.io import read_frame
+from edc_pdutils.dataframes import get_eos, get_subject_consent, get_subject_visit
+
+from meta_subject.models import Glucose, GlucoseFbg
+
+
+def get_glucose_df() -> pd.DataFrame:
+    qs_glucose_fbg = GlucoseFbg.objects.all()
+    df_glucose_fbg = read_frame(qs_glucose_fbg)
+    df_glucose_fbg.rename(
+        columns={"fbg_fasting": "fasting", "subject_visit": "subject_visit_id"},
+        inplace=True,
+    )
+
+    df_glucose_fbg.loc[(df_glucose_fbg["fasting"] == "fasting"), "fasting"] = "Yes"
+    df_glucose_fbg.loc[(df_glucose_fbg["fasting"] == "non_fasting"), "fasting"] = "No"
+    df_glucose_fbg["fasting_hrs"] = np.nan
+    df_glucose_fbg["fasting_hrs"] = df_glucose_fbg["fasting_duration_delta"].apply(
+        lambda x: x.total_seconds() / 3600
+    )
+    df_glucose_fbg["fasting_hrs"] = df_glucose_fbg["fasting_hrs"].apply(
+        lambda x: 8.05 if not x else x
+    )
+    # df_glucose_fbg = df_glucose_fbg.loc[df_glucose_fbg["fasting_hrs"] >= 8.0]
+    # df_glucose_fbg.reset_index(drop=True, inplace=True)
+    df_glucose_fbg.loc[
+        :,
+        ["ogtt_value", "ogtt_units", "ogtt_datetime"],
+    ] = [np.nan, None, pd.NaT]
+    df_glucose_fbg["source"] = "meta_subject.glucosefbg"
+
+    qs_glucose = Glucose.objects.all()
+    df_glucose = read_frame(qs_glucose)
+    df_glucose.rename(columns={"subject_visit": "subject_visit_id"}, inplace=True)
+    df_glucose.loc[(df_glucose["fasting"] == "fasting"), "fasting"] = "Yes"
+    df_glucose.loc[(df_glucose["fasting"] == "non_fasting"), "fasting"] = "No"
+    df_glucose["fasting_hrs"] = np.nan
+    df_glucose["fasting_hrs"] = df_glucose[df_glucose["fasting"] == "Yes"][
+        "fasting_duration_delta"
+    ].apply(lambda x: x.total_seconds() / 3600)
+    df_glucose["fasting_hrs"] = df_glucose["fasting_hrs"].apply(lambda x: 8.05 if not x else x)
+    # df_glucose = df_glucose.loc[df_glucose["fasting_hrs"] >= 8.0]
+    # df_glucose.reset_index(drop=True, inplace=True)
+    df_glucose["source"] = "meta_subject.glucose"
+
+    keep_cols = [
+        "subject_visit_id",
+        "fasting",
+        "fasting_hrs",
+        "fbg_value",
+        "fbg_units",
+        "fbg_datetime",
+        "ogtt_value",
+        "ogtt_units",
+        "ogtt_datetime",
+        "source",
+        "revision",
+        "report_datetime",
+    ]
+    df_glucose = df_glucose[keep_cols]
+    df_glucose_fbg = df_glucose_fbg[keep_cols]
+    # df = pd.concat([df_glucose_fbg, df_glucose])
+    df = pd.merge(
+        df_glucose,
+        df_glucose_fbg,
+        on="subject_visit_id",
+        how="outer",
+        indicator=True,
+        suffixes=("", "_2"),
+    )
+
+    for suffix in ["", "_2"]:
+        cols = [f"fasting_hrs{suffix}", f"fbg_value{suffix}", f"ogtt_value{suffix}"]
+        df[cols] = df[cols].apply(pd.to_numeric)
+        cols = [f"fbg_datetime{suffix}", f"ogtt_datetime{suffix}"]
+        df[cols] = df[cols].apply(pd.to_datetime)
+        df.loc[
+            (df[f"fbg_units{suffix}"] != "mmol/L (millimoles/L)")
+            & (df[f"fbg_value{suffix}"] >= 0),
+            f"fbg_units{suffix}",
+        ] = "mmol/L (millimoles/L)"
+        df.loc[
+            (df[f"ogtt_units{suffix}"] != "mmol/L (millimoles/L)")
+            & (df[f"ogtt_value{suffix}"] >= 0),
+            f"ogtt_units{suffix}",
+        ] = "mmol/L (millimoles/L)"
+        # remove values if not fasted
+        # df.loc[(df[f"fasting{suffix}"] != YES), f"fbg_value{suffix}"] = np.nan
+        # df.loc[(df[f"fasting{suffix}"] != YES), f"ogtt_value{suffix}"] = np.nan
+
+    # reconcile all to single column
+    df.loc[(df["fbg_value"].isna()) & (df["fbg_value_2"].notna()), "fbg_value"] = df[
+        "fbg_value_2"
+    ]
+    df.loc[(df["ogtt_value"].isna()) & (df["ogtt_value_2"].notna()), "ogtt_value"] = df[
+        "ogtt_value_2"
+    ]
+    cols = [col for col in list(df.columns) if col.endswith("_2")]
+    df.drop(columns=cols, inplace=True)
+    cols = [col for col in list(df.columns) if col.endswith("_3")]
+    df.drop(columns=cols, inplace=True)
+
+    df_subject_visit = get_subject_visit("meta_subject.subjectvisit")
+    df_consent = get_subject_consent("meta_consent.subjectconsent")
+    df_eos = get_eos("meta_prn.endofstudy")
+
+    df = pd.merge(df_subject_visit, df, on="subject_visit_id", how="left")
+    df = pd.merge(df, df_consent, on="subject_identifier", how="left")
+    df = pd.merge(df, df_eos, on="subject_identifier", how="left")
+
+    df["visit_days"] = df["baseline_datetime"].rsub(df["visit_datetime"]).dt.days
+    df["fgb_days"] = df["baseline_datetime"].rsub(df["fbg_datetime"]).dt.days
+    df["ogtt_days"] = df["baseline_datetime"].rsub(df["ogtt_datetime"]).dt.days
+    df["visit_days"] = pd.to_numeric(df["visit_days"], downcast="integer")
+    df["fgb_days"] = pd.to_numeric(df["fgb_days"], downcast="integer")
+    df["ogtt_days"] = pd.to_numeric(df["ogtt_days"], downcast="integer")
+
+    df = df.sort_values(by=["subject_identifier", "visit_code"])
+    df.reset_index(drop=True, inplace=True)
+    return df

meta_edc-0.3.19/meta_analytics/dataframes/glucose_endpoints/__init__.py

@@ -0,0 +1,2 @@
+from .endpoint_by_date import EndpointByDate
+from .glucose_endpoints_by_date import GlucoseEndpointsByDate

meta_edc-0.3.19/meta_analytics/dataframes/glucose_endpoints/constants.py

@@ -0,0 +1,31 @@
+OGTT_THRESHOLD_MET = "OGTT >= 11.1"
+EOS_DM_MET = "EOS - Patient developed diabetes"
+CASE_OGTT = 1
+CASE_EOS = 7
+CASE_FBG_ONLY = 4
+
+endpoint_columns = [
+    "subject_identifier",
+    "site",
+    "baseline_datetime",
+    "visit_datetime",
+    "interval_in_days",
+    "visit_code",
+    "fbg_value",
+    "ogtt_value",
+    "fbg_datetime",
+    "fasting",
+    "endpoint_label",
+    "endpoint_type",
+    "endpoint",
+    "offstudy_datetime",
+    "offstudy_reason",
+]
+
+endpoint_cases = {
+    CASE_OGTT: OGTT_THRESHOLD_MET,
+    2: "FBG >= 7 x 2, first OGTT<=11.1",
+    3: "FBG >= 7 x 2, second OGTT<=11.1",
+    4: "FBG >= 7 x 2, OGTT not considered",
+    CASE_EOS: EOS_DM_MET,
+}

meta_edc-0.3.19/meta_analytics/dataframes/glucose_endpoints/endpoint_by_date.py

@@ -0,0 +1,174 @@
+import numpy as np
+import pandas as pd
+from edc_constants.constants import YES
+
+from .constants import endpoint_cases
+
+
+class EndpointTdeltaError(Exception):
+    pass
+
+
+class InvalidCaseList(Exception):
+    pass
+
+
+class EndpointByDate:
+    """Given all timepoints for a subject, flag the first timepoint
+    where the protocol endpoint is reached.
+
+    IMPORTANT: Remove case one before passing to this class
+        * case 1. any OGTT >= 11.1
+
+    Evaluation is done in order
+
+    Order of protocol endpoint evaluation:
+      * case 2. FBG >= 7 x 2, first OGTT<=11.1
+      * case 3.  FBG >= 7 x 2, second OGTT<=11.1
+
+    Additional criteria considered:
+      1. any threshhold FBG must be taken while fasted (fasting=YES)
+      2. threshhold FBG readings must be consecutive (no
+         readings below threshold in the sequence regardless
+         of fasting)
+      3. at least 7 days between threshhold FBG readings.
+      4. at least one of the two threshold FBG readings must be taken
+         with an OGTT at the same timepoint.
+
+    Note:
+        case 4 is not a protocol endpoint. It considers only FBG and fasting.
+        It looks for two consecutive fasted threshold FBG readings.
+    """
+
+    valid_case_list = [2, 3, 4]
+
+    def __init__(
+        self,
+        subject_df: pd.DataFrame = None,
+        fbg_threshhold: float = None,
+        ogtt_threshhold: float = None,
+        case_list: list[int] | None = None,
+    ):
+        self.row = None
+        self.index = None
+        self.subject_df = subject_df[subject_df["fbg_value"].notna()]
+        self.subject_df = self.subject_df.reset_index(drop=True)
+        self.fbg_threshhold = fbg_threshhold
+        self.ogtt_threshhold = ogtt_threshhold
+        self.case_list = case_list or [2, 3]
+        if [x for x in self.case_list if x not in self.valid_case_list]:
+            raise InvalidCaseList(f"Expected any of {self.valid_case_list}. Got {case_list}.")
+        self.endpoint_cases = {k: v for k, v in endpoint_cases.items() if k in self.case_list}
+        self.evaluate()
+
+    def evaluate(self):
+        for index, _ in self.subject_df.iterrows():
+            if 2 in self.case_list and self.case_two(index):
+                break
+            elif 3 in self.case_list and self.case_three(index):
+                break
+            elif 4 in self.case_list and self.case_four(index):
+                break
+
+    def endpoint_reached(self, index: int, case: int, next_is_endpoint: bool | None = None):
+        """Update the subject_df"""
+        fbg_datetime = (
+            self.get_next("fbg_datetime", index)
+            if next_is_endpoint
+            else self.get("fbg_datetime", index)
+        )
+        self.subject_df.loc[self.subject_df["fbg_datetime"] == fbg_datetime, "endpoint"] = 1
+        self.subject_df["interval_in_days"] = np.nan
+        try:
+            self.subject_df.loc[
+                self.subject_df["fbg_datetime"] == fbg_datetime, "interval_in_days"
+            ] = self.sequential_assessments_in_days(index)
+        except EndpointTdeltaError:
+            pass
+        self.subject_df["interval_in_days"] = pd.to_numeric(
+            self.subject_df["interval_in_days"]
+        )
+        self.subject_df.loc[
+            self.subject_df["fbg_datetime"] == fbg_datetime, "endpoint_type"
+        ] = case
+        self.subject_df.loc[
+            self.subject_df["fbg_datetime"] == fbg_datetime, "endpoint_label"
+        ] = self.endpoint_cases[case]
+
+    def case_two(self, index: int):
+        """FBG >= 7 x 2, first OGTT<=11.1.
+
+        First FBG must be done with corresponding OGTT.
+        """
+        reached = (
+            self.get_next("fbg_datetime", index)
+            and self.get("fbg_value", index)
+            and self.get("ogtt_value", index)
+            and self.get_next("fbg_value", index)
+            and self.get("fbg_value", index) >= self.fbg_threshhold
+            and self.get("ogtt_value", index) < self.ogtt_threshhold
+            and self.get_next("fbg_value", index) >= self.fbg_threshhold
+            and self.fasting(index)
+            and (self.get_next("fbg_datetime", index) - self.get("fbg_datetime", index)).days
+            >= 7
+        )
+        if reached:
+            self.endpoint_reached(index, case=2, next_is_endpoint=True)
+        return reached
+
+    def case_three(self, index: int):
+        """FBG >= 7 x 2, second OGTT<=11.1.
+
+        Second FBG must be done with corresponding OGTT.
+        """
+        reached = (
+            self.get_next("fbg_datetime", index)
+            and self.get("fbg_value", index)
+            and self.get_next("fbg_value", index)
+            and self.get_next("ogtt_value", index)
+            and self.get("fbg_value", index) >= self.fbg_threshhold
+            and self.get_next("fbg_value", index) >= self.fbg_threshhold
+            and self.get_next("ogtt_value", index) < self.ogtt_threshhold
+            and self.fasting(index)
+            and (self.get_next("fbg_datetime", index) - self.get("fbg_datetime", index)).days
+            >= 7
+        )
+        if reached:
+            self.endpoint_reached(index, case=3, next_is_endpoint=True)
+        return reached
+
+    def case_four(self, index: int):
+        """FBG >= 7 x 2, OGTT not considered
+
+        This is not a protocol endpoint.
+        """
+        reached = (
+            self.get_next("fbg_datetime", index)
+            and self.get("fbg_value", index)
+            and self.get_next("fbg_value", index)
+            and self.get_next("ogtt_value", index)
+            and self.get("fbg_value", index) >= self.fbg_threshhold
+            and self.get_next("fbg_value", index) >= self.fbg_threshhold
+            and self.fasting(index)
+        )
+        if reached:
+            self.endpoint_reached(index, case=4, next_is_endpoint=True)
+        return reached
+
+    def fasting(self, index) -> bool:
+        return self.get("fasting", index) == YES and self.get_next("fasting", index) == YES
+
+    def sequential_assessments_in_days(self, index) -> int:
+        if not self.get_next("fbg_value", index):
+            raise EndpointTdeltaError
+        return (self.get_next("fbg_datetime", index) - self.get("visit_datetime", index)).days
+
+    def get(self, col: str, index: int) -> float | None:
+        try:
+            next_value = self.subject_df.iloc[index : index + 1][col].item()
+        except ValueError:
+            next_value = None
+        return next_value
+
+    def get_next(self, col: str, index: int) -> float | None:
+        return self.get(col, index + 1)