iints-sdk-python35 0.1.7__py3-none-any.whl

iints/data/importer.py

@@ -0,0 +1,275 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Any, Dict, Iterable, List, Optional, Tuple, Union
+import io
+import re
+import sys
+
+import pandas as pd
+
+from iints.data.ingestor import DataIngestor
+
+
+def _normalize_column(name: str) -> str:
+    return re.sub(r"[^a-z0-9]+", "", name.lower())
+
+
+def _find_column(columns: Iterable[str], candidates: Iterable[str]) -> Optional[str]:
+    normalized = {col: _normalize_column(col) for col in columns}
+    candidate_set = {_normalize_column(c) for c in candidates}
+    for col, norm in normalized.items():
+        if norm in candidate_set:
+            return col
+    return None
+
+
+DEFAULT_MAPPINGS: Dict[str, Dict[str, List[str]]] = {
+    "generic": {
+        "timestamp": ["timestamp", "time", "datetime", "date", "eventtime", "device timestamp"],
+        "glucose": ["glucose", "bg", "sgv", "sensorglucose", "glucosemgdl", "glucosevalue"],
+        "carbs": ["carbs", "carb", "carbohydrates", "carbsg", "carbgrams"],
+        "insulin": ["insulin", "insulinunits", "bolus", "basal", "totalinsulin"],
+    },
+    "dexcom": {
+        "timestamp": ["timestamp", "eventtime", "device timestamp"],
+        "glucose": ["glucose", "glucosevalue", "sgv", "sensorglucose"],
+        "carbs": ["carbs", "carb", "carbohydrates"],
+        "insulin": ["insulin", "insulinunits", "bolus", "basal"],
+    },
+    "libre": {
+        "timestamp": ["timestamp", "device timestamp", "datetime", "time"],
+        "glucose": ["glucose", "glucosevalue", "sensorglucose", "sgv"],
+        "carbs": ["carbs", "carb", "carbohydrates"],
+        "insulin": ["insulin", "insulinunits", "bolus", "basal"],
+    },
+}
+
+IMPORT_FORMAT_SCHEMAS: Dict[str, Dict[str, List[str]]] = {
+    "generic": {
+        "required": ["timestamp", "glucose"],
+        "optional": ["carbs", "insulin"],
+    },
+    "dexcom": {
+        "required": ["timestamp", "glucose"],
+        "optional": ["carbs", "insulin"],
+    },
+    "libre": {
+        "required": ["timestamp", "glucose"],
+        "optional": ["carbs", "insulin"],
+    },
+}
+
+
+@dataclass
+class ImportResult:
+    dataframe: pd.DataFrame
+    scenario: Dict[str, Any]
+
+
+def guess_column_mapping(columns: Iterable[str], data_format: str = "generic") -> Dict[str, Optional[str]]:
+    candidates = DEFAULT_MAPPINGS.get(data_format, DEFAULT_MAPPINGS["generic"])
+    return {
+        "timestamp": _find_column(columns, candidates.get("timestamp", [])),
+        "glucose": _find_column(columns, candidates.get("glucose", [])),
+        "carbs": _find_column(columns, candidates.get("carbs", [])),
+        "insulin": _find_column(columns, candidates.get("insulin", [])),
+    }
+
+
+def validate_import_schema(
+    columns: Iterable[str],
+    data_format: str,
+    column_map: Optional[Dict[str, str]] = None,
+) -> None:
+    schema = IMPORT_FORMAT_SCHEMAS.get(data_format, IMPORT_FORMAT_SCHEMAS["generic"])
+    candidates = DEFAULT_MAPPINGS.get(data_format, DEFAULT_MAPPINGS["generic"])
+    mapping = column_map or {}
+
+    missing: List[str] = []
+    for key in schema["required"]:
+        if key in mapping:
+            continue
+        found = _find_column(columns, candidates.get(key, []))
+        if found is None:
+            missing.append(key)
+
+    if missing:
+        raise ValueError(
+            f"Missing required columns for format '{data_format}': {', '.join(missing)}. "
+            f"Columns: {list(columns)}"
+        )
+
+
+def import_cgm_dataframe(
+    df: pd.DataFrame,
+    data_format: str = "generic",
+    column_map: Optional[Dict[str, str]] = None,
+    time_unit: str = "minutes",
+    source: Optional[str] = None,
+) -> pd.DataFrame:
+    """
+    Import CGM data from an in-memory DataFrame into the universal IINTS schema.
+    """
+    columns = list(df.columns)
+    mapping = column_map or {}
+    mapping = {k: v for k, v in mapping.items() if v}
+
+    candidates = DEFAULT_MAPPINGS.get(data_format, DEFAULT_MAPPINGS["generic"])
+
+    validate_import_schema(columns, data_format=data_format, column_map=mapping)
+
+    def resolve(key: str, required: bool = True) -> Optional[str]:
+        if key in mapping:
+            return mapping[key]
+        col = _find_column(columns, candidates.get(key, []))
+        if required and col is None:
+            raise ValueError(f"Missing required column for '{key}'. Columns: {columns}")
+        return col
+
+    ts_col = resolve("timestamp", required=True)
+    glucose_col = resolve("glucose", required=True)
+    carbs_col = resolve("carbs", required=False)
+    insulin_col = resolve("insulin", required=False)
+
+    df = df.rename(
+        columns={
+            ts_col: "timestamp",
+            glucose_col: "glucose",
+            carbs_col: "carbs",
+            insulin_col: "insulin",
+        }
+    )
+
+    if "carbs" not in df.columns:
+        df["carbs"] = 0.0
+    if "insulin" not in df.columns:
+        df["insulin"] = 0.0
+
+    # Parse timestamps
+    if pd.api.types.is_numeric_dtype(df["timestamp"]):
+        # Assume numeric (minutes or seconds)
+        if time_unit == "seconds":
+            df["timestamp"] = df["timestamp"].astype(float) / 60.0
+        else:
+            df["timestamp"] = df["timestamp"].astype(float)
+    elif pd.api.types.is_datetime64_any_dtype(df["timestamp"]):
+        ts = df["timestamp"]
+        df["timestamp"] = (ts - ts.iloc[0]).dt.total_seconds() / 60.0
+    else:
+        # Try datetime parsing, fallback to numeric
+        ts = pd.to_datetime(df["timestamp"], errors="coerce")
+        if ts.isna().all():
+            if time_unit == "seconds":
+                df["timestamp"] = df["timestamp"].astype(float) / 60.0
+            else:
+                df["timestamp"] = df["timestamp"].astype(float)
+        else:
+            df["timestamp"] = (ts - ts.iloc[0]).dt.total_seconds() / 60.0
+
+    df["source"] = source or data_format
+    ingestor = DataIngestor()
+    ingestor._validate_schema(df, ingestor.UNIVERSAL_SCHEMA)
+    return df[list(ingestor.UNIVERSAL_SCHEMA.keys())]
+
+
+def import_cgm_csv(
+    path: Union[str, Path],
+    data_format: str = "generic",
+    column_map: Optional[Dict[str, str]] = None,
+    time_unit: str = "minutes",
+    source: Optional[str] = None,
+) -> pd.DataFrame:
+    """
+    Import CGM data from CSV into the universal IINTS schema.
+    """
+    df = pd.read_csv(path)
+    return import_cgm_dataframe(
+        df,
+        data_format=data_format,
+        column_map=column_map,
+        time_unit=time_unit,
+        source=source,
+    )
+
+
+def scenario_from_dataframe(
+    df: pd.DataFrame,
+    scenario_name: str,
+    scenario_version: str = "1.0",
+    description: str = "Imported CGM scenario",
+    carb_threshold: float = 0.1,
+    absorption_delay_minutes: int = 10,
+    duration_minutes: int = 60,
+) -> Dict[str, Any]:
+    stress_events = []
+    if "carbs" in df.columns:
+        for _, row in df[df["carbs"] > carb_threshold].iterrows():
+            stress_events.append(
+                {
+                    "start_time": int(row["timestamp"]),
+                    "event_type": "meal",
+                    "value": float(row["carbs"]),
+                    "absorption_delay_minutes": absorption_delay_minutes,
+                    "duration": duration_minutes,
+                }
+            )
+
+    return {
+        "scenario_name": scenario_name,
+        "scenario_version": scenario_version,
+        "description": description,
+        "stress_events": stress_events,
+    }
+
+
+def scenario_from_csv(
+    path: Union[str, Path],
+    scenario_name: str = "Imported CGM Scenario",
+    scenario_version: str = "1.0",
+    data_format: str = "generic",
+    column_map: Optional[Dict[str, str]] = None,
+    time_unit: str = "minutes",
+    carb_threshold: float = 0.1,
+) -> ImportResult:
+    df = import_cgm_csv(
+        path,
+        data_format=data_format,
+        column_map=column_map,
+        time_unit=time_unit,
+    )
+    scenario = scenario_from_dataframe(
+        df,
+        scenario_name=scenario_name,
+        scenario_version=scenario_version,
+        carb_threshold=carb_threshold,
+    )
+    return ImportResult(dataframe=df, scenario=scenario)
+
+
+def export_standard_csv(df: pd.DataFrame, output_path: Union[str, Path]) -> str:
+    output_path = Path(output_path)
+    output_path.parent.mkdir(parents=True, exist_ok=True)
+    df.to_csv(output_path, index=False)
+    return str(output_path)
+
+
+def _read_demo_csv_text() -> str:
+    if sys.version_info >= (3, 9):
+        from importlib.resources import files
+        return files("iints.data.demo").joinpath("demo_cgm.csv").read_text()
+    from importlib import resources
+    return resources.read_text("iints.data.demo", "demo_cgm.csv")
+
+
+def load_demo_dataframe() -> pd.DataFrame:
+    text = _read_demo_csv_text()
+    return pd.read_csv(io.StringIO(text))
+
+
+def export_demo_csv(output_path: Union[str, Path]) -> str:
+    output_path = Path(output_path)
+    output_path.parent.mkdir(parents=True, exist_ok=True)
+    output_path.write_text(_read_demo_csv_text())
+    return str(output_path)

iints/data/ingestor.py

@@ -0,0 +1,162 @@
+import pandas as pd
+from pathlib import Path
+from typing import Dict, Any, Union, Optional
+import yaml
+
+class DataIngestor:
+    """
+    Standardized Data Bridge for ingesting various diabetes datasets into a
+    universal IINTS-AF format.
+    """
+    UNIVERSAL_SCHEMA = {
+        "timestamp": float,
+        "glucose": float,
+        "carbs": float, # Can be null, but pandas will infer float if mixed
+        "insulin": float, # Can be null, but pandas will infer float if mixed
+        "source": str,
+    }
+
+    def __init__(self, data_dir: Optional[str] = None):
+        self.data_dir = data_dir or "./data"
+        # Zorg dat de rest van je code zelf dit pad gebruikt om bestanden te vinden
+
+    def _load_ohio_t1dm_csv(self, file_path: Path) -> pd.DataFrame:
+        """
+        Loads and transforms Ohio T1DM dataset CSV into universal schema.
+        Expected columns in Ohio T1DM: 'timestamp', 'glucose', 'carbs', 'insulin'
+        """
+        df = pd.read_csv(file_path)
+
+        # Assuming 'timestamp' is already in minutes from start or can be converted
+        # For simplicity, assuming it's already a float representing minutes
+        # If it's a datetime, conversion would be needed:
+        # df['timestamp'] = (df['timestamp'] - df['timestamp'].min()).dt.total_seconds() / 60.0
+
+        # Rename columns to match universal schema if necessary
+        # Example: if original columns were different, map them here.
+        # For Ohio T1DM, let's assume they are already lowercase 'glucose', 'carbs', 'insulin'
+        # if 'BG' in df.columns: df = df.rename(columns={'BG': 'glucose'})
+        # if 'Carbs' in df.columns: df = df.rename(columns={'Carbs': 'carbs'})
+        # if 'Insulin' in df.columns: df = df.rename(columns={'Insulin': 'insulin'})
+
+        # Add 'source' column
+        df['source'] = 'public_ohio_t1dm'
+        
+        # Ensure only universal schema columns are present and in order
+        required_cols = list(self.UNIVERSAL_SCHEMA.keys())
+        for col in required_cols:
+            if col not in df.columns:
+                df[col] = pd.NA # Add missing columns as NA
+        
+        return df[required_cols]
+
+    def _validate_schema(self, df: pd.DataFrame, schema: Dict[str, type]) -> None:
+        """
+        Validates DataFrame against the expected schema.
+        Raises ValueError if validation fails.
+        """
+        for col, expected_type in schema.items():
+            if col not in df.columns:
+                raise ValueError(f"Missing required column: {col}")
+            # Basic type check (pandas dtypes are more complex, this is a simplified check)
+            # if not pd.api.types.is_dtype_equal(df[col].dtype, pd.Series(dtype=expected_type).dtype):
+            #     print(f"Warning: Column '{col}' type mismatch. Expected {expected_type}, got {df[col].dtype}")
+        
+        # Basic quality checks (from DATA_SCHEMA.md)
+        if 'glucose' in df.columns:
+            if not ((df['glucose'] >= 20) & (df['glucose'] <= 600)).all():
+                raise ValueError("Glucose values outside acceptable range (20-600 mg/dL)")
+        if 'insulin' in df.columns and not df['insulin'].isna().all():
+            if not ((df['insulin'] >= 0) & (df['insulin'] <= 50)).all():
+                raise ValueError("Insulin values outside acceptable range (0-50 units)")
+        
+        # Check for missing timestamps (assuming 'timestamp' exists)
+        if df['timestamp'].isnull().any():
+            raise ValueError("Missing values in 'timestamp' column.")
+
+    def get_patient_model(self, file_path: Union[str, Path], data_type: str) -> pd.DataFrame:
+        """
+        Loads patient data from a file and returns it as a standardized DataFrame.
+
+        Args:
+            file_path (Union[str, Path]): Path to the data file. Can be extension-less for 'model' type.
+            data_type (str): Type of the data source (e.g., 'ohio_t1dm', 'iints_standard_csv', 'model').
+
+        Returns:
+            pd.DataFrame: A DataFrame conforming to the universal IINTS-AF schema.
+        
+        Raises:
+            ValueError: If the data_type is not supported or validation fails.
+            FileNotFoundError: If the data file cannot be found.
+        """
+        file_path = Path(file_path)
+
+        # If data type is 'model' (JSON), ensure the .json extension is present.
+        if data_type == 'model' or data_type == 'iints_standard_json':
+            if not file_path.suffix:
+                file_path = file_path.with_suffix('.json')
+
+        if not file_path.is_file():
+            raise FileNotFoundError(f"Data file not found: {file_path}")
+
+        df = pd.DataFrame()
+        if data_type == 'ohio_t1dm':
+            df = self._load_ohio_t1dm_csv(file_path)
+        elif data_type == 'iints_standard_csv':
+            # Assume this is already in the universal schema format
+            df = pd.read_csv(file_path)
+        elif data_type == 'model' or data_type == 'iints_standard_json':
+            # Assumes a records-oriented JSON file.
+            df = pd.read_json(file_path, orient='records')
+            # Add source column if not present
+            if 'source' not in df.columns:
+                df['source'] = 'iints_standard_json'
+
+            # Ensure only universal schema columns are present and in order
+            required_cols = list(self.UNIVERSAL_SCHEMA.keys())
+            for col in required_cols:
+                if col not in df.columns:
+                    df[col] = pd.NA # Add missing columns as NA
+            df = df[required_cols]
+        else:
+            raise ValueError(f"Unsupported data type: {data_type}")
+        
+        # Validate the loaded data against the universal schema
+        self._validate_schema(df, self.UNIVERSAL_SCHEMA)
+
+        return df
+
+if __name__ == "__main__":
+    # Example usage:
+    # This assumes you have a timeseries.csv in data_packs/public/ohio_t1dm/patient_XXX/
+    # For testing, we'll try to find one.
+    
+    ohio_data_path = Path("data_packs/public/ohio_t1dm")
+    
+    patient_dirs = [d for d in ohio_data_path.iterdir() if d.is_dir() and d.name.startswith("patient_")]
+    
+    if patient_dirs:
+        sample_timeseries_file = None
+        for patient_dir in patient_dirs:
+            if (patient_dir / "timeseries.csv").is_file():
+                sample_timeseries_file = patient_dir / "timeseries.csv"
+                break
+        
+        if sample_timeseries_file:
+            print(f"Loading sample Ohio T1DM data from: {sample_timeseries_file}")
+            ingestor = DataIngestor()
+            try:
+                df = ingestor.get_patient_model(sample_timeseries_file, 'ohio_t1dm')
+                print("Data loaded successfully and validated:")
+                print(df.head())
+                df.info()
+            except FileNotFoundError as e:
+                print(f"Error loading data: {e}")
+            except ValueError as e:
+                print(f"Data quality issue detected during validation for {sample_timeseries_file.name}: {e}")
+            except Exception as e:
+                print(f"An unexpected error occurred: {e}")
+        else:
+            print(f"No 'timeseries.csv' found in any patient directory within {ohio_data_path}. Cannot run example.")
+    else:
+        print(f"No patient directories found in {ohio_data_path}. Cannot run example.")