iints-sdk-python35 0.0.18__py3-none-any.whl

iints/data/ingestor.py

@@ -0,0 +1,162 @@
+import pandas as pd
+from pathlib import Path
+from typing import Dict, Any, Union, Optional
+import yaml
+
+class DataIngestor:
+    """
+    Standardized Data Bridge for ingesting various diabetes datasets into a
+    universal IINTS-AF format.
+    """
+    UNIVERSAL_SCHEMA = {
+        "timestamp": float,
+        "glucose": float,
+        "carbs": float, # Can be null, but pandas will infer float if mixed
+        "insulin": float, # Can be null, but pandas will infer float if mixed
+        "source": str,
+    }
+
+    def __init__(self, data_dir: Optional[str] = None):
+        self.data_dir = data_dir or "./data"
+        # Zorg dat de rest van je code zelf dit pad gebruikt om bestanden te vinden
+
+    def _load_ohio_t1dm_csv(self, file_path: Path) -> pd.DataFrame:
+        """
+        Loads and transforms Ohio T1DM dataset CSV into universal schema.
+        Expected columns in Ohio T1DM: 'timestamp', 'glucose', 'carbs', 'insulin'
+        """
+        df = pd.read_csv(file_path)
+
+        # Assuming 'timestamp' is already in minutes from start or can be converted
+        # For simplicity, assuming it's already a float representing minutes
+        # If it's a datetime, conversion would be needed:
+        # df['timestamp'] = (df['timestamp'] - df['timestamp'].min()).dt.total_seconds() / 60.0
+
+        # Rename columns to match universal schema if necessary
+        # Example: if original columns were different, map them here.
+        # For Ohio T1DM, let's assume they are already lowercase 'glucose', 'carbs', 'insulin'
+        # if 'BG' in df.columns: df = df.rename(columns={'BG': 'glucose'})
+        # if 'Carbs' in df.columns: df = df.rename(columns={'Carbs': 'carbs'})
+        # if 'Insulin' in df.columns: df = df.rename(columns={'Insulin': 'insulin'})
+
+        # Add 'source' column
+        df['source'] = 'public_ohio_t1dm'
+        
+        # Ensure only universal schema columns are present and in order
+        required_cols = list(self.UNIVERSAL_SCHEMA.keys())
+        for col in required_cols:
+            if col not in df.columns:
+                df[col] = pd.NA # Add missing columns as NA
+        
+        return df[required_cols]
+
+    def _validate_schema(self, df: pd.DataFrame, schema: Dict[str, type]) -> None:
+        """
+        Validates DataFrame against the expected schema.
+        Raises ValueError if validation fails.
+        """
+        for col, expected_type in schema.items():
+            if col not in df.columns:
+                raise ValueError(f"Missing required column: {col}")
+            # Basic type check (pandas dtypes are more complex, this is a simplified check)
+            # if not pd.api.types.is_dtype_equal(df[col].dtype, pd.Series(dtype=expected_type).dtype):
+            #     print(f"Warning: Column '{col}' type mismatch. Expected {expected_type}, got {df[col].dtype}")
+        
+        # Basic quality checks (from DATA_SCHEMA.md)
+        if 'glucose' in df.columns:
+            if not ((df['glucose'] >= 20) & (df['glucose'] <= 600)).all():
+                raise ValueError("Glucose values outside acceptable range (20-600 mg/dL)")
+        if 'insulin' in df.columns and not df['insulin'].isna().all():
+            if not ((df['insulin'] >= 0) & (df['insulin'] <= 50)).all():
+                raise ValueError("Insulin values outside acceptable range (0-50 units)")
+        
+        # Check for missing timestamps (assuming 'timestamp' exists)
+        if df['timestamp'].isnull().any():
+            raise ValueError("Missing values in 'timestamp' column.")
+
+    def get_patient_model(self, file_path: Union[str, Path], data_type: str) -> pd.DataFrame:
+        """
+        Loads patient data from a file and returns it as a standardized DataFrame.
+
+        Args:
+            file_path (Union[str, Path]): Path to the data file. Can be extension-less for 'model' type.
+            data_type (str): Type of the data source (e.g., 'ohio_t1dm', 'iints_standard_csv', 'model').
+
+        Returns:
+            pd.DataFrame: A DataFrame conforming to the universal IINTS-AF schema.
+        
+        Raises:
+            ValueError: If the data_type is not supported or validation fails.
+            FileNotFoundError: If the data file cannot be found.
+        """
+        file_path = Path(file_path)
+
+        # If data type is 'model' (JSON), ensure the .json extension is present.
+        if data_type == 'model' or data_type == 'iints_standard_json':
+            if not file_path.suffix:
+                file_path = file_path.with_suffix('.json')
+
+        if not file_path.is_file():
+            raise FileNotFoundError(f"Data file not found: {file_path}")
+
+        df = pd.DataFrame()
+        if data_type == 'ohio_t1dm':
+            df = self._load_ohio_t1dm_csv(file_path)
+        elif data_type == 'iints_standard_csv':
+            # Assume this is already in the universal schema format
+            df = pd.read_csv(file_path)
+        elif data_type == 'model' or data_type == 'iints_standard_json':
+            # Assumes a records-oriented JSON file.
+            df = pd.read_json(file_path, orient='records')
+            # Add source column if not present
+            if 'source' not in df.columns:
+                df['source'] = 'iints_standard_json'
+
+            # Ensure only universal schema columns are present and in order
+            required_cols = list(self.UNIVERSAL_SCHEMA.keys())
+            for col in required_cols:
+                if col not in df.columns:
+                    df[col] = pd.NA # Add missing columns as NA
+            df = df[required_cols]
+        else:
+            raise ValueError(f"Unsupported data type: {data_type}")
+        
+        # Validate the loaded data against the universal schema
+        self._validate_schema(df, self.UNIVERSAL_SCHEMA)
+
+        return df
+
+if __name__ == "__main__":
+    # Example usage:
+    # This assumes you have a timeseries.csv in data_packs/public/ohio_t1dm/patient_XXX/
+    # For testing, we'll try to find one.
+    
+    ohio_data_path = Path("data_packs/public/ohio_t1dm")
+    
+    patient_dirs = [d for d in ohio_data_path.iterdir() if d.is_dir() and d.name.startswith("patient_")]
+    
+    if patient_dirs:
+        sample_timeseries_file = None
+        for patient_dir in patient_dirs:
+            if (patient_dir / "timeseries.csv").is_file():
+                sample_timeseries_file = patient_dir / "timeseries.csv"
+                break
+        
+        if sample_timeseries_file:
+            print(f"Loading sample Ohio T1DM data from: {sample_timeseries_file}")
+            ingestor = DataIngestor()
+            try:
+                df = ingestor.get_patient_model(sample_timeseries_file, 'ohio_t1dm')
+                print("Data loaded successfully and validated:")
+                print(df.head())
+                df.info()
+            except FileNotFoundError as e:
+                print(f"Error loading data: {e}")
+            except ValueError as e:
+                print(f"Data quality issue detected during validation for {sample_timeseries_file.name}: {e}")
+            except Exception as e:
+                print(f"An unexpected error occurred: {e}")
+        else:
+            print(f"No 'timeseries.csv' found in any patient directory within {ohio_data_path}. Cannot run example.")
+    else:
+        print(f"No patient directories found in {ohio_data_path}. Cannot run example.")

iints/data/nightscout.py

@@ -0,0 +1,128 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import Any, Dict, Iterable, List, Optional
+import asyncio
+
+import pandas as pd
+
+from iints.data.importer import ImportResult, import_cgm_dataframe, scenario_from_dataframe
+
+
+@dataclass
+class NightscoutConfig:
+    url: str
+    api_secret: Optional[str] = None
+    token: Optional[str] = None
+    start: Optional[str] = None  # ISO string or date-like
+    end: Optional[str] = None
+    limit: Optional[int] = None
+
+
+def _require_nightscout():
+    try:
+        import py_nightscout as nightscout  # type: ignore
+        return nightscout
+    except Exception:
+        try:
+            import nightscout as nightscout  # type: ignore
+            return nightscout
+        except Exception as exc:  # pragma: no cover - optional dependency
+            raise ImportError(
+                "py-nightscout is required. Install with `pip install iints-sdk-python35[nightscout]`."
+            ) from exc
+
+
+async def _fetch_entries_async(config: NightscoutConfig) -> List[Any]:
+    nightscout = _require_nightscout()
+    if not hasattr(nightscout, "Api"):
+        raise RuntimeError("py-nightscout API wrapper missing 'Api' class.")
+    api = nightscout.Api(  # type: ignore[attr-defined]
+        config.url,
+        api_secret=config.api_secret,
+        token=config.token,
+    )
+    if not hasattr(api, "get_sgvs"):
+        raise RuntimeError("py-nightscout client missing 'get_sgvs' method.")
+    entries = await api.get_sgvs()
+    return list(entries or [])
+
+
+def _entry_get(entry: Any, key: str) -> Any:
+    if isinstance(entry, dict):
+        return entry.get(key)
+    return getattr(entry, key, None)
+
+
+def _entries_to_dataframe(entries: Iterable[Any]) -> pd.DataFrame:
+    rows: List[Dict[str, Any]] = []
+    for entry in entries:
+        glucose = _entry_get(entry, "sgv") or _entry_get(entry, "glucose")
+        if glucose is None:
+            continue
+        ts_raw = (
+            _entry_get(entry, "date")
+            or _entry_get(entry, "dateString")
+            or _entry_get(entry, "timestamp")
+        )
+        if ts_raw is None:
+            continue
+        if isinstance(ts_raw, (int, float)):
+            timestamp = pd.to_datetime(ts_raw, unit="ms", errors="coerce")
+        else:
+            timestamp = pd.to_datetime(ts_raw, errors="coerce")
+        if pd.isna(timestamp):
+            continue
+        rows.append(
+            {
+                "timestamp": timestamp,
+                "glucose": float(glucose),
+                "carbs": 0.0,
+                "insulin": 0.0,
+            }
+        )
+    return pd.DataFrame(rows)
+
+
+def fetch_nightscout_dataframe(config: NightscoutConfig) -> pd.DataFrame:
+    entries = asyncio.run(_fetch_entries_async(config))
+    df = _entries_to_dataframe(entries)
+    if df.empty:
+        return df
+
+    if config.start:
+        start_ts = pd.to_datetime(config.start, errors="coerce")
+        if pd.notna(start_ts):
+            df = df[df["timestamp"] >= start_ts]
+    if config.end:
+        end_ts = pd.to_datetime(config.end, errors="coerce")
+        if pd.notna(end_ts):
+            df = df[df["timestamp"] <= end_ts]
+    if config.limit:
+        df = df.head(int(config.limit))
+    return df.reset_index(drop=True)
+
+
+def import_nightscout(
+    config: NightscoutConfig,
+    scenario_name: str = "Nightscout Import",
+    scenario_version: str = "1.0",
+    carb_threshold: float = 0.1,
+) -> ImportResult:
+    df = fetch_nightscout_dataframe(config)
+    if df.empty:
+        raise ValueError("No Nightscout CGM entries found for the given parameters.")
+
+    standard_df = import_cgm_dataframe(
+        df,
+        data_format="generic",
+        time_unit="minutes",
+        source="nightscout",
+    )
+    scenario = scenario_from_dataframe(
+        standard_df,
+        scenario_name=scenario_name,
+        scenario_version=scenario_version,
+        carb_threshold=carb_threshold,
+    )
+    return ImportResult(dataframe=standard_df, scenario=scenario)