SleePyPhases 0.2.0__tar.gz → 0.3.0__tar.gz

{sleepyphases-0.2.0 → sleepyphases-0.3.0}/PKG-INFO

@@ -1,6 +1,6 @@
-Metadata-Version: 2.2
+Metadata-Version: 2.4
 Name: SleePyPhases
-Version: 0.2.0
+Version: 0.3.0
 Summary: A framwork for creating deep learning pipelines for sleep data
 Home-page: https://gitlab.com/sleep-is-all-you-need/sleepyphases
 Author: Franz Ehrlich
@@ -20,6 +20,7 @@ Dynamic: classifier
 Dynamic: description
 Dynamic: description-content-type
 Dynamic: home-page
+Dynamic: license-file
 Dynamic: requires-dist
 Dynamic: requires-python
 Dynamic: summary

{sleepyphases-0.2.0 → sleepyphases-0.3.0}/SleePyPhases/DataManipulation.py

@@ -14,8 +14,8 @@ class DataManipulation(pyPhasesDataManipulation, Swappable):
     def __init__(self, steps, splitName, projectconfig, seed=2, threshold=None, recordMetadata=None, **kwargs) -> None:
         super().__init__(steps, splitName, **kwargs)
         self.catMatrix = None
-        self.numLabels = projectconfig["numLabels"]
-        self.numClasses = projectconfig["numClasses"]
+        self.numLabels = projectconfig["numLabels"] if "numLabels" in projectconfig else 1
+        self.numClasses = projectconfig["numClasses"] if "numLabels" in projectconfig else [1]
         self.rng = default_rng(seed=2)
         self.threshold = threshold
         self.recordMetadata = recordMetadata

{sleepyphases-0.2.0 → sleepyphases-0.3.0}/SleePyPhases/MultiScorer.py

@@ -42,19 +42,27 @@ class MultiScorer(Scorer):
             if isinstance(predictions, list) or isinstance(predictions, tuple):
                 p = predictions[i]
             elif predictions.shape[-1] == len(self.multiClassNums):
+                if len(predictions.shape) == 3:
+                    predictions = predictions.reshape(-1, predictions.shape[-1])
                 p = predictions[:, i]
             else:
                 start = sum(self.multiClassNums[:i])
                 end = start + self.multiClassNums[i]
+                if len(predictions.shape) == 3:
+                    predictions = predictions.reshape(-1, predictions.shape[-1])
                 p = predictions[:, start:end]
             
             if isinstance(truth, list) or isinstance(truth, tuple):
                 t = truth[i]
             elif truth.shape[-1] == len(self.multiClassNums):
+                if len(truth.shape) == 3:
+                    truth = truth.reshape(-1, truth.shape[-1])
                 t = truth[:, i]
             else:
                 start = sum(self.multiClassNums[:i])
                 end = start + self.multiClassNums[i]
+                if len(truth.shape) == 3:
+                    truth = truth.reshape(-1, truth.shape[-1])
                 t = truth[:, start:end]
 
             

{sleepyphases-0.2.0 → sleepyphases-0.3.0}/SleePyPhases/RecordFeatureExtraction.py

@@ -25,6 +25,10 @@ class RecordFeatureExtraction(pyPhasesFeatureExtraction):
         
     def updateMetadataByStep(self, stepname: str, recordSignal: RecordSignal, eventlist: List[Event], metadata: pd.DataFrame, channel=None, **options):
         features = self.step(stepname, recordSignal, eventlist, channel, **options)
+
+        if features is None:
+            return None
+        
         recordId = recordSignal.recordId
         # update row with index recordId and channel with features dict or create new if not exist
         
@@ -252,15 +256,83 @@ class RecordFeatureExtraction(pyPhasesFeatureExtraction):
         ret = {
             "cycle-count": len(cycles["start"]),
         }
+        frequencyFactor = 30
         
-        for cycleNumber, (start, stop) in enumerate(zip(cycles["start"], cycles["end"])):
-            if lastStop > 0:
-                remTime = sum([e.duration for e in remEvents if e.start >= lastStop and e.start < start])
-                ret[f"cycle-{cycleNumber}-rem-duration"] = remTime
+        for cycleNumber, (start, stop) in enumerate(zip(cycles["start"], cycles["stop"])):
+            start *= frequencyFactor
+            stop *= frequencyFactor
+            # if lastStop > 0:
+            remTime = sum([e.duration for e in remEvents if e.start >= lastStop and e.start < start])
+            ret[f"cycle-{cycleNumber}-rem-duration"] = remTime
 
             lastStop = stop
 
+        # remTime = sum([e.duration for e in remEvents if e.start >= lastStop and e.start < start])
+        # ret[f"cycle-{cycleNumber}-rem-duration"] = remTime
+
+        return ret
+    
+    def spindles(self, segmentSignal: RecordSignal, eventlist: List[Event], channel: str):
+        from .external.sumo.sumo.scripts.a7.butter_filter import butter_bandpass_filter, downsample
+        # from .external.sumo.a7.detect_spindles import detect_spindle
+        from .external.sumo.sumo.scripts.predict_plain_data import SimpleDataset
+        # from .external.sumo.sumo.predict_plain_data import A7
+        import pytorch_lightning as pl
+        from torch.utils.data import Dataset, DataLoader
+        import torch
+        from .external.sumo.sumo.model.sumo import SUMO
+        from .external.sumo.sumo.config.config import Config
+        # from .external.sumo.model.data import spindle_vect_to_indices
+
+        config = Config('predict', create_dirs=False)
+
+        def get_model(p):
+            # p = Path(p)
+
+            model_checkpoint = torch.load(p)
+
+            model = SUMO(config)
+            model.load_state_dict(model_checkpoint['state_dict'])
+
+            return model
+
+        eegSignal = segmentSignal.getSignalByName(channel)
+        resample_rate = 100
+        model_path = "SleePyPhases/external/sumo/sumo/output/final.ckpt"
+        
+        if len(eegSignal.signal) == 0:
+            return None
+
+        eegs = []
+        eegSignal = downsample(butter_bandpass_filter(eegSignal.signal, 0.3, 30.0, eegSignal.frequency, 10), eegSignal.frequency, resample_rate)
+        eegs.append(eegSignal)
+
+        dataset = SimpleDataset(eegs)
+        dataloader = DataLoader(dataset)
+        model = get_model(model_path)
+        trainer = pl.Trainer(num_sanity_val_steps=0, logger=False)
+        predictions = trainer.predict(model, dataloader)
+        pred = predictions[0]
+
+        # spindles_a7 = A7(x, resample_rate)
+        spindle_vect = pred[0].numpy()
+        
+        diff = np.diff(np.r_[0, spindle_vect, 0])  # be able to detect spindles at the start and end of vector
+        spindles = np.c_[np.argwhere(diff == 1), np.argwhere(diff == -1)]  / resample_rate
+        # spindles = spindle_vect_to_indices(spindle_vect) / resample_rate
+
+
+        # frequencies, psd = signal.welch(eegs[0][s.start:s.end()], 
+        #                                 fs=100,
+        #                                 nperseg=min(256, s.duration),
+        #                                 scaling='spectrum')
+        # dominant_freq = frequencies[np.argmax(psd)]
+
+        spindles = [Event("spindle", s[0], s[1] - s[0]) for s in spindles]
+
+        
         return {
-            "cycle-count": len(cycles["start"]),
+            "spindles-count": len(spindles),
+            "spindles-mean-druation": np.array([s.duration for s in spindles]).mean()
         }
-    
+    

{sleepyphases-0.2.0 → sleepyphases-0.3.0}/SleePyPhases/SleepMetaData.py

@@ -76,6 +76,7 @@ class SleepMetaData:
                 self.fromSleepSignalBin(signal, frequency=frequency)
             case 'Sleep4':
                 self.fromSleepSignalBin(signal, frequency=frequency)
+                self.fromSleepSignalFour(signal, frequency=frequency)
 
     def fromSleepSignalBin(self, signal, frequency = 1):
         # all time values in seconds
@@ -100,6 +101,11 @@ class SleepMetaData:
         self.registerMetadata("waso", trt-sLatency-tst, unit="s")
         self.registerMetadata("sEfficiency", tst / trt * 100, unit="%")
 
+    def fromSleepSignalFour(self, signal, frequency = 1):
+        whereREM = np.where(signal == 1)
+        rLatency = whereREM[0] - self.getMetaData("sLatency")
+        self.registerMetadata("rLatency", rLatency)
+
     
     def fromApneaSignal(self, signal, classification=None):
         # classification = classification or ["None", "obstructive", "mixed", "central", "hypopnea"]

{sleepyphases-0.2.0 → sleepyphases-0.3.0}/SleePyPhases/phases/Eval.py

@@ -81,13 +81,12 @@ class Eval(Phase):
 
         # setup data and scorer
         
-        classNums = self.getConfig("model.multiclassificationSize", [2])
+        classNums = self.getConfig("numClasses", [2])
         labelNames = self.getConfig("classification.labelNames")
         ignoreIndex = self.getConfig("classification.ignoreIndex", -1)
         metrics = self.getConfig("trainingParameter.validationMetrics")
 
-
-        scorer = MultiScorer(classNums, metrics, scorerNames=labelNames, ignoreClasses=[ignoreIndex], scorerClass=EventScorer)
+        scorer = MultiScorer(classNums, metrics, scorerNames=labelNames, ignoreClasses=[ignoreIndex])
         for s in scorer.scorer.values():
             s.trace = True
         threshold = self.getConfig("fixedThreshold", False)
@@ -100,16 +99,10 @@ class Eval(Phase):
 
         for data in tqdm(testData):
             x, truth = data
-            # make sure to have a batchsize if batchSize == 1
-            x = x.reshape(batchSize, x.shape[0], x.shape[1])
-            truth = truth.reshape(batchSize, truth.shape[0], truth.shape[1])
 
-            prediction = model.predict(x, get_likelihood=True, returnNumpy=False)
+            prediction = model.predict(x, returnNumpy=False)
             prediction, truth = da((prediction, truth))
 
-            prediction = prediction.reshape(-1, prediction.shape[2])
-            truth = truth.reshape(-1, truth.shape[2])
-
             scorer.score(truth, prediction, trace=True)
 
         result = scorer.scoreAllRecords()
@@ -150,18 +143,19 @@ class Eval(Phase):
         testData, recordsMap = self.getTestDataAndRecordMap()
 
         with self.project:
-            self.project.config.update(self.getConfig("eventEval"))
+            self.project.config.update(self.getConfig("eventEval", {}))
 
             evalMetrics = self.getConfig("eval.metrics")
-            classNums = self.getConfig("model.multiclassificationSize", [2])
+            classNums = self.getConfig("numClasses", [2])
             labelNames = self.getConfig("classification.labelNames")
             predictionNames = self.getConfig("classification.predictionSignals")
             predictionFrequencies = self.getConfig("classification.predictionFrequencies")
             ignoreIndex = self.getConfig("classification.ignoreIndex", -1)
+            scorerTypes = [(EventScorer if sc == "event" else Scorer) for sc in self.getConfig("classification.scorerTypes")]
 
             threshold = self.getConfig("fixedThreshold", False)
             thresholds = threshold or self.getData("threshold", list)
-            mScorer = MultiScorer(classNums, evalMetrics, scorerNames=labelNames, ignoreClasses=[ignoreIndex], scorerClass=EventScorer)
+            mScorer = MultiScorer(classNums, evalMetrics, scorerNames=labelNames, ignoreClasses=[ignoreIndex], scorerClasses=scorerTypes)
             for threshold, s in zip(thresholds, mScorer.scorer.values()):
                 s.majorityVote = self.getConfig("eventEval.tpStrat", "overlap") == "majority"
                 s.noTN = self.getConfig("eventEval.tnStrat", "eventcount") == "noTN"
@@ -172,7 +166,7 @@ class Eval(Phase):
 
             metaData = pd.DataFrame(self.getTestMetadata())
             metaData.replace("M", None, inplace=True)
-            metaData["indexArousal"] = metaData["indexArousal"].astype(float)
+            # metaData["indexArousal"] = metaData["indexArousal"].astype(float)
             numClasses = self.getConfig("numClasses")
 
             model = self.getModel()
@@ -193,7 +187,7 @@ class Eval(Phase):
                     x = x.reshape([1] + list(x.shape))
                     truth = truth.reshape([1] + list(truth.shape))
 
-                prediction = model.predict(x, get_likelihood=True, returnNumpy=False)
+                prediction = model.predict(x, returnNumpy=False)
                 prediction, truth = da((prediction, truth))
 
                 # for labelIndex, labelName in enumerate(recordResults.keys()):
@@ -212,11 +206,12 @@ class Eval(Phase):
                     labelTruth = labelTruth.reshape(-1)
 
                     # score the prediction
-                    r = scorer.score(labelTruth.reshape(-1), labelPrediction.reshape(-1, numClass))
+                    r = scorer.score(labelTruth, labelPrediction)
+                    # r = scorer.score(labelTruth.reshape(-1), labelPrediction.reshape(-1, numClass))
                     resultRow.update({f"{m}_{labelName}": v for m,v in r.items()})
 
                     # add examples
-                    resultRow[f"examples_{labelName}"] = self.getExamples(labelIndex, labelTruth, labelPrediction)
+                    # resultRow[f"examples_{labelName}"] = self.getExamples(labelIndex, labelTruth, labelPrediction)
 
 
                     recordData = metaData.query(f'recordId == "{recordId}"')
@@ -250,4 +245,6 @@ class Eval(Phase):
             self.registerData("eventResults", eventResults)
 
     def main(self):
-        self.getData("eventResults", list)
+        segmentResultsRecords, segmentResults = self.getData("evalResults")
+
+        self.logSuccess(f"result segments: {segmentResults}")

{sleepyphases-0.2.0 → sleepyphases-0.3.0}/SleePyPhases/phases/EvalReport.py

@@ -10,7 +10,6 @@ from pyPhasesRecordloader import RecordSignal, Signal
 
 from SleePyPhases import Plot
 from SleePyPhases.DataManipulation import DataManipulation
-from SleePyPhases.Reporter import Reporter
 
 
 class EvalReport(Phase):
@@ -38,7 +37,7 @@ class EvalReport(Phase):
 
     def createLogFolder(self, path):
         Path(path).mkdir(parents=True, exist_ok=True)
-        Path(path).joinpath("/examples").mkdir(parents=True, exist_ok=True)
+        # Path(path).joinpath("/examples").mkdir(parents=True, exist_ok=True)
 
     def addAsset(self, name, extension="png", bbox_inches="tight", dpi=300):
         import matplotlib.pyplot as plt
@@ -178,22 +177,20 @@ class EvalReport(Phase):
         
     def main(self):
         import pandas as pd
-        from SleePyPhases.Plot import Plot
         
         threshold = self.getConfig("fixedThreshold", False)
         self.threshold = threshold or self.getData("threshold", float)
 
         modelConfigString = self.project.getDataFromName("eventResults").getTagString()
-        evalPath = f"eval/{modelConfigString}/"
 
-        Reporter.basePath  = evalPath
+        evalPath = self.getConfig("eval-path", "eval/")
+        evalPath = f"{evalPath}/{modelConfigString}/"
+
+        self.createLogFolder(evalPath)
+
         self.evalPath = evalPath
-        self.reporter = Reporter("EvalReport")
-        self.reporter.createFolder()
-        # self.reporter.save()
 
         self.copyRawData()
-        # self.modelInfo()
 
         # self.log("save report to: %s" % self.reporter.getFilePath())
 
@@ -246,8 +243,8 @@ class EvalReport(Phase):
     def copyRawData(self):
         modelConfigString = self.project.getDataFromName("modelState").getTagString()
         trainingLogPath = f"logs/{modelConfigString}/"
-        shutil.copyfile(trainingLogPath + "log.csv", Reporter.basePath + "training.log")
-        shutil.copyfile(trainingLogPath + "project.config", Reporter.basePath + "project.json")
+        shutil.copyfile(trainingLogPath + "log.csv", self.evalPath + "training.log")
+        shutil.copyfile(trainingLogPath + "project.config", self.evalPath + "project.json")
 
     def trainingPlot(self, logFile):
         import matplotlib.pyplot as plt
@@ -257,11 +254,9 @@ class EvalReport(Phase):
         df = pd.read_csv(logFile)
         plt.plot(df["epoch"], df["loss"], label="Training Loss")
         plt.plot(df["epoch"], df[self.metrics[0]], label=self.metrics[0])
-        self.reporter.addAsset("trainingsProcess")
+        self.addAsset("trainingsProcess")
 
     def modelInfo(self):
         model = self.project.getPhase("Eval").getModel()
         sum = model.summary()
-        self.reporter.addVar("modelName", self.getConfig("modelName"))
-        self.reporter.addVar("parameter", model.parameter, "humannumber")
-        self.reporter.addVar("modelSummary", sum)
+        print(sum)

{sleepyphases-0.2.0 → sleepyphases-0.3.0}/SleePyPhases/phases/ExportMetadataToMeticalDB.py

@@ -18,32 +18,10 @@ class ExportMetadataToMeticalDB(Phase):
         from SleePyPhases.recordloaders.MedicalDB import MedicalDB
 
         metadata = self.getData("metadata", pd.DataFrame)
-        relevantData = [v for v in AliceTextReportLoader.relevantRows.values() if v != ""]
-        # recordMetadataRows = []
-        # for dataRow in metadata.iloc:
-
-        #     idDict = {
-        #         "acquisition_id": dataRow["recordId"],
-        #         "patient_psn": dataRow["patient"] if "patient" in dataRow else None,
-        #         "bmi": dataRow["bmi"] if "bmi" in dataRow else None,
-        #         "domino": self.getConfig("useLoader") == "tsm-domino"
-        #     }
-        #     for col in cols:
-        #         if col in dataRow:
-        #             idDict[col] = dataRow[col]
-        #     recordMetadataRows.append(idDict)
+        relevantCols = [v for v in AliceTextReportLoader.relevantRows.values() if v != ""]
+        channelMetaData = self.getData("metadata-channels", pd.DataFrame)
 
-        
-        # channelMetadata = self.getData("metadata-channels", pd.DataFrame)
-        # features = self.getData("features", pd.DataFrame)
-
-        # for dataRow in list(channelMetadata.iloc) + list(features.iloc):
-        #     pass
-
-        # if self.exportTo == "csv":
-        #     df = pd.DataFrame.from_records(recordMetadataRows, index="acquisition_id")
-        #     df.to_csv("recordMetadata.csv")
-        # else:
+        featureData = self.getData("features", pd.DataFrame)
         MedicalDB.config = self.getConfig("medicalDB")
         medDB = MedicalDB.get()
         medDB.connect()
@@ -58,18 +36,8 @@ class ExportMetadataToMeticalDB(Phase):
         somnoemetricTable = schema.somnometric_type
         
         for rowIndex, row in tqdm(metadata.iterrows(), total=len(metadata)):
-            # df = metaData.items()
-            recordId = row["recordId"]
-            channelData = self.getData("metadata-channels", pd.DataFrame, recordId=recordId)
-            features = {}
-            try:
-                features = self.getData("features", pd.DataFrame, recordId=recordId)
-            except ChannelsNotPresent:
-                pass
-            except AnnotationNotFound:
-                pass
-            except ParseError:
-                pass
+            channelData = channelMetaData.query(f"recordId == '{rowIndex}'")
+            features = featureData.query(f"recordId == '{rowIndex}'")
 
             missingCaseCounts = {}
             def getMissingCid(pid):
@@ -143,7 +111,7 @@ class ExportMetadataToMeticalDB(Phase):
                 if isinstance(val, bool):
                     return "float", 1 if val else 0
                 
-                dtype = "float" if val.dtype == float else "string"
+                dtype = "float" if isinstance(val, float) or val.dtype == float else "string"
 
                 return dtype, val
             
@@ -166,29 +134,56 @@ class ExportMetadataToMeticalDB(Phase):
                 # where source == psg
             medDB.execute(str(q))
 
+            for metricName in relevantCols:
+                if metricName in metadata.columns:
+                    addMetricType(metricName, metadata[metricName])
+                
+            # add all record metadata
+            for column, val in row.items():
+                if column in relevantCols and not pd.isna(val) and val is not None:
+                    typestr, val = getTypeAndValue(row[column])
+                    q = PostgreSQLQuery.into(recordMetaTable)\
+                        .columns(["id_record", "id_somnometric_type", f"value_{typestr}"])\
+                        .insert([recordId, column, val])
+                    medDB.execute(str(q))
             
             # add all the metadata types (somnometrics types)
             # channelMetrics = [col for col in channelData.columns if col not in ["label", "signalName", "recordId"]]
             # for metricName in channelMetrics:
                 # addMetricType(metricName, channelData[metricName].dtype)
 
+            # delete all chanel data for the reocord
+            q = PostgreSQLQuery.from_(channelMetaTable)\
+                .using(channelTable)\
+                .where(channelMetaTable.id_channel == channelTable.id_channel)\
+                .where(channelTable.id_record == recordId)\
+                .delete()
+            medDB.execute(str(q))
+            
+            
+            q = PostgreSQLQuery.from_(channelTable)\
+                .delete()\
+                .where(channelTable.id_record == recordId)
+            
+            medDB.execute(str(q))
+
             if len(features) > 0:
                 featureMetrics = [col for col in features.columns if col not in ["channel", "recordId"]]
                 for metricName in featureMetrics:
                     addMetricType(metricName, features[metricName])
 
+            
             for channel in channelData.iloc:
                 if "signalName" not in channel:
                     continue
                 chanelDict = {
                     "name": channel["signalName"],
-                    "frequency": channel["sample_rate"], # sample_frequency
+                    "frequency": channel["sample_rate"] if "sample_rate" in channel else channel["sample_frequency"], # sample_frequency
                     "prefilter": channel["prefilter"],
                     "physical_max": channel["physical_max"],
                     "physical_min": channel["physical_min"],
                     "digital_max": channel["digital_max"],
                     "digital_min": channel["digital_min"],
-                    "prefilter": channel["prefilter"],
                     "transducer": channel["transducer"],
                     "type": channel["type"],
                     "id_record": recordId
@@ -197,7 +192,7 @@ class ExportMetadataToMeticalDB(Phase):
                 q = PostgreSQLQuery.into(channelTable)\
                     .columns(*list(chanelDict.keys()))\
                     .insert(*list(chanelDict.values()))
-                channelId = medDB.execute(str(q))
+                channelId = medDB.executeAndFetchOne(f"{q} RETURNING id_channel")[0]
 
 
                 # add channel specific features
@@ -232,37 +227,32 @@ class ExportMetadataToMeticalDB(Phase):
                 # medDB.execute(str(q))
             
 
-            q = PostgreSQLQuery.from_(channelTable)\
-                .delete()\
-                .where(channelTable.id_record == recordId)
-            medDB.execute(str(q))
-
-            if "channels" not in row:
-                continue
-
-            for channel in row["channels"]:
-                channelName = channel["signalName"]
-
-                chanelDict = {
-                    "name": channelName,
-                    "frequency": channel["sample_rate"], # sample_frequency
-                    "prefilter": channel["prefilter"],
-                    "physical_max": channel["physical_max"],
-                    "physical_min": channel["physical_min"],
-                    "digital_max": channel["digital_max"],
-                    "digital_min": channel["digital_min"],
-                    "prefilter": channel["prefilter"],
-                    "transducer": channel["transducer"],
-                    "type": channel["type"],
-                    "id_record": recordId
-                }
-                q = PostgreSQLQuery.into(channelTable)\
-                    .columns(*list(chanelDict.keys()))\
-                    .insert(*list(chanelDict.values()))
-                medDB.execute(str(q))
-
-            if rowIndex % 25 == 0:
-                medDB.commit()
+            # if "channels" not in row:
+            #     continue
+
+            # for channel in row["channels"]:
+            #     channelName = channel["signalName"]
+
+            #     chanelDict = {
+            #         "name": channelName,
+            #         "frequency": channel["sample_rate"], # sample_frequency
+            #         "prefilter": channel["prefilter"],
+            #         "physical_max": channel["physical_max"],
+            #         "physical_min": channel["physical_min"],
+            #         "digital_max": channel["digital_max"],
+            #         "digital_min": channel["digital_min"],
+            #         "prefilter": channel["prefilter"],
+            #         "transducer": channel["transducer"],
+            #         "type": channel["type"],
+            #         "id_record": recordId
+            #     }
+            #     q = PostgreSQLQuery.into(channelTable)\
+            #         .columns(*list(chanelDict.keys()))\
+            #         .insert(*list(chanelDict.values()))
+            #     medDB.execute(str(q))
+
+            # if rowIndex % 25 == 0:
+            medDB.commit()
             # table = schema.acq_metadata
 
             # checkQuery = PostgreSQLQuery.from_(table).select(1).where(table.acquisition_id == metaData['acquisition_id'])

{sleepyphases-0.2.0 → sleepyphases-0.3.0}/SleePyPhases/phases/Extract.py

@@ -1,3 +1,4 @@
+from pathlib import Path
 import numpy as np
 from pyPhases import Phase
 from pyPhases.util import BatchProgress
@@ -228,25 +229,33 @@ class Extract(Phase):
         exporterSignals.options["dtype"] = self.getConfig("preprocessing.dtype", "float32")
         exporterFeatures.options["dtype"] = self.getConfig("preprocessing.dtype", "float32")
 
+        removedRecordIds = []
+        removed_records_path = exporterSignals.getPath(f"{dataIdSignals}-removed-records-tmp.npy")
+        
         if force is False:
             if exporterSignals.exists(dataIdSignals) and exporterFeatures.exists(dataIdFeatures):
-                self.logSuccess("Datafiles allready exist")
-                return
+                self.logSuccess("Datafiles already exist")
+                return []
 
             if exporterSignals.existsTmp(dataIdSignals) and exporterFeatures.existsTmp(dataIdFeatures):
                 exporterSignals.loadTmp(dataIdSignals)
                 exporterFeatures.loadTmp(dataIdFeatures)
+                
+                if Path(removed_records_path).exists():
+                    removedRecordIds = np.load(removed_records_path, allow_pickle=True).tolist()
+
                 recordsLoaded = len(exporterFeatures.currentArrayShapes[dataIdSignals][0])
-                records = records[recordsLoaded:]
-                self.logSuccess("Resuming extraction: %i records allready loaded / %i to go" % (recordsLoaded, len(records)))
+                total_processed = recordsLoaded + len(removedRecordIds)
+                
+                # Skip both successfully processed and removed records
+                records = records[total_processed:]
+                self.logSuccess(f"Resuming extraction: {recordsLoaded} records already loaded, {len(removedRecordIds)} records skipped, {len(records)} to go")
 
         bp = BatchProgress(records)
         bp.useMultiThreading = self.useMultiThreading
         if self.threads is not None:
             bp.threads = self.threads
 
-        removedRecordIds = []
-
         def combine(recordList, dataIndex):
             recordList = recordList.copy()
             # append the the data and the shape to a tmp file
@@ -254,7 +263,10 @@ class Extract(Phase):
             # check if all records where loaded, else extend the removedRecord list
             if None in recordList:
                 rm = [i + dataIndex for i, r in enumerate(recordList) if r is None]
-                [removedRecordIds.append(records[r]) for r in rm]
+                new_removed = [records[r] for r in rm]
+                removedRecordIds.extend(new_removed)
+                np.save(removed_records_path, removedRecordIds)
+                
                 recordList = [r for r in recordList if r is not None]
                 self.logWarning("Skipped %i records: channels/annotation not present" % len(rm))
 

{sleepyphases-0.2.0 → sleepyphases-0.3.0}/SleePyPhases/phases/ExtractFeatures.py

@@ -174,6 +174,8 @@ class ExtractFeatures(Phase):
             recordSignal, eventlist = recordLoader.loadRecord(recordId)
         except AnnotationNotFound as e:
             self.logError(f"Annotation missing in {recordId}: {e.name} ... skipping record")
+        except ChannelsNotPresent as e:
+            self.logError(f"Channels missing in {recordId}: {e.channels} ... skipping record")
         except ParseError as e:
             self.logError(f"ParseError in record {recordId}, record will be skipped: {e}")
 
@@ -203,21 +205,19 @@ class ExtractFeatures(Phase):
         return self.features.loc[idx]
 
     def main(self):
+        import pandas as pd
         self.features = None
-        groupedRecords = self.project.getData("allDBRecordIds", list)
+        metadata = self.project.getData("metadata", pd.DataFrame)
+        recordIds = metadata["recordId"]
+        forceCalculation = self.getConfig("featureExtraction.features", True)
 
         reqFeatures = {f["name"]: True for f in self.getConfig("featureExtraction.features", [])}
 
-        # groupedRecords = {"pid_202436073": "acq_237478592"}
-        for patient, recordIds in tqdm(groupedRecords.items()):
-            for recordId in recordIds:
-                # tryToGenerate has no meaning, it is just known to be not in the metadata, so the record will always be recalculated
-                # self.getData("features", pd.DataFrame, recordId=recordId, tryToGenerate=True, **reqFeatures)
-                rows = self.getData("features", pd.DataFrame, recordId=recordId, **reqFeatures)
-                if len(rows) == 0:
-                    self.logError(f"no features for {recordId}")
+        for recordId in tqdm(recordIds.iloc):
+            if forceCalculation:
+                # the parameter forced has no meaning, it is just known to be not in the metadata, so the record will always be recalculated
+                reqFeatures["forced"] = True # should never be found
+            rows = self.getData("features", pd.DataFrame, recordId=recordId, **reqFeatures)
+            if len(rows) == 0:
+                self.logError(f"no features for {recordId}")
                 
-
-        # get a single feature:
-        # feat = {"swa": True}
-        # x = self.getData("features", pd.DataFrame, recordId="shhs1-200002", channel="EEG", **feat)   

{sleepyphases-0.2.0 → sleepyphases-0.3.0}/SleePyPhases/phases/Setup.py

@@ -4,7 +4,6 @@ from pyPhases import Phase
 from pyPhasesML import DataversionManager, Scorer
 from pyPhasesRecordloader import RecordLoader
 
-from SleePyPhases.recordloaders.MedicalDB import MedicalDB
 
 
 class Setup(Phase):
@@ -17,21 +16,6 @@ class Setup(Phase):
         
         defaultValidationbatchManipulation = self.getConfig("segmentManipulationEval", self.getConfig("segmentManipulation"))
         self.setConfig("segmentManipulationEval", defaultValidationbatchManipulation)
-        
-        # alice loader
-        RecordLoader.registerRecordLoader("RecordLoaderTest", "SleePyPhases.recordloaders")
-        RecordLoader.registerRecordLoader("RecordLoaderTSM", "SleePyPhases.recordloaders")
-        tsmPath = self.getConfig("tsm-path", False)
-        if tsmPath is not False:
-            tsmPath = Path(tsmPath)
-            self.project.setConfig("loader.tsm.filePath", tsmPath.as_posix())
-            self.project.setConfig("loader.tsm.dataset.downloader.basePath", tsmPath.as_posix())
-
-        tsmPath = self.getConfig("tsm-domino-path", False)
-        if tsmPath is not False:
-            tsmPath = Path(tsmPath)
-            self.project.setConfig("loader.tsm-domino.filePath", tsmPath.as_posix())
-            self.project.setConfig("loader.tsm-domino.dataset.downloader.basePath", tsmPath.as_posix())
 
         preprocessingConfig = self.getConfig("preprocessing")
         if "combineChannels" in preprocessingConfig:
@@ -62,9 +46,6 @@ class Setup(Phase):
             return kappa
         Scorer.registerMetric("reduceAndKappa", reduceKappa, combineReduceKappa)
         
-        MedicalDB.config = self.getConfig("medicalDB", {})
-
-            
     def getDataVersionManager(self):
         groupedRecords = self.project.getData("allDBRecordIds", list)
         seed = self.getConfig("dataversion.seed", None)

{sleepyphases-0.2.0 → sleepyphases-0.3.0}/SleePyPhases/phases/TestRun.py

@@ -48,6 +48,8 @@ class TestRun(Phase):
             self.log(f"Creating a new Data Folder ({testRunData.as_posix()}) for the testrun")
             if not testRunData.exists():
                 testRunData.mkdir()
+            self.setConfig("data-path", testRunData)
+            self.project.prepareAllPhases()
             # else:
             #     # remove all files from the testrun folder
             #     for f in testRunData.iterdir():
@@ -121,6 +123,8 @@ class TestRun(Phase):
             self.log(f"Test Model: {self.getConfig('modelName')}")
 
             model = ModelManager.getModel()
+            model.config.logPath = testRunData / "logs"
+
             self.log(f"Current Model: {type(model)}")
             self.log(model.summary())
             train = self.project.generateData("dataset-training")

{sleepyphases-0.2.0 → sleepyphases-0.3.0}/SleePyPhases/phases/ThresholdOptimisation.py

@@ -4,6 +4,7 @@ from pyPhasesML import Model, ModelManager
 from tqdm import tqdm
 
 from SleePyPhases.DataManipulation import DataManipulation
+from pyPhasesML import Scorer
 from SleePyPhases.EventScorer import EventScorer
 from SleePyPhases.MultiScorer import MultiScorer
 
@@ -35,12 +36,8 @@ class ThresholdOptimisation(Phase):
             x, t = validationData[i]
             p = model.predict(x, get_likelihood=True, returnNumpy=False)
             p, t = da((p, t))
-            prediction.append(p)
-            truth.append(t)
-
-        # we assume output is reduces to bachsize 1 during augmentation (1, X, channels)
-        prediction = np.concatenate(prediction, axis=1)
-        truth = np.concatenate(truth, axis=1)
+            [prediction.append(x) for x in p]
+            [truth.append(x) for x in t]
 
         return prediction, truth
 
@@ -54,8 +51,10 @@ class ThresholdOptimisation(Phase):
         classNums = self.getConfig("model.multiclassificationSize", [2])
         labelNames = self.getConfig("classification.labelNames")
         ignoreIndex = self.getConfig("classification.ignoreIndex", -1)
+        optimizeThresholdFor = self.getConfig("optimizeThresholdFor", None)
+        scorerTypes = [(EventScorer if sc == "event" else Scorer) for sc in self.getConfig("classification.scorerTypes")]
 
-        scorer = MultiScorer(classNums, thresholdMetrics, scorerNames=labelNames, ignoreClasses=[ignoreIndex], scorerClass=EventScorer)
+        scorer = MultiScorer(classNums, thresholdMetrics, scorerNames=labelNames, ignoreClasses=[ignoreIndex], scorerClasses=scorerTypes)
         for s in scorer.scorer.values():
             s.majorityVote = self.getConfig("eventEval.tpStrat", "overlap") == "majority"
             s.noTN = self.getConfig("eventEval.tnStrat", "eventcount") == "noTN"
@@ -88,6 +87,9 @@ class ThresholdOptimisation(Phase):
         
         thresholds = []
         for scorerIndex, (scorerName, thresholdMetric) in enumerate(zip(labelNames, thresholdMetrics)):
+            if optimizeThresholdFor is not None and scorerName not in optimizeThresholdFor:
+                thresholds.append(None)
+                continue
             self.log(f"Optimize threshold for {scorerName} {thresholdMetric}")
             t = scipy.optimize.fmin(score, args=(prediction, truth, scorerIndex, scorerName, thresholdMetric), x0=0.3, xtol=0.01, ftol=0.01)
             self.logSuccess(f"Optimized threshold for {scorerName} {thresholdMetric}: {t}")

{sleepyphases-0.2.0 → sleepyphases-0.3.0}/SleePyPhases.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
-Metadata-Version: 2.2
+Metadata-Version: 2.4
 Name: SleePyPhases
-Version: 0.2.0
+Version: 0.3.0
 Summary: A framwork for creating deep learning pipelines for sleep data
 Home-page: https://gitlab.com/sleep-is-all-you-need/sleepyphases
 Author: Franz Ehrlich
@@ -20,6 +20,7 @@ Dynamic: classifier
 Dynamic: description
 Dynamic: description-content-type
 Dynamic: home-page
+Dynamic: license-file
 Dynamic: requires-dist
 Dynamic: requires-python
 Dynamic: summary

{sleepyphases-0.2.0 → sleepyphases-0.3.0}/SleePyPhases.egg-info/SOURCES.txt

@@ -40,9 +40,6 @@ SleePyPhases/phases/Training.py
 SleePyPhases/phases/Validation.py
 SleePyPhases/phases/VisualizeConfig.py
 SleePyPhases/phases/__init__.py
-SleePyPhases/recordloaders/MedicalDB.py
-SleePyPhases/recordloaders/RecordLoaderTSM.py
-SleePyPhases/recordloaders/__init__.py
 tests/__init__.py
 tests/conftest.py
 tests/unit/__init__.py

{sleepyphases-0.2.0 → sleepyphases-0.3.0}/setup.py

@@ -5,7 +5,7 @@ with open("README.md", "r") as fh:
 
 setuptools.setup(
     name="SleePyPhases",
-    version="v0.2.0"[1:],
+    version="v0.3.0"[1:],
     author="Franz Ehrlich",
     author_email="fehrlichd@gmail.com",
     description="A framwork for creating deep learning pipelines for sleep data",

sleepyphases-0.2.0/SleePyPhases/recordloaders/MedicalDB.py

@@ -1,124 +0,0 @@
-import sys
-
-from pyPhases.util.Logger import classLogger
-from pypika import Table, PostgreSQLQuery as Query
-from pypika.queries import QueryBuilder
-
-
-@classLogger
-class MedicalDB:
-    connection = None
-    client = None
-    assocCursor = None
-    _instance = None
-
-    config = {
-        "user": "root",
-        "password": "example",
-        "host": "localhost",
-        "port": 5432,
-        "database": "data",
-    }
-
-    @classmethod
-    def get(cls) -> "MedicalDB":
-        if cls._instance is None:
-            cls._instance = cls.__new__(cls)
-            cls._instance.connect()
-        return cls._instance
-
-    def connect(self):
-
-        import psycopg2
-
-        try:
-            c = MedicalDB.config
-            conn = psycopg2.connect(
-                "host=%s port=%i dbname=%s user=%s password=%s"
-                % (
-                    c["host"],
-                    c["port"],
-                    c["database"],
-                    c["user"],
-                    c["password"],
-                )
-            )
-
-        except psycopg2.Error as e:
-            self.logError(f"Error connecting to Postgres Platform: {e}")
-            sys.exit(1)
-        self.conn = conn
-        self.cursor = conn.cursor()
-
-    def execute(self, sql, **vars):
-        if isinstance(sql, QueryBuilder):
-            sql = sql.get_sql()
-        return self.cursor.execute(sql, **vars)
-
-    def executeAndFetch(self, sql, **vars):
-        self.execute(sql, **vars)
-        return self.cursor.fetchall()
-
-    def commit(self):
-        self.conn.commit()
-    
-    def executeAndFetchOne(self, sql, **vars):
-        self.execute(sql, **vars)
-        return self.cursor.fetchone()
-
-    def close(self):
-        if self.connection is not None:
-            self.connection.close()
-
-    def fillRecordId(self, id):
-        return id
-
-    def getRecordData(self, recordId):
-        record = Table("record", schema=self.config["schema"])
-        patient = Table("patient", schema=self.config["schema"])
-        medCase = Table("medical_case", schema=self.config["schema"])
-        q = (
-            Query.from_(record)
-            .select(patient.id_patient, record.id_medical_case, record.start, patient.date_of_birth, patient.gender)
-            .join(medCase).on(record.id_medical_case == medCase.id_medical_case)
-            .join(patient).on(patient.id_patient == medCase.id_patient)
-            .where(record.id_record == recordId)
-        )
-        row = self.executeAndFetchOne(q)
-
-        if row is None:
-            return None
-        
-        row = list(row)
-        age = None if row[2] or row[3] is None else row[2] - row[3]
-        row[3] = age
-
-        return row
-
-    def getDiagnoses(self, recordId):
-        diag = Table("diagnosis", schema=self.config["schema"])
-        record = Table("record", schema=self.config["schema"])
-        medCase = Table("medical_case", schema=self.config["schema"])
-        q = Query.from_(diag)\
-            .select("diagnosis_category_id", "diagnosis")\
-            .join(medCase).on(diag.id_medical_case == medCase.id_medical_case)\
-            .join(record).on(medCase.id_medical_case == record.id_medical_case)\
-            .where(record.id_record == recordId)\
-            .get_sql()
-        
-        diagnoses = self.executeAndFetch(q)
-
-        diagnoseCats = {d[0]: d[1] for d in diagnoses}
-        return diagnoseCats
-
-    def getPSGType(self, recordId):
-
-        record = Table("record", schema=self.config["schema"])
-        q = (
-            Query.from_(record)
-            .select("psg_config")
-            .where(record.id_record == recordId)
-        )
-        r = self.executeAndFetchOne(q)
-
-        return r[0] if bool(r) else ""

sleepyphases-0.2.0/SleePyPhases/recordloaders/RecordLoaderTSM.py

@@ -1,34 +0,0 @@
-from SleePyPhases.recordloaders.MedicalDB import MedicalDB
-from SleepHarmonizer import RecordLoaderAlice
-
-
-class RecordLoaderTSM(RecordLoaderAlice):
-    config = {}
-
-    def getMetaData(self, recordName):
-        metadata = super().getMetaData(recordName)
-        del metadata["patientName"]
-        del metadata["patientCode"]
-        del metadata["patientAdd"]
-        del metadata["technician"]
-
-        db = MedicalDB.get()
-        row = db.getRecordData(recordName)
-        diagnoses = db.getDiagnoses(recordName)
-        metadata["psg_type"] = db.getPSGType(recordName)
-
-        if row is not None:
-            patient_id, case_id, recordDate, patientAge, gender = row
-
-            metadata["patient"] = patient_id
-            metadata["case"] = case_id
-            metadata["age"] = patientAge
-            metadata["gender"] = gender
-            metadata["startRecord"] = recordDate
-
-        if diagnoses is not None:
-            metadata.update(diagnoses)
-        
-        db.close()
-
-        return metadata