SleePyPhases 0.2.0__tar.gz → 0.2.1__tar.gz

{sleepyphases-0.2.0 → sleepyphases-0.2.1}/PKG-INFO

@@ -1,6 +1,6 @@
-Metadata-Version: 2.2
+Metadata-Version: 2.4
 Name: SleePyPhases
-Version: 0.2.0
+Version: 0.2.1
 Summary: A framwork for creating deep learning pipelines for sleep data
 Home-page: https://gitlab.com/sleep-is-all-you-need/sleepyphases
 Author: Franz Ehrlich
@@ -20,6 +20,7 @@ Dynamic: classifier
 Dynamic: description
 Dynamic: description-content-type
 Dynamic: home-page
+Dynamic: license-file
 Dynamic: requires-dist
 Dynamic: requires-python
 Dynamic: summary

{sleepyphases-0.2.0 → sleepyphases-0.2.1}/SleePyPhases/DataManipulation.py

@@ -14,8 +14,8 @@ class DataManipulation(pyPhasesDataManipulation, Swappable):
     def __init__(self, steps, splitName, projectconfig, seed=2, threshold=None, recordMetadata=None, **kwargs) -> None:
         super().__init__(steps, splitName, **kwargs)
         self.catMatrix = None
-        self.numLabels = projectconfig["numLabels"]
-        self.numClasses = projectconfig["numClasses"]
+        self.numLabels = projectconfig["numLabels"] if "numLabels" in projectconfig else 1
+        self.numClasses = projectconfig["numClasses"] if "numLabels" in projectconfig else [1]
         self.rng = default_rng(seed=2)
         self.threshold = threshold
         self.recordMetadata = recordMetadata

{sleepyphases-0.2.0 → sleepyphases-0.2.1}/SleePyPhases/RecordFeatureExtraction.py

@@ -25,6 +25,10 @@ class RecordFeatureExtraction(pyPhasesFeatureExtraction):
         
     def updateMetadataByStep(self, stepname: str, recordSignal: RecordSignal, eventlist: List[Event], metadata: pd.DataFrame, channel=None, **options):
         features = self.step(stepname, recordSignal, eventlist, channel, **options)
+
+        if features is None:
+            return None
+        
         recordId = recordSignal.recordId
         # update row with index recordId and channel with features dict or create new if not exist
         
@@ -252,15 +256,83 @@ class RecordFeatureExtraction(pyPhasesFeatureExtraction):
         ret = {
             "cycle-count": len(cycles["start"]),
         }
+        frequencyFactor = 30
         
-        for cycleNumber, (start, stop) in enumerate(zip(cycles["start"], cycles["end"])):
-            if lastStop > 0:
-                remTime = sum([e.duration for e in remEvents if e.start >= lastStop and e.start < start])
-                ret[f"cycle-{cycleNumber}-rem-duration"] = remTime
+        for cycleNumber, (start, stop) in enumerate(zip(cycles["start"], cycles["stop"])):
+            start *= frequencyFactor
+            stop *= frequencyFactor
+            # if lastStop > 0:
+            remTime = sum([e.duration for e in remEvents if e.start >= lastStop and e.start < start])
+            ret[f"cycle-{cycleNumber}-rem-duration"] = remTime
 
             lastStop = stop
 
+        # remTime = sum([e.duration for e in remEvents if e.start >= lastStop and e.start < start])
+        # ret[f"cycle-{cycleNumber}-rem-duration"] = remTime
+
+        return ret
+    
+    def spindles(self, segmentSignal: RecordSignal, eventlist: List[Event], channel: str):
+        from .external.sumo.sumo.scripts.a7.butter_filter import butter_bandpass_filter, downsample
+        # from .external.sumo.a7.detect_spindles import detect_spindle
+        from .external.sumo.sumo.scripts.predict_plain_data import SimpleDataset
+        # from .external.sumo.sumo.predict_plain_data import A7
+        import pytorch_lightning as pl
+        from torch.utils.data import Dataset, DataLoader
+        import torch
+        from .external.sumo.sumo.model.sumo import SUMO
+        from .external.sumo.sumo.config.config import Config
+        # from .external.sumo.model.data import spindle_vect_to_indices
+
+        config = Config('predict', create_dirs=False)
+
+        def get_model(p):
+            # p = Path(p)
+
+            model_checkpoint = torch.load(p)
+
+            model = SUMO(config)
+            model.load_state_dict(model_checkpoint['state_dict'])
+
+            return model
+
+        eegSignal = segmentSignal.getSignalByName(channel)
+        resample_rate = 100
+        model_path = "SleePyPhases/external/sumo/sumo/output/final.ckpt"
+        
+        if len(eegSignal.signal) == 0:
+            return None
+
+        eegs = []
+        eegSignal = downsample(butter_bandpass_filter(eegSignal.signal, 0.3, 30.0, eegSignal.frequency, 10), eegSignal.frequency, resample_rate)
+        eegs.append(eegSignal)
+
+        dataset = SimpleDataset(eegs)
+        dataloader = DataLoader(dataset)
+        model = get_model(model_path)
+        trainer = pl.Trainer(num_sanity_val_steps=0, logger=False)
+        predictions = trainer.predict(model, dataloader)
+        pred = predictions[0]
+
+        # spindles_a7 = A7(x, resample_rate)
+        spindle_vect = pred[0].numpy()
+        
+        diff = np.diff(np.r_[0, spindle_vect, 0])  # be able to detect spindles at the start and end of vector
+        spindles = np.c_[np.argwhere(diff == 1), np.argwhere(diff == -1)]  / resample_rate
+        # spindles = spindle_vect_to_indices(spindle_vect) / resample_rate
+
+
+        # frequencies, psd = signal.welch(eegs[0][s.start:s.end()], 
+        #                                 fs=100,
+        #                                 nperseg=min(256, s.duration),
+        #                                 scaling='spectrum')
+        # dominant_freq = frequencies[np.argmax(psd)]
+
+        spindles = [Event("spindle", s[0], s[1] - s[0]) for s in spindles]
+
+        
         return {
-            "cycle-count": len(cycles["start"]),
+            "spindles-count": len(spindles),
+            "spindles-mean-druation": np.array([s.duration for s in spindles]).mean()
         }
-    
+    

{sleepyphases-0.2.0 → sleepyphases-0.2.1}/SleePyPhases/SleepMetaData.py

@@ -76,6 +76,7 @@ class SleepMetaData:
                 self.fromSleepSignalBin(signal, frequency=frequency)
             case 'Sleep4':
                 self.fromSleepSignalBin(signal, frequency=frequency)
+                self.fromSleepSignalFour(signal, frequency=frequency)
 
     def fromSleepSignalBin(self, signal, frequency = 1):
         # all time values in seconds
@@ -100,6 +101,11 @@ class SleepMetaData:
         self.registerMetadata("waso", trt-sLatency-tst, unit="s")
         self.registerMetadata("sEfficiency", tst / trt * 100, unit="%")
 
+    def fromSleepSignalFour(self, signal, frequency = 1):
+        whereREM = np.where(signal == 1)
+        rLatency = whereREM[0] - self.getMetaData("sLatency")
+        self.registerMetadata("rLatency", rLatency)
+
     
     def fromApneaSignal(self, signal, classification=None):
         # classification = classification or ["None", "obstructive", "mixed", "central", "hypopnea"]

{sleepyphases-0.2.0 → sleepyphases-0.2.1}/SleePyPhases/phases/Eval.py

@@ -86,8 +86,7 @@ class Eval(Phase):
         ignoreIndex = self.getConfig("classification.ignoreIndex", -1)
         metrics = self.getConfig("trainingParameter.validationMetrics")
 
-
-        scorer = MultiScorer(classNums, metrics, scorerNames=labelNames, ignoreClasses=[ignoreIndex], scorerClass=EventScorer)
+        scorer = MultiScorer(classNums, metrics, scorerNames=labelNames, ignoreClasses=[ignoreIndex])
         for s in scorer.scorer.values():
             s.trace = True
         threshold = self.getConfig("fixedThreshold", False)
@@ -107,8 +106,8 @@ class Eval(Phase):
             prediction = model.predict(x, get_likelihood=True, returnNumpy=False)
             prediction, truth = da((prediction, truth))
 
-            prediction = prediction.reshape(-1, prediction.shape[2])
-            truth = truth.reshape(-1, truth.shape[2])
+            # prediction = prediction.reshape(-1, prediction.shape[2])
+            # truth = truth.reshape(-1, truth.shape[2])
 
             scorer.score(truth, prediction, trace=True)
 
@@ -150,7 +149,7 @@ class Eval(Phase):
         testData, recordsMap = self.getTestDataAndRecordMap()
 
         with self.project:
-            self.project.config.update(self.getConfig("eventEval"))
+            self.project.config.update(self.getConfig("eventEval", {}))
 
             evalMetrics = self.getConfig("eval.metrics")
             classNums = self.getConfig("model.multiclassificationSize", [2])
@@ -158,10 +157,11 @@ class Eval(Phase):
             predictionNames = self.getConfig("classification.predictionSignals")
             predictionFrequencies = self.getConfig("classification.predictionFrequencies")
             ignoreIndex = self.getConfig("classification.ignoreIndex", -1)
+            scorerTypes = [(EventScorer if sc == "event" else Scorer) for sc in self.getConfig("classification.scorerTypes")]
 
             threshold = self.getConfig("fixedThreshold", False)
             thresholds = threshold or self.getData("threshold", list)
-            mScorer = MultiScorer(classNums, evalMetrics, scorerNames=labelNames, ignoreClasses=[ignoreIndex], scorerClass=EventScorer)
+            mScorer = MultiScorer(classNums, evalMetrics, scorerNames=labelNames, ignoreClasses=[ignoreIndex], scorerClasses=scorerTypes)
             for threshold, s in zip(thresholds, mScorer.scorer.values()):
                 s.majorityVote = self.getConfig("eventEval.tpStrat", "overlap") == "majority"
                 s.noTN = self.getConfig("eventEval.tnStrat", "eventcount") == "noTN"
@@ -172,7 +172,7 @@ class Eval(Phase):
 
             metaData = pd.DataFrame(self.getTestMetadata())
             metaData.replace("M", None, inplace=True)
-            metaData["indexArousal"] = metaData["indexArousal"].astype(float)
+            # metaData["indexArousal"] = metaData["indexArousal"].astype(float)
             numClasses = self.getConfig("numClasses")
 
             model = self.getModel()
@@ -212,11 +212,12 @@ class Eval(Phase):
                     labelTruth = labelTruth.reshape(-1)
 
                     # score the prediction
-                    r = scorer.score(labelTruth.reshape(-1), labelPrediction.reshape(-1, numClass))
+                    r = scorer.score(labelTruth, labelPrediction)
+                    # r = scorer.score(labelTruth.reshape(-1), labelPrediction.reshape(-1, numClass))
                     resultRow.update({f"{m}_{labelName}": v for m,v in r.items()})
 
                     # add examples
-                    resultRow[f"examples_{labelName}"] = self.getExamples(labelIndex, labelTruth, labelPrediction)
+                    # resultRow[f"examples_{labelName}"] = self.getExamples(labelIndex, labelTruth, labelPrediction)
 
 
                     recordData = metaData.query(f'recordId == "{recordId}"')
@@ -250,4 +251,6 @@ class Eval(Phase):
             self.registerData("eventResults", eventResults)
 
     def main(self):
-        self.getData("eventResults", list)
+        segmentResultsRecords, segmentResults = self.getData("evalResults")
+
+        self.logSuccess(f"result segments: {segmentResults}")

{sleepyphases-0.2.0 → sleepyphases-0.2.1}/SleePyPhases/phases/EvalReport.py

@@ -38,7 +38,7 @@ class EvalReport(Phase):
 
     def createLogFolder(self, path):
         Path(path).mkdir(parents=True, exist_ok=True)
-        Path(path).joinpath("/examples").mkdir(parents=True, exist_ok=True)
+        # Path(path).joinpath("/examples").mkdir(parents=True, exist_ok=True)
 
     def addAsset(self, name, extension="png", bbox_inches="tight", dpi=300):
         import matplotlib.pyplot as plt
@@ -178,22 +178,20 @@ class EvalReport(Phase):
         
     def main(self):
         import pandas as pd
-        from SleePyPhases.Plot import Plot
         
         threshold = self.getConfig("fixedThreshold", False)
         self.threshold = threshold or self.getData("threshold", float)
 
         modelConfigString = self.project.getDataFromName("eventResults").getTagString()
-        evalPath = f"eval/{modelConfigString}/"
 
-        Reporter.basePath  = evalPath
+        evalPath = self.getConfig("eval-path", "eval/")
+        evalPath = f"{evalPath}/{modelConfigString}/"
+
+        self.createLogFolder(evalPath)
+
         self.evalPath = evalPath
-        self.reporter = Reporter("EvalReport")
-        self.reporter.createFolder()
-        # self.reporter.save()
 
         self.copyRawData()
-        # self.modelInfo()
 
         # self.log("save report to: %s" % self.reporter.getFilePath())
 
@@ -257,11 +255,9 @@ class EvalReport(Phase):
         df = pd.read_csv(logFile)
         plt.plot(df["epoch"], df["loss"], label="Training Loss")
         plt.plot(df["epoch"], df[self.metrics[0]], label=self.metrics[0])
-        self.reporter.addAsset("trainingsProcess")
+        self.addAsset("trainingsProcess")
 
     def modelInfo(self):
         model = self.project.getPhase("Eval").getModel()
         sum = model.summary()
-        self.reporter.addVar("modelName", self.getConfig("modelName"))
-        self.reporter.addVar("parameter", model.parameter, "humannumber")
-        self.reporter.addVar("modelSummary", sum)
+        print(sum)

{sleepyphases-0.2.0 → sleepyphases-0.2.1}/SleePyPhases/phases/ExportMetadataToMeticalDB.py

@@ -18,32 +18,10 @@ class ExportMetadataToMeticalDB(Phase):
         from SleePyPhases.recordloaders.MedicalDB import MedicalDB
 
         metadata = self.getData("metadata", pd.DataFrame)
-        relevantData = [v for v in AliceTextReportLoader.relevantRows.values() if v != ""]
-        # recordMetadataRows = []
-        # for dataRow in metadata.iloc:
-
-        #     idDict = {
-        #         "acquisition_id": dataRow["recordId"],
-        #         "patient_psn": dataRow["patient"] if "patient" in dataRow else None,
-        #         "bmi": dataRow["bmi"] if "bmi" in dataRow else None,
-        #         "domino": self.getConfig("useLoader") == "tsm-domino"
-        #     }
-        #     for col in cols:
-        #         if col in dataRow:
-        #             idDict[col] = dataRow[col]
-        #     recordMetadataRows.append(idDict)
+        relevantCols = [v for v in AliceTextReportLoader.relevantRows.values() if v != ""]
+        channelMetaData = self.getData("metadata-channels", pd.DataFrame)
 
-        
-        # channelMetadata = self.getData("metadata-channels", pd.DataFrame)
-        # features = self.getData("features", pd.DataFrame)
-
-        # for dataRow in list(channelMetadata.iloc) + list(features.iloc):
-        #     pass
-
-        # if self.exportTo == "csv":
-        #     df = pd.DataFrame.from_records(recordMetadataRows, index="acquisition_id")
-        #     df.to_csv("recordMetadata.csv")
-        # else:
+        featureData = self.getData("features", pd.DataFrame)
         MedicalDB.config = self.getConfig("medicalDB")
         medDB = MedicalDB.get()
         medDB.connect()
@@ -58,18 +36,8 @@ class ExportMetadataToMeticalDB(Phase):
         somnoemetricTable = schema.somnometric_type
         
         for rowIndex, row in tqdm(metadata.iterrows(), total=len(metadata)):
-            # df = metaData.items()
-            recordId = row["recordId"]
-            channelData = self.getData("metadata-channels", pd.DataFrame, recordId=recordId)
-            features = {}
-            try:
-                features = self.getData("features", pd.DataFrame, recordId=recordId)
-            except ChannelsNotPresent:
-                pass
-            except AnnotationNotFound:
-                pass
-            except ParseError:
-                pass
+            channelData = channelMetaData.query(f"recordId == '{rowIndex}'")
+            features = featureData.query(f"recordId == '{rowIndex}'")
 
             missingCaseCounts = {}
             def getMissingCid(pid):
@@ -143,7 +111,7 @@ class ExportMetadataToMeticalDB(Phase):
                 if isinstance(val, bool):
                     return "float", 1 if val else 0
                 
-                dtype = "float" if val.dtype == float else "string"
+                dtype = "float" if isinstance(val, float) or val.dtype == float else "string"
 
                 return dtype, val
             
@@ -166,29 +134,56 @@ class ExportMetadataToMeticalDB(Phase):
                 # where source == psg
             medDB.execute(str(q))
 
+            for metricName in relevantCols:
+                if metricName in metadata.columns:
+                    addMetricType(metricName, metadata[metricName])
+                
+            # add all record metadata
+            for column, val in row.items():
+                if column in relevantCols and not pd.isna(val) and val is not None:
+                    typestr, val = getTypeAndValue(row[column])
+                    q = PostgreSQLQuery.into(recordMetaTable)\
+                        .columns(["id_record", "id_somnometric_type", f"value_{typestr}"])\
+                        .insert([recordId, column, val])
+                    medDB.execute(str(q))
             
             # add all the metadata types (somnometrics types)
             # channelMetrics = [col for col in channelData.columns if col not in ["label", "signalName", "recordId"]]
             # for metricName in channelMetrics:
                 # addMetricType(metricName, channelData[metricName].dtype)
 
+            # delete all chanel data for the reocord
+            q = PostgreSQLQuery.from_(channelMetaTable)\
+                .using(channelTable)\
+                .where(channelMetaTable.id_channel == channelTable.id_channel)\
+                .where(channelTable.id_record == recordId)\
+                .delete()
+            medDB.execute(str(q))
+            
+            
+            q = PostgreSQLQuery.from_(channelTable)\
+                .delete()\
+                .where(channelTable.id_record == recordId)
+            
+            medDB.execute(str(q))
+
             if len(features) > 0:
                 featureMetrics = [col for col in features.columns if col not in ["channel", "recordId"]]
                 for metricName in featureMetrics:
                     addMetricType(metricName, features[metricName])
 
+            
             for channel in channelData.iloc:
                 if "signalName" not in channel:
                     continue
                 chanelDict = {
                     "name": channel["signalName"],
-                    "frequency": channel["sample_rate"], # sample_frequency
+                    "frequency": channel["sample_rate"] if "sample_rate" in channel else channel["sample_frequency"], # sample_frequency
                     "prefilter": channel["prefilter"],
                     "physical_max": channel["physical_max"],
                     "physical_min": channel["physical_min"],
                     "digital_max": channel["digital_max"],
                     "digital_min": channel["digital_min"],
-                    "prefilter": channel["prefilter"],
                     "transducer": channel["transducer"],
                     "type": channel["type"],
                     "id_record": recordId
@@ -197,7 +192,7 @@ class ExportMetadataToMeticalDB(Phase):
                 q = PostgreSQLQuery.into(channelTable)\
                     .columns(*list(chanelDict.keys()))\
                     .insert(*list(chanelDict.values()))
-                channelId = medDB.execute(str(q))
+                channelId = medDB.executeAndFetchOne(f"{q} RETURNING id_channel")[0]
 
 
                 # add channel specific features
@@ -232,37 +227,32 @@ class ExportMetadataToMeticalDB(Phase):
                 # medDB.execute(str(q))
             
 
-            q = PostgreSQLQuery.from_(channelTable)\
-                .delete()\
-                .where(channelTable.id_record == recordId)
-            medDB.execute(str(q))
-
-            if "channels" not in row:
-                continue
-
-            for channel in row["channels"]:
-                channelName = channel["signalName"]
-
-                chanelDict = {
-                    "name": channelName,
-                    "frequency": channel["sample_rate"], # sample_frequency
-                    "prefilter": channel["prefilter"],
-                    "physical_max": channel["physical_max"],
-                    "physical_min": channel["physical_min"],
-                    "digital_max": channel["digital_max"],
-                    "digital_min": channel["digital_min"],
-                    "prefilter": channel["prefilter"],
-                    "transducer": channel["transducer"],
-                    "type": channel["type"],
-                    "id_record": recordId
-                }
-                q = PostgreSQLQuery.into(channelTable)\
-                    .columns(*list(chanelDict.keys()))\
-                    .insert(*list(chanelDict.values()))
-                medDB.execute(str(q))
-
-            if rowIndex % 25 == 0:
-                medDB.commit()
+            # if "channels" not in row:
+            #     continue
+
+            # for channel in row["channels"]:
+            #     channelName = channel["signalName"]
+
+            #     chanelDict = {
+            #         "name": channelName,
+            #         "frequency": channel["sample_rate"], # sample_frequency
+            #         "prefilter": channel["prefilter"],
+            #         "physical_max": channel["physical_max"],
+            #         "physical_min": channel["physical_min"],
+            #         "digital_max": channel["digital_max"],
+            #         "digital_min": channel["digital_min"],
+            #         "prefilter": channel["prefilter"],
+            #         "transducer": channel["transducer"],
+            #         "type": channel["type"],
+            #         "id_record": recordId
+            #     }
+            #     q = PostgreSQLQuery.into(channelTable)\
+            #         .columns(*list(chanelDict.keys()))\
+            #         .insert(*list(chanelDict.values()))
+            #     medDB.execute(str(q))
+
+            # if rowIndex % 25 == 0:
+            medDB.commit()
             # table = schema.acq_metadata
 
             # checkQuery = PostgreSQLQuery.from_(table).select(1).where(table.acquisition_id == metaData['acquisition_id'])

{sleepyphases-0.2.0 → sleepyphases-0.2.1}/SleePyPhases/phases/Extract.py

@@ -1,3 +1,4 @@
+from pathlib import Path
 import numpy as np
 from pyPhases import Phase
 from pyPhases.util import BatchProgress
@@ -228,25 +229,33 @@ class Extract(Phase):
         exporterSignals.options["dtype"] = self.getConfig("preprocessing.dtype", "float32")
         exporterFeatures.options["dtype"] = self.getConfig("preprocessing.dtype", "float32")
 
+        removedRecordIds = []
+        removed_records_path = exporterSignals.getPath(f"{dataIdSignals}-removed-records-tmp.npy")
+        
         if force is False:
             if exporterSignals.exists(dataIdSignals) and exporterFeatures.exists(dataIdFeatures):
-                self.logSuccess("Datafiles allready exist")
-                return
+                self.logSuccess("Datafiles already exist")
+                return []
 
             if exporterSignals.existsTmp(dataIdSignals) and exporterFeatures.existsTmp(dataIdFeatures):
                 exporterSignals.loadTmp(dataIdSignals)
                 exporterFeatures.loadTmp(dataIdFeatures)
+                
+                if Path(removed_records_path).exists():
+                    removedRecordIds = np.load(removed_records_path, allow_pickle=True).tolist()
+
                 recordsLoaded = len(exporterFeatures.currentArrayShapes[dataIdSignals][0])
-                records = records[recordsLoaded:]
-                self.logSuccess("Resuming extraction: %i records allready loaded / %i to go" % (recordsLoaded, len(records)))
+                total_processed = recordsLoaded + len(removedRecordIds)
+                
+                # Skip both successfully processed and removed records
+                records = records[total_processed:]
+                self.logSuccess(f"Resuming extraction: {recordsLoaded} records already loaded, {len(removedRecordIds)} records skipped, {len(records)} to go")
 
         bp = BatchProgress(records)
         bp.useMultiThreading = self.useMultiThreading
         if self.threads is not None:
             bp.threads = self.threads
 
-        removedRecordIds = []
-
         def combine(recordList, dataIndex):
             recordList = recordList.copy()
             # append the the data and the shape to a tmp file
@@ -254,7 +263,10 @@ class Extract(Phase):
             # check if all records where loaded, else extend the removedRecord list
             if None in recordList:
                 rm = [i + dataIndex for i, r in enumerate(recordList) if r is None]
-                [removedRecordIds.append(records[r]) for r in rm]
+                new_removed = [records[r] for r in rm]
+                removedRecordIds.extend(new_removed)
+                np.save(removed_records_path, removedRecordIds)
+                
                 recordList = [r for r in recordList if r is not None]
                 self.logWarning("Skipped %i records: channels/annotation not present" % len(rm))
 

{sleepyphases-0.2.0 → sleepyphases-0.2.1}/SleePyPhases/phases/ExtractFeatures.py

@@ -174,6 +174,8 @@ class ExtractFeatures(Phase):
             recordSignal, eventlist = recordLoader.loadRecord(recordId)
         except AnnotationNotFound as e:
             self.logError(f"Annotation missing in {recordId}: {e.name} ... skipping record")
+        except ChannelsNotPresent as e:
+            self.logError(f"Channels missing in {recordId}: {e.channels} ... skipping record")
         except ParseError as e:
             self.logError(f"ParseError in record {recordId}, record will be skipped: {e}")
 
@@ -203,19 +205,20 @@ class ExtractFeatures(Phase):
         return self.features.loc[idx]
 
     def main(self):
+        import pandas as pd
         self.features = None
-        groupedRecords = self.project.getData("allDBRecordIds", list)
+        metadata = self.project.getData("metadata", pd.DataFrame)
+        recordIds = metadata["recordId"]
 
         reqFeatures = {f["name"]: True for f in self.getConfig("featureExtraction.features", [])}
 
         # groupedRecords = {"pid_202436073": "acq_237478592"}
-        for patient, recordIds in tqdm(groupedRecords.items()):
-            for recordId in recordIds:
-                # tryToGenerate has no meaning, it is just known to be not in the metadata, so the record will always be recalculated
-                # self.getData("features", pd.DataFrame, recordId=recordId, tryToGenerate=True, **reqFeatures)
-                rows = self.getData("features", pd.DataFrame, recordId=recordId, **reqFeatures)
-                if len(rows) == 0:
-                    self.logError(f"no features for {recordId}")
+        for recordId in tqdm(recordIds.iloc):
+            # tryToGenerate has no meaning, it is just known to be not in the metadata, so the record will always be recalculated
+            # self.getData("features", pd.DataFrame, recordId=recordId, tryToGenerate=True, **reqFeatures)
+            rows = self.getData("features", pd.DataFrame, recordId=recordId, **reqFeatures)
+            if len(rows) == 0:
+                self.logError(f"no features for {recordId}")
                 
 
         # get a single feature:

{sleepyphases-0.2.0 → sleepyphases-0.2.1}/SleePyPhases/phases/TestRun.py

@@ -48,6 +48,8 @@ class TestRun(Phase):
             self.log(f"Creating a new Data Folder ({testRunData.as_posix()}) for the testrun")
             if not testRunData.exists():
                 testRunData.mkdir()
+            self.setConfig("data-path", testRunData)
+            self.project.prepareAllPhases()
             # else:
             #     # remove all files from the testrun folder
             #     for f in testRunData.iterdir():
@@ -121,6 +123,8 @@ class TestRun(Phase):
             self.log(f"Test Model: {self.getConfig('modelName')}")
 
             model = ModelManager.getModel()
+            model.config.logPath = testRunData / "logs"
+
             self.log(f"Current Model: {type(model)}")
             self.log(model.summary())
             train = self.project.generateData("dataset-training")

{sleepyphases-0.2.0 → sleepyphases-0.2.1}/SleePyPhases/phases/ThresholdOptimisation.py

@@ -4,6 +4,7 @@ from pyPhasesML import Model, ModelManager
 from tqdm import tqdm
 
 from SleePyPhases.DataManipulation import DataManipulation
+from pyPhasesML import Scorer
 from SleePyPhases.EventScorer import EventScorer
 from SleePyPhases.MultiScorer import MultiScorer
 
@@ -35,12 +36,8 @@ class ThresholdOptimisation(Phase):
             x, t = validationData[i]
             p = model.predict(x, get_likelihood=True, returnNumpy=False)
             p, t = da((p, t))
-            prediction.append(p)
-            truth.append(t)
-
-        # we assume output is reduces to bachsize 1 during augmentation (1, X, channels)
-        prediction = np.concatenate(prediction, axis=1)
-        truth = np.concatenate(truth, axis=1)
+            [prediction.append(x) for x in p]
+            [truth.append(x) for x in t]
 
         return prediction, truth
 
@@ -54,8 +51,10 @@ class ThresholdOptimisation(Phase):
         classNums = self.getConfig("model.multiclassificationSize", [2])
         labelNames = self.getConfig("classification.labelNames")
         ignoreIndex = self.getConfig("classification.ignoreIndex", -1)
+        optimizeThresholdFor = self.getConfig("optimizeThresholdFor", None)
+        scorerTypes = [(EventScorer if sc == "event" else Scorer) for sc in self.getConfig("classification.scorerTypes")]
 
-        scorer = MultiScorer(classNums, thresholdMetrics, scorerNames=labelNames, ignoreClasses=[ignoreIndex], scorerClass=EventScorer)
+        scorer = MultiScorer(classNums, thresholdMetrics, scorerNames=labelNames, ignoreClasses=[ignoreIndex], scorerClasses=scorerTypes)
         for s in scorer.scorer.values():
             s.majorityVote = self.getConfig("eventEval.tpStrat", "overlap") == "majority"
             s.noTN = self.getConfig("eventEval.tnStrat", "eventcount") == "noTN"
@@ -88,6 +87,9 @@ class ThresholdOptimisation(Phase):
         
         thresholds = []
         for scorerIndex, (scorerName, thresholdMetric) in enumerate(zip(labelNames, thresholdMetrics)):
+            if optimizeThresholdFor is not None and scorerName not in optimizeThresholdFor:
+                thresholds.append(None)
+                continue
             self.log(f"Optimize threshold for {scorerName} {thresholdMetric}")
             t = scipy.optimize.fmin(score, args=(prediction, truth, scorerIndex, scorerName, thresholdMetric), x0=0.3, xtol=0.01, ftol=0.01)
             self.logSuccess(f"Optimized threshold for {scorerName} {thresholdMetric}: {t}")

{sleepyphases-0.2.0 → sleepyphases-0.2.1}/SleePyPhases.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
-Metadata-Version: 2.2
+Metadata-Version: 2.4
 Name: SleePyPhases
-Version: 0.2.0
+Version: 0.2.1
 Summary: A framwork for creating deep learning pipelines for sleep data
 Home-page: https://gitlab.com/sleep-is-all-you-need/sleepyphases
 Author: Franz Ehrlich
@@ -20,6 +20,7 @@ Dynamic: classifier
 Dynamic: description
 Dynamic: description-content-type
 Dynamic: home-page
+Dynamic: license-file
 Dynamic: requires-dist
 Dynamic: requires-python
 Dynamic: summary

{sleepyphases-0.2.0 → sleepyphases-0.2.1}/setup.py

@@ -5,7 +5,7 @@ with open("README.md", "r") as fh:
 
 setuptools.setup(
     name="SleePyPhases",
-    version="v0.2.0"[1:],
+    version="v0.2.1"[1:],
     author="Franz Ehrlich",
     author_email="fehrlichd@gmail.com",
     description="A framwork for creating deep learning pipelines for sleep data",