SleePyPhases 0.1.6__tar.gz → 0.2.0__tar.gz

{sleepyphases-0.1.6 → sleepyphases-0.2.0}/PKG-INFO

@@ -1,6 +1,6 @@
-Metadata-Version: 2.1
+Metadata-Version: 2.2
 Name: SleePyPhases
-Version: 0.1.6
+Version: 0.2.0
 Summary: A framwork for creating deep learning pipelines for sleep data
 Home-page: https://gitlab.com/sleep-is-all-you-need/sleepyphases
 Author: Franz Ehrlich
@@ -14,6 +14,15 @@ License-File: LICENSE
 Requires-Dist: SleepHarmonizer
 Requires-Dist: pyPhasesML
 Requires-Dist: phases
+Dynamic: author
+Dynamic: author-email
+Dynamic: classifier
+Dynamic: description
+Dynamic: description-content-type
+Dynamic: home-page
+Dynamic: requires-dist
+Dynamic: requires-python
+Dynamic: summary
 
 # Arousal Detector
 

{sleepyphases-0.1.6 → sleepyphases-0.2.0}/SleePyPhases/DataManipulation.py

@@ -880,4 +880,13 @@ class DataManipulation(pyPhasesDataManipulation, Swappable):
         # Update the channel in Y
         Y[:, :, channel] = y_channel
         
+        return X, Y
+    
+    
+    def combineY(self, X, Y, values, channel=0):
+        """" Combine multiple Y values into one. """
+        smallest = min(values)
+        Y[:, :, channel] = np.where(np.isin(Y[:, :, channel], values), values[-1], Y[:, :, channel])
+        Y[:, :, channel][Y[:, :, channel] > smallest] -= len(values) - 1
+        
         return X, Y

{sleepyphases-0.1.6 → sleepyphases-0.2.0}/SleePyPhases/Plugin.py

@@ -11,6 +11,8 @@ from SleePyPhases.phases.Validation import Validation
 from SleePyPhases.phases.Eval import Eval
 from SleePyPhases.phases.EvalReport import EvalReport
 from SleePyPhases.phases.ThresholdOptimisation import ThresholdOptimisation
+from SleePyPhases.phases.TestRun import TestRun
+
 
 from pyPhases.exporter.PickleExporter import PickleExporter
 from pyPhases.exporter.PandasExporter import PandasExporter
@@ -31,6 +33,7 @@ class Plugin(PluginAdapter):
             "ThresholdOptimisation": (ThresholdOptimisation, ["threshold", "validationResult"]),
             "Eval": (Eval, ["evalResults", "eventResults"]),
             "EvalReport": (EvalReport, []),
+            "TestRun": (TestRun, [])
 
         }
         dataMap = {

{sleepyphases-0.1.6 → sleepyphases-0.2.0}/SleePyPhases/RecordFeatureExtraction.py

@@ -56,7 +56,7 @@ class RecordFeatureExtraction(pyPhasesFeatureExtraction):
         rem_starts = [rem[i].start for i in range(1, len(rem)) if rem[i].start - rem[i-1].end() > 1]
         rem_ends = [rem[i-1].end() for i in range(1, len(rem)) if rem[i].start - rem[i-1].end() > 1]
 
-        if len(rem_ends) == 0:
+        if len(rem_ends) == 0 or len(nrem) == 0:
             return cycles
         
         # Set cycle start and stop times
@@ -123,12 +123,35 @@ class RecordFeatureExtraction(pyPhasesFeatureExtraction):
         
         return offsetStart, offsetEnd
         
+    def _updateEvents(self, eventlist, offsetStart, offsetEnd):
+        
+        # update events
+        newEvents = []
+        for event in eventlist:
+            if offsetStart > 0:
+                if event.end() <= offsetStart:
+                    continue
+                if event.start < offsetStart:
+                    event.duration -= offsetStart - event.start
+                    event.start = 0
+                else:
+                    event.start -= offsetStart
+            if offsetEnd is not None:
+                if event.start >= offsetEnd - offsetStart:
+                    continue
+                if event.end() > offsetEnd - offsetStart:
+                    event.duration -= event.end() - offsetEnd + offsetStart
+            newEvents.append(event)
+        return newEvents
 
     # TODO: outsource to PSGEventManager
     def _tailorToSleepScoring(self, segmentSignal: RecordSignal, eventlist):
 
         sleepEventNames = ["R", "N1", "N2", "N3", "W"]
         sleepEvents = [e for e in eventlist if e.name in sleepEventNames]
+
+        if len(sleepEvents) == 0:
+            return segmentSignal, []
         
         # fix if the last sleep stage has no duration (should be fixed in Recordloader (Alice))
         if sleepEvents[-1].duration == 0:
@@ -145,23 +168,7 @@ class RecordFeatureExtraction(pyPhasesFeatureExtraction):
 
         segmentSignal.signalOffset(offsetStart, offsetEnd)
 
-        # update events
-        newEvents = []
-        for event in eventlist:
-            if offsetStart > 0:
-                if event.end() <= offsetStart:
-                    continue
-                if event.start < offsetStart:
-                    event.duration -= offsetStart - event.start
-                    event.start = 0
-                else:
-                    event.start -= offsetStart
-            if offsetEnd is not None:
-                if event.start >= offsetEnd - offsetStart:
-                    continue
-                if event.end() > offsetEnd - offsetStart:
-                    event.duration -= event.end() - offsetEnd + offsetStart
-            newEvents.append(event)
+        newEvents = self._updateEvents(eventlist, offsetStart, offsetEnd)
 
         return segmentSignal, newEvents
 
@@ -173,6 +180,17 @@ class RecordFeatureExtraction(pyPhasesFeatureExtraction):
 
         signal = segmentSignal.getSignalByName(channel)
 
+        # check if the scoring is longer than the actual signal
+        sleepEventNames = ["R", "N1", "N2", "N3", "W"]
+        sleepEvents = [e for e in eventlist if e.name in sleepEventNames]        
+        scoringLength = sleepEvents[-1].end() - sleepEvents[0].start
+        signalLength = signal.signal.shape[0] / signal.frequency
+        if scoringLength > signalLength:
+            newLength = (len(signal.signal)/signal.frequency//30)*30
+            newLength = min(newLength, signalLength)
+            eventlist = self._updateEvents(eventlist, 0, newLength)
+            signal.signal = signal.signal[:int(newLength*signal.frequency)]
+
 
         labelSignal = PSGEventManager().getEventSignalFromList(
             eventlist,
@@ -190,7 +208,6 @@ class RecordFeatureExtraction(pyPhasesFeatureExtraction):
         stage_mask[stage_mask == 0] = np.nan
 
         power_epoch_mean = self._swa(signal)
-
         assert len(power_epoch_mean) == len(labelSignal)
 
         signal_masked = stage_mask * power_epoch_mean
@@ -208,6 +225,8 @@ class RecordFeatureExtraction(pyPhasesFeatureExtraction):
     def SWA3(self, segmentSignal: RecordSignal, eventlist: List[Event], channel: str):
         
         segmentSignal, eventlist = self._tailorToSleepScoring(segmentSignal, eventlist)
+        if len(eventlist) == 0:
+            return {}
 
         swa_cycles_all, swa_cycles_n2n3, cycles = self._SWAandCycles(segmentSignal, eventlist, channel)
 
@@ -222,4 +241,26 @@ class RecordFeatureExtraction(pyPhasesFeatureExtraction):
             "deltapower_0.5_n2n3_first": swa_cycles_n2n3.iloc[0].to_list()[0],
             "deltapower_0.5_n2n3_max": swa_cycles_n2n3.max().to_list()[0],
             "deltapower_0.5_n2n3_last": swa_cycles_n2n3.iloc[-1].to_list()[0],
-        }
+        }
+    
+    def REMCycleLength(self, segmentSignal: RecordSignal, eventlist: List[Event], channel: str):
+        cycles = self._SleepCycles(segmentSignal, eventlist)
+
+        lastStop = 0
+        remEvents = [e for e in eventlist if e.name == "R"]
+
+        ret = {
+            "cycle-count": len(cycles["start"]),
+        }
+        
+        for cycleNumber, (start, stop) in enumerate(zip(cycles["start"], cycles["end"])):
+            if lastStop > 0:
+                remTime = sum([e.duration for e in remEvents if e.start >= lastStop and e.start < start])
+                ret[f"cycle-{cycleNumber}-rem-duration"] = remTime
+
+            lastStop = stop
+
+        return {
+            "cycle-count": len(cycles["start"]),
+        }
+    

{sleepyphases-0.1.6 → sleepyphases-0.2.0}/SleePyPhases/SignalPreprocessing.py

@@ -244,7 +244,7 @@ class SignalPreprocessing(pyPhaseSignalPreprocessing, Swappable):
         return np.concatenate(signal)
        
     def fir(self, signal: Signal, recordSignal: RecordSignal, nFir, cutoff, pass_zero=False):
-        from scipy import signal
+        from scipy import signal as scipySignal
 
         fs = signal.frequency
         nyq = fs/2

{sleepyphases-0.1.6 → sleepyphases-0.2.0}/SleePyPhases/SleepMetaData.py

@@ -68,16 +68,20 @@ class SleepMetaData:
                 self.fromArousalSignal(signal)
             case 'ApneaBin':
                 self.fromApneaSignal(signal, ["None", "respEvent"])
+            case 'Apnea4':
+                self.fromApneaSignal(signal, ["None", "obstructive/mixed", "central", "hypopnea"])
             case 'LMBin':
                 self.fromLegMovementSignal(signal)
             case 'SleepBin':
                 self.fromSleepSignalBin(signal, frequency=frequency)
+            case 'Sleep4':
+                self.fromSleepSignalBin(signal, frequency=frequency)
 
     def fromSleepSignalBin(self, signal, frequency = 1):
         # all time values in seconds
         trt = len(signal) / (frequency)
         tst = sum(signal) / (frequency)
-        whereSleep = np.where(signal == 1)[0]
+        whereSleep = np.where(signal > 0)[0]
         if len(whereSleep) == 0:
             sLatency = 0
         else:

{sleepyphases-0.1.6 → sleepyphases-0.2.0}/SleePyPhases/__init__.py

@@ -1,4 +1,5 @@
 
 from .DataManipulation import DataManipulation
 from .SignalPreprocessing import SignalPreprocessing
-from .FeatureExtraction import FeatureExtraction
+from .FeatureExtraction import FeatureExtraction
+from .PreManipulation import PreManipulation

{sleepyphases-0.1.6 → sleepyphases-0.2.0}/SleePyPhases/phases/BuildDataset.py

@@ -30,7 +30,7 @@ class BuildDataset(Phase):
 
                 self.project.updateConfig(overwriteConfig)
                 memmapOptions = {
-                    "dtype": self.getConfig("preprocessing.dtype"),
+                    "dtype": self.getConfig("preprocessing.dtype", "float32"),
                 }
                 dataExporterSignals = self.project.getData("data-processed", np.memmap, options=memmapOptions)
                 dataExporterFeatures = self.project.getData("data-features", np.memmap, options=memmapOptions)

{sleepyphases-0.1.6 → sleepyphases-0.2.0}/SleePyPhases/phases/Extract.py

@@ -1,13 +1,12 @@
 import numpy as np
 from pyPhases import Phase
 from pyPhases.util import BatchProgress
-from pyPhasesML import DataversionManager
-from pyPhasesRecordloader import AnnotationNotFound, ChannelsNotPresent, RecordLoader, ParseError
+from pyPhasesRecordloader import AnnotationNotFound, ChannelsNotPresent, ParseError, RecordLoader
 
+from SleePyPhases.FeatureExtraction import FeatureExtraction
 from SleePyPhases.PreManipulation import PreManipulation
 from SleePyPhases.PSGEventManager import PSGEventManager
 from SleePyPhases.SignalPreprocessing import SignalPreprocessing
-from SleePyPhases.FeatureExtraction import FeatureExtraction
 
 
 class RecordProcessor:
@@ -207,7 +206,7 @@ class Extract(Phase):
         processRecord = RecordProcessor(
             recordLoader=RecordLoader.get(),
             preProcessingConfig=preprocessingConfig,
-            signalProcessing=SignalPreprocessing(preprocessingConfig),
+            signalProcessing=SignalPreprocessing.getInstance(preprocessingConfig),
             eventManager=PSGEventManager(),
             labelChannels=self.getConfig("labelChannels"),
             project=self.project,

{sleepyphases-0.1.6 → sleepyphases-0.2.0}/SleePyPhases/phases/Setup.py

@@ -97,7 +97,12 @@ class Setup(Phase):
 
         # add training folding splits
         if "trainval" in splits:
-            dm.addSplitsByFold("training", "validation", splits["trainval"], self.getConfig("dataversion.folds", 0), self.getConfig("fold", 0))
+            foldcount = self.getConfig("dataversion.folds", 0)
+            if foldcount > 0:
+                dm.addSplitsByFold("training", "validation", splits["trainval"], self.getConfig("dataversion.folds", 0), self.getConfig("fold", 0))
+            else:
+                dm.addSplitByRemaining("validation", valSplit) # , remainingSplit=trainingSlice
+                dm.addSplitByRemaining("training", 1)
 
             if "training" in splits or "validation" in splits:
                 raise Exception("trainval and training/validation split are mutually exclusive. Please remove one of them from the config. This can be caused by loading multiple configs with different splits.")

sleepyphases-0.2.0/SleePyPhases/phases/TestRun.py

@@ -0,0 +1,147 @@
+from pathlib import Path
+
+
+from SleePyPhases import SignalPreprocessing as SP, PreManipulation as PM, FeatureExtraction as FE, DataManipulation as DM
+from SleePyPhases.phases.Extract import Extract
+from SleePyPhases.phases.BuildDataset import BuildDataset
+from pyPhases import Phase
+from pyPhasesRecordloader import RecordLoader
+import os
+from pyPhasesML import ModelManager, TrainingSetLoader
+
+class TestRun(Phase):
+    useMultiThreading = False
+    def main(self):
+        import numpy as np
+        import pandas as pd
+        
+        debugConfig = self.getConfig("debugConfig", {})
+        print(f"Overwrite Config with debug values in debugConfig: {debugConfig}")
+
+        with self.project:
+            self.project.updateConfig(debugConfig)
+            loader = self.getConfig("useLoader")
+            rl = RecordLoader.get()
+            # rl.debug = True
+
+            if not self.useMultiThreading:
+                self.log("Disable multi threading for better error messages. To test multi threading, set TestRun.useMultiThreading to True")
+                Extract.useMultiThreading = False
+                BuildDataset.useMultiThreading = False
+            
+            self.log("Testing pyPhases settings")
+            dataPath = Path(self.getConfig("data-path"))
+            if dataPath.exists():
+                self.logSuccess(f"Path {dataPath} exists: {dataPath.exists()}")
+                
+                # Test if directory is writable
+                if os.access(dataPath, os.W_OK):
+                    self.logSuccess(f"Data Path ({dataPath}) is writable")
+                else:
+                    self.logError(f"Data Path ({dataPath}) is not writable. Make sure to set the correct path (Config: data-path) and permissions.")
+                    exit(1)
+            else:
+                self.logError(f"Data Path ({dataPath}) does not exist. Please set the correct path (Config: data-path)")
+                exit(1)
+
+            testRunData = dataPath / "testrun"
+            self.log(f"Creating a new Data Folder ({testRunData.as_posix()}) for the testrun")
+            if not testRunData.exists():
+                testRunData.mkdir()
+            # else:
+            #     # remove all files from the testrun folder
+            #     for f in testRunData.iterdir():
+            #         f.unlink()
+
+            self.log("Creating a new Data Folder for the testrun")
+
+            self.log(f"Test specified RecordLoader: {loader} ({type(rl)})")
+            datasetPath = Path(rl.filePath)
+            if datasetPath.exists():
+                self.logSuccess(f"RL Path ({rl.filePath}) exists: {Path(rl.filePath).exists()}")
+            else:
+                self.logError(f"RL Path ({rl.filePath}) does not exist. Make sure to download the dataset and set the correct path (Config: {loader}-path)")
+                exit(1)
+
+            self.setConfig("testrun", True)
+
+            self.log("Test generating dataset metadata, for first records")
+            df = self.getData("metadata", pd.DataFrame)
+            self.log(f"Metadata: {df.head()} / {df.shape}")
+
+            if len(df) > 0:
+                self.logSuccess("Dataset records loaded")
+            else:
+                self.logError("Dataset records not loaded")
+            # metadata-channels, dataIsFinal
+
+            self.log("Test data version and split")
+            dm = self.getData("dataversionmanager")
+            self.log(f"Splits (assuming all records exist): {dm.splits}")
+
+            self.log("For debugging purpose, we will reduce the split using the same 2 records for all splits")
+            dm.splits["training"][0] = slice(0, 2)
+            dm.splits["validation"][0] = slice(0, 2)
+            dm.splits["test"][0] = slice(0, 2)
+
+            self.log("Test extracting data with preprocessing defined in (Config: preprocessing)")
+            self.log(f"Preprocessing Config: {self.getConfig('preprocessing')}")
+            self.log(f"Preprocessing target frequency: {self.getConfig('preprocessing.targetFrequency')}")
+            self.log(f"Preprocessing target labelFrequency: {self.getConfig('preprocessing.labelFrequency')}")
+            self.log(f"Preprocessing target channels: {self.getConfig('preprocessing.targetChannels')}")
+            # check that resample is in each target signal related preprocessing
+
+            self.log(f"Current SignalPreprocessing Class: {type(SP.getInstance(self.getConfig('preprocessing')))}")
+            self.log(f"Current PreManipulation Class: {type(PM.getInstance(self.getConfig('preprocessing')))}")
+            # self.log(f"Current FeatureExtraction: {type(FE.getInstance(self.getConfig('preprocessing')))}")
+            self.log(f"Current SignalPreprocessing: {self.getConfig('preprocessing.stepsPerType')}")
+            self.log(f"Current PreManipulation: {self.getConfig('preprocessing.manipulationSteps')}")
+            # self.log(f"Current FeatureExtraction: {type(FE.getInstance(self.getConfig('preprocessing')))}")
+
+            memmapOptions = {
+                "dtype": self.getConfig("preprocessing.dtype", "float32"),
+            }
+            dataExporterSignals = self.getData("data-processed", np.memmap, options=memmapOptions)
+            dataExporterFeatures = self.getData("data-features", np.memmap, options=memmapOptions)
+
+            self.logSuccess(f"Extract finished: len X: {len(dataExporterSignals)} / len Y: {len(dataExporterFeatures)}")
+
+            self.log("Test data manipulation")
+            manipulationSteps = self.getConfig("segmentManipulation")
+            self.log(f"Current segmentManipulation: {manipulationSteps}")
+            segmentManipulation = DM.getInstance(manipulationSteps, "training", self.project.config)
+            self.log(f"Current DataManipulation Class: {type(segmentManipulation)}")
+            # segmentManipulation = DM.getInstance(manipulationSteps, "training", self.project.config, recordMetadata=recordMetaData)
+            
+            X, Y = dataExporterSignals[0], dataExporterFeatures[0]
+            X, Y = segmentManipulation((X, Y), None, 0)
+
+            self.logSuccess(f"Segment Manipulation finished: X: {X.shape} / Y: {Y.shape}")
+
+            self.log(f"Test Model: {self.getConfig('modelName')}")
+
+            model = ModelManager.getModel()
+            self.log(f"Current Model: {type(model)}")
+            self.log(model.summary())
+            train = self.project.generateData("dataset-training")
+            val = self.project.generateData("dataset-validation")
+            trainingsSet = TrainingSetLoader(trainingData=train, validationData=val)
+            model.debug = True
+            trainedModel = model.train(trainingsSet)
+
+            self.logSuccess("Debug Model trained")
+
+
+
+
+            # allDBRecordIds
+            # self.getData("allRecordIds")
+            # extract data
+            # build data
+            # data manipulation
+            # load model
+            # training
+            # validation
+            # threshold optimization
+            # segment evaluation
+            # event evaluation

{sleepyphases-0.1.6 → sleepyphases-0.2.0}/SleePyPhases.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
-Metadata-Version: 2.1
+Metadata-Version: 2.2
 Name: SleePyPhases
-Version: 0.1.6
+Version: 0.2.0
 Summary: A framwork for creating deep learning pipelines for sleep data
 Home-page: https://gitlab.com/sleep-is-all-you-need/sleepyphases
 Author: Franz Ehrlich
@@ -14,6 +14,15 @@ License-File: LICENSE
 Requires-Dist: SleepHarmonizer
 Requires-Dist: pyPhasesML
 Requires-Dist: phases
+Dynamic: author
+Dynamic: author-email
+Dynamic: classifier
+Dynamic: description
+Dynamic: description-content-type
+Dynamic: home-page
+Dynamic: requires-dist
+Dynamic: requires-python
+Dynamic: summary
 
 # Arousal Detector
 

{sleepyphases-0.1.6 → sleepyphases-0.2.0}/SleePyPhases.egg-info/SOURCES.txt

@@ -34,6 +34,7 @@ SleePyPhases/phases/ExtractFeatures.py
 SleePyPhases/phases/ExtractRecordFeatures.py
 SleePyPhases/phases/GatherMetadata.py
 SleePyPhases/phases/Setup.py
+SleePyPhases/phases/TestRun.py
 SleePyPhases/phases/ThresholdOptimisation.py
 SleePyPhases/phases/Training.py
 SleePyPhases/phases/Validation.py

{sleepyphases-0.1.6 → sleepyphases-0.2.0}/setup.py

@@ -5,7 +5,7 @@ with open("README.md", "r") as fh:
 
 setuptools.setup(
     name="SleePyPhases",
-    version="v0.1.6"[1:],
+    version="v0.2.0"[1:],
     author="Franz Ehrlich",
     author_email="fehrlichd@gmail.com",
     description="A framwork for creating deep learning pipelines for sleep data",

{sleepyphases-0.1.6 → sleepyphases-0.2.0}/tests/unit/test_DataManipulation.py

@@ -268,4 +268,30 @@ class TestDataManipulation(unittest.TestCase):
         
         npt.assert_equal(Y_new.shape, expected.shape)
         npt.assert_equal(Y_new[0, :, 0], expected[0, :, 0])
-        npt.assert_equal(X_new, X)  # X should remain unchanged
+        npt.assert_equal(X_new, X)  # X should remain unchanged
+        
+    def test_combineY(self):
+        da = self.getDA()
+        
+        X = np.zeros((2, 10, 2))
+        Y = np.array([
+            [[0], [1], [2], [3], [4], [0], [1], [2], [3], [4]],
+            [[4], [3], [2], [1], [0], [4], [3], [2], [1], [0]]
+        ])
+
+        # Test combining values 2,1 into 0
+        _, Y_out = da.combineY(X, Y.copy(), values=[2, 1, 0])
+        expected = np.array([
+            [[0], [0], [0], [1], [2], [0], [0], [0], [1], [2]],
+            [[2], [1], [0], [0], [0], [2], [1], [0], [0], [0]]
+        ])
+
+        np.testing.assert_array_equal(Y_out, expected)
+        
+        # Test combining values 4,3,2 into 1
+        _, Y_out = da.combineY(X, Y.copy(), values=[1, 2])
+        expected = np.array([
+            [[0], [1], [1], [2], [3], [0], [1], [1], [2], [3]],
+            [[3], [2], [1], [1], [0], [3], [2], [1], [1], [0]]
+        ])
+        np.testing.assert_array_equal(Y_out, expected)