SleePyPhases 0.6.0__tar.gz → 0.6.2__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {sleepyphases-0.6.0 → sleepyphases-0.6.2}/PKG-INFO +1 -1
- {sleepyphases-0.6.0 → sleepyphases-0.6.2}/SleePyPhases/EventScorer.py +1 -1
- {sleepyphases-0.6.0 → sleepyphases-0.6.2}/SleePyPhases/Plugin.py +4 -0
- {sleepyphases-0.6.0 → sleepyphases-0.6.2}/SleePyPhases/RecordFeatureExtraction.py +1 -1
- {sleepyphases-0.6.0 → sleepyphases-0.6.2}/SleePyPhases/SleepMetaData.py +1 -1
- {sleepyphases-0.6.0 → sleepyphases-0.6.2}/SleePyPhases/phases/BuildDataset.py +0 -1
- {sleepyphases-0.6.0 → sleepyphases-0.6.2}/SleePyPhases/phases/DataReport.py +1 -1
- {sleepyphases-0.6.0 → sleepyphases-0.6.2}/SleePyPhases/phases/Extract.py +7 -8
- {sleepyphases-0.6.0 → sleepyphases-0.6.2}/SleePyPhases/phases/ExtractEvents.py +79 -55
- {sleepyphases-0.6.0 → sleepyphases-0.6.2}/SleePyPhases/phases/ExtractFeatures.py +5 -4
- {sleepyphases-0.6.0 → sleepyphases-0.6.2}/SleePyPhases/phases/Predict.py +31 -23
- {sleepyphases-0.6.0 → sleepyphases-0.6.2}/SleePyPhases/phases/Setup.py +6 -5
- {sleepyphases-0.6.0 → sleepyphases-0.6.2}/SleePyPhases.egg-info/PKG-INFO +1 -1
- {sleepyphases-0.6.0 → sleepyphases-0.6.2}/SleePyPhases.egg-info/SOURCES.txt +0 -1
- {sleepyphases-0.6.0 → sleepyphases-0.6.2}/setup.py +1 -1
- {sleepyphases-0.6.0 → sleepyphases-0.6.2}/tests/unit/test_RecordFeatureExtraction.py +1 -1
- sleepyphases-0.6.0/SleePyPhases/PSGEventManager.py +0 -137
- {sleepyphases-0.6.0 → sleepyphases-0.6.2}/LICENSE +0 -0
- {sleepyphases-0.6.0 → sleepyphases-0.6.2}/README.md +0 -0
- {sleepyphases-0.6.0 → sleepyphases-0.6.2}/SleePyPhases/DataManipulation.py +0 -0
- {sleepyphases-0.6.0 → sleepyphases-0.6.2}/SleePyPhases/FeatureExtraction.py +0 -0
- {sleepyphases-0.6.0 → sleepyphases-0.6.2}/SleePyPhases/MultiScorer.py +0 -0
- {sleepyphases-0.6.0 → sleepyphases-0.6.2}/SleePyPhases/Plot.py +0 -0
- {sleepyphases-0.6.0 → sleepyphases-0.6.2}/SleePyPhases/PreManipulation.py +0 -0
- {sleepyphases-0.6.0 → sleepyphases-0.6.2}/SleePyPhases/Reporter.py +0 -0
- {sleepyphases-0.6.0 → sleepyphases-0.6.2}/SleePyPhases/SignalPreprocessing.py +0 -0
- {sleepyphases-0.6.0 → sleepyphases-0.6.2}/SleePyPhases/__init__.py +0 -0
- {sleepyphases-0.6.0 → sleepyphases-0.6.2}/SleePyPhases/phases/DataAnalysis.py +0 -0
- {sleepyphases-0.6.0 → sleepyphases-0.6.2}/SleePyPhases/phases/Eval.py +0 -0
- {sleepyphases-0.6.0 → sleepyphases-0.6.2}/SleePyPhases/phases/EvalPlotExamples.py +0 -0
- {sleepyphases-0.6.0 → sleepyphases-0.6.2}/SleePyPhases/phases/EvalReport.py +0 -0
- {sleepyphases-0.6.0 → sleepyphases-0.6.2}/SleePyPhases/phases/ExportMetadataToMeticalDB.py +0 -0
- {sleepyphases-0.6.0 → sleepyphases-0.6.2}/SleePyPhases/phases/ExtractRecordFeatures.py +0 -0
- {sleepyphases-0.6.0 → sleepyphases-0.6.2}/SleePyPhases/phases/GatherMetadata.py +0 -0
- {sleepyphases-0.6.0 → sleepyphases-0.6.2}/SleePyPhases/phases/LogCleanup.py +0 -0
- {sleepyphases-0.6.0 → sleepyphases-0.6.2}/SleePyPhases/phases/LogOverview.py +0 -0
- {sleepyphases-0.6.0 → sleepyphases-0.6.2}/SleePyPhases/phases/TestRun.py +0 -0
- {sleepyphases-0.6.0 → sleepyphases-0.6.2}/SleePyPhases/phases/ThresholdOptimisation.py +0 -0
- {sleepyphases-0.6.0 → sleepyphases-0.6.2}/SleePyPhases/phases/Training.py +0 -0
- {sleepyphases-0.6.0 → sleepyphases-0.6.2}/SleePyPhases/phases/Validation.py +0 -0
- {sleepyphases-0.6.0 → sleepyphases-0.6.2}/SleePyPhases/phases/VisualizeConfig.py +0 -0
- {sleepyphases-0.6.0 → sleepyphases-0.6.2}/SleePyPhases/phases/__init__.py +0 -0
- {sleepyphases-0.6.0 → sleepyphases-0.6.2}/SleePyPhases.egg-info/dependency_links.txt +0 -0
- {sleepyphases-0.6.0 → sleepyphases-0.6.2}/SleePyPhases.egg-info/requires.txt +0 -0
- {sleepyphases-0.6.0 → sleepyphases-0.6.2}/SleePyPhases.egg-info/top_level.txt +0 -0
- {sleepyphases-0.6.0 → sleepyphases-0.6.2}/pyproject.toml +0 -0
- {sleepyphases-0.6.0 → sleepyphases-0.6.2}/setup.cfg +0 -0
- {sleepyphases-0.6.0 → sleepyphases-0.6.2}/tests/__init__.py +0 -0
- {sleepyphases-0.6.0 → sleepyphases-0.6.2}/tests/conftest.py +0 -0
- {sleepyphases-0.6.0 → sleepyphases-0.6.2}/tests/unit/__init__.py +0 -0
- {sleepyphases-0.6.0 → sleepyphases-0.6.2}/tests/unit/test_BuildDataset.py +0 -0
- {sleepyphases-0.6.0 → sleepyphases-0.6.2}/tests/unit/test_BuildDatasetFolded.py +0 -0
- {sleepyphases-0.6.0 → sleepyphases-0.6.2}/tests/unit/test_BuildDatasetSegments.py +0 -0
- {sleepyphases-0.6.0 → sleepyphases-0.6.2}/tests/unit/test_DataManipulation.py +0 -0
- {sleepyphases-0.6.0 → sleepyphases-0.6.2}/tests/unit/test_Eval.py +0 -0
- {sleepyphases-0.6.0 → sleepyphases-0.6.2}/tests/unit/test_EventScorer.py +0 -0
- {sleepyphases-0.6.0 → sleepyphases-0.6.2}/tests/unit/test_Extract.py +0 -0
- {sleepyphases-0.6.0 → sleepyphases-0.6.2}/tests/unit/test_PreManipulation.py +0 -0
- {sleepyphases-0.6.0 → sleepyphases-0.6.2}/tests/unit/test_RecordProcessor.py +0 -0
- {sleepyphases-0.6.0 → sleepyphases-0.6.2}/tests/unit/test_Setup.py +0 -0
- {sleepyphases-0.6.0 → sleepyphases-0.6.2}/tests/unit/test_SignalPreprocessing.py +0 -0
|
@@ -13,6 +13,8 @@ from SleePyPhases.phases.EvalReport import EvalReport
|
|
|
13
13
|
from SleePyPhases.phases.ThresholdOptimisation import ThresholdOptimisation
|
|
14
14
|
from SleePyPhases.phases.TestRun import TestRun
|
|
15
15
|
from SleePyPhases.phases.Predict import Predict
|
|
16
|
+
from SleePyPhases.phases.ExtractEvents import ExtractEvents
|
|
17
|
+
|
|
16
18
|
|
|
17
19
|
|
|
18
20
|
from pyPhases.exporter.PickleExporter import PickleExporter
|
|
@@ -39,10 +41,12 @@ class Plugin(PluginAdapter):
|
|
|
39
41
|
"LogOverview": (LogOverview, []),
|
|
40
42
|
"LogCleanup": (LogCleanup, []),
|
|
41
43
|
"Predict": (Predict, ["prediction"]),
|
|
44
|
+
"ExtractEvents": (ExtractEvents, ["events", "eventDF"])
|
|
42
45
|
}
|
|
43
46
|
dataMap = {
|
|
44
47
|
"features": ["metadata"],
|
|
45
48
|
"events": ["allDBRecordIds"],
|
|
49
|
+
"eventDF": ["events"],
|
|
46
50
|
"data-processed": ["allDBRecordIds", "preprocessing", "labelChannels"],
|
|
47
51
|
"record-preprocessed": ["recordId", "data-processed"],
|
|
48
52
|
"record-data": [],
|
|
@@ -7,7 +7,7 @@ from pyPhases import Swappable
|
|
|
7
7
|
from pyPhasesML import FeatureExtraction as pyPhasesFeatureExtraction
|
|
8
8
|
from pyPhasesRecordloader import ChannelsNotPresent, Event, RecordSignal, Signal
|
|
9
9
|
|
|
10
|
-
from
|
|
10
|
+
from SleepHarmonizer.PSGEventManager import PSGEventManager
|
|
11
11
|
|
|
12
12
|
|
|
13
13
|
class RecordFeatureExtraction(pyPhasesFeatureExtraction, Swappable):
|
|
@@ -20,7 +20,6 @@ class BuildDataset(Phase):
|
|
|
20
20
|
splitName = self.getConfig("datasetSplit")
|
|
21
21
|
|
|
22
22
|
datasetVersions = self.getConfig("datafold.datasets", {"config": {}})
|
|
23
|
-
datasetVersions = {"config": {}}
|
|
24
23
|
datasetVersions = datasetVersions if isinstance(datasetVersions, list) else [datasetVersions]
|
|
25
24
|
|
|
26
25
|
seed = self.getConfig("datafold.seed", 2024)
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
from pathlib import Path
|
|
2
2
|
|
|
3
3
|
import numpy as np
|
|
4
|
-
from
|
|
4
|
+
from SleepHarmonizer.PSGEventManager import PSGEventManager
|
|
5
5
|
from pyPhases import Phase
|
|
6
6
|
from pyPhasesRecordloader import RecordLoader, RecordSignal, Signal
|
|
7
7
|
|
|
@@ -6,7 +6,7 @@ from pyPhasesRecordloader import AnnotationNotFound, AnnotationInvalid, Channels
|
|
|
6
6
|
|
|
7
7
|
from SleePyPhases.FeatureExtraction import FeatureExtraction
|
|
8
8
|
from SleePyPhases.PreManipulation import PreManipulation
|
|
9
|
-
from
|
|
9
|
+
from SleepHarmonizer import PSGEventManager
|
|
10
10
|
from SleePyPhases.SignalPreprocessing import SignalPreprocessing
|
|
11
11
|
|
|
12
12
|
|
|
@@ -59,13 +59,12 @@ class RecordProcessor:
|
|
|
59
59
|
elif name in ["SleepApnea", "RespEvents"]:
|
|
60
60
|
if "apnea" in eventSignal:
|
|
61
61
|
evs = eventSignal["apnea"]
|
|
62
|
-
labelSignalArray[evs == PSGEventManager.
|
|
63
|
-
labelSignalArray[evs == PSGEventManager.
|
|
64
|
-
labelSignalArray[evs == PSGEventManager.
|
|
65
|
-
labelSignalArray[evs == PSGEventManager.
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
labelSignalArray[evs == PSGEventManager.INDEX_AROUSAL_RERA] = 5
|
|
62
|
+
labelSignalArray[evs == PSGEventManager.INDEX_RESPEVENT_OBSTRUCTIVE] = 1
|
|
63
|
+
labelSignalArray[evs == PSGEventManager.INDEX_RESPEVENT_MIXED] = 2
|
|
64
|
+
labelSignalArray[evs == PSGEventManager.INDEX_RESPEVENT_CENTRAL] = 3
|
|
65
|
+
labelSignalArray[evs == PSGEventManager.INDEX_RESPEVENT_HYPO] = 4
|
|
66
|
+
if name == "RespEvents":
|
|
67
|
+
labelSignalArray[evs == PSGEventManager.INDEX_RESPEVENT_INDEX_RERA] = 5
|
|
69
68
|
elif name == "SleepLegMovements":
|
|
70
69
|
if "limb" in eventSignal:
|
|
71
70
|
eventSignal = eventSignal["limb"]
|
|
@@ -2,7 +2,10 @@
|
|
|
2
2
|
import pandas as pd
|
|
3
3
|
from pyPhases import Phase
|
|
4
4
|
from pyPhases.util import BatchProgress
|
|
5
|
-
from pyPhasesRecordloader import AnnotationNotFound, RecordLoader
|
|
5
|
+
from pyPhasesRecordloader import AnnotationNotFound, RecordLoader, AnnotationInvalid, ParseError
|
|
6
|
+
from pyPhasesML import DataversionManager
|
|
7
|
+
from SleePyPhases.PreManipulation import PreManipulation
|
|
8
|
+
|
|
6
9
|
import tqdm
|
|
7
10
|
|
|
8
11
|
|
|
@@ -12,36 +15,54 @@ class ExtractEvents(Phase):
|
|
|
12
15
|
"""
|
|
13
16
|
|
|
14
17
|
useMultiThreading = True
|
|
15
|
-
threads =
|
|
18
|
+
threads = 8
|
|
16
19
|
validatDataset = True
|
|
20
|
+
manipulationSteps = None
|
|
17
21
|
|
|
18
22
|
def getEventList(self, recordId):
|
|
19
23
|
rl = RecordLoader.get()
|
|
20
24
|
try:
|
|
21
|
-
|
|
25
|
+
eventList = rl.getEventList(recordId)
|
|
26
|
+
|
|
27
|
+
if self.manipulationSteps is not None:
|
|
28
|
+
dataManipulation = PreManipulation.getInstance(self.manipulationSteps)
|
|
29
|
+
_, eventList = dataManipulation(None, eventList)
|
|
30
|
+
|
|
31
|
+
recordDf = pd.DataFrame(eventList)
|
|
32
|
+
recordDf["recordId"] = recordId
|
|
33
|
+
return recordDf
|
|
22
34
|
except AnnotationNotFound:
|
|
23
35
|
return None
|
|
36
|
+
except AnnotationInvalid:
|
|
37
|
+
return None
|
|
38
|
+
except ParseError:
|
|
39
|
+
return None
|
|
24
40
|
|
|
25
41
|
|
|
26
42
|
def getEvents(self, records, force=False):
|
|
27
43
|
rl = RecordLoader.get()
|
|
28
|
-
|
|
44
|
+
|
|
29
45
|
bp = BatchProgress(records)
|
|
30
46
|
bp.useMultiThreading = self.useMultiThreading
|
|
31
47
|
if self.threads is not None:
|
|
32
48
|
bp.threads = self.threads
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
def combine(recordDfs, index):
|
|
33
52
|
|
|
34
|
-
|
|
35
|
-
|
|
53
|
+
# combine.df.append([e for e in events if e is not None])
|
|
54
|
+
df = pd.concat(recordDfs, ignore_index=True)
|
|
55
|
+
combine.df = pd.concat([combine.df, df], ignore_index=True)
|
|
56
|
+
# pd.concat(recordDfs, ignore_index=True)
|
|
36
57
|
# def combineFlatten(events, index):
|
|
37
58
|
# combine.events += [e for ev in events if ev is not None for e in ev]
|
|
38
59
|
# def getEventList(recordId):
|
|
39
60
|
# rl.getEventList(recordId)
|
|
40
|
-
combine.
|
|
61
|
+
combine.df = pd.DataFrame()
|
|
41
62
|
|
|
42
63
|
# bp.asyncQueue(rl.getEventList, records, combine)
|
|
43
64
|
bp.start(self.getEventList, records, combine)
|
|
44
|
-
return combine.
|
|
65
|
+
return combine.df
|
|
45
66
|
|
|
46
67
|
def appendDataframe(self, df, dfToAppend, colMap, offsetStart=None, offsetEnd=None):
|
|
47
68
|
cols = {n: [] for n in colMap}
|
|
@@ -60,42 +81,41 @@ class ExtractEvents(Phase):
|
|
|
60
81
|
|
|
61
82
|
def generateData(self, name):
|
|
62
83
|
if name == "events":
|
|
63
|
-
groupedRecords = self.project.getData("allDBRecordIds", list)
|
|
84
|
+
# groupedRecords = self.project.getData("allDBRecordIds", list)
|
|
85
|
+
dm = self.getData("dataversionmanager", DataversionManager)
|
|
86
|
+
groupedRecords = dm.groupedRecords
|
|
87
|
+
|
|
64
88
|
flatten = [r for records in groupedRecords.values() for r in records]
|
|
65
89
|
# extract the data
|
|
66
90
|
events = self.getEvents(flatten)
|
|
67
91
|
self.registerData("events", events)
|
|
68
92
|
if name == "eventDF":
|
|
69
|
-
allRecordEventsBatches = self.getData("events", list)
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
93
|
+
# allRecordEventsBatches = self.getData("events", list)
|
|
94
|
+
df = self.getData("events", pd.DataFrame)
|
|
95
|
+
# events = []
|
|
96
|
+
# recordIndex = 0
|
|
97
|
+
# for allRecordEvents in tqdm.tqdm(allRecordEventsBatches):
|
|
98
|
+
# for recordEvents in allRecordEvents:
|
|
99
|
+
# for event in recordEvents:
|
|
100
|
+
# events.append({
|
|
101
|
+
# "recordIndex": recordIndex,
|
|
102
|
+
# "name": event.name,
|
|
103
|
+
# "start": event.start,
|
|
104
|
+
# "duration": event.duration,
|
|
105
|
+
# })
|
|
106
|
+
# recordIndex += 1
|
|
107
|
+
# df = pd.DataFrame.from_dict(events)
|
|
83
108
|
df["end"] = df["start"] + df["duration"]
|
|
84
109
|
dfSleepStage = df.query("name in ['W', 'N1', 'N2', 'N3', 'R']")
|
|
85
110
|
dfArousal = df.query("name in ('arousal', 'arousal_rera')")
|
|
86
111
|
dfLM = df.query("name in ('LegMovement-Left', 'LegMovement-Right')")
|
|
87
112
|
eventResp = df.query("name in ('resp_hypopnea', 'resp_centralapnea', 'resp_obstructiveapnea', 'resp_mixedapnea', 'resp_cheynestokesbreath')")
|
|
88
113
|
|
|
89
|
-
self.
|
|
90
|
-
self.
|
|
91
|
-
self.
|
|
92
|
-
self.registerData("eventResp", eventResp)
|
|
93
|
-
|
|
94
|
-
self.appendDataframe(dfArousal, dfSleepStage, {"sleepStage": "name"}, offsetStart=-0.5, offsetEnd=0.5)
|
|
95
|
-
self.appendDataframe(dfArousal, dfLM, {"legmovements": "name"}, offsetStart=-0.5, offsetEnd=0.5)
|
|
96
|
-
self.appendDataframe(dfArousal, eventResp, {"resp_event": "name"}, offsetStart=-0.5, offsetEnd=0.5)
|
|
114
|
+
# self.appendDataframe(dfArousal, dfSleepStage, {"sleepStage": "name"}, offsetStart=-0.5, offsetEnd=0.5)
|
|
115
|
+
# self.appendDataframe(dfArousal, dfLM, {"legmovements": "name"}, offsetStart=-0.5, offsetEnd=0.5)
|
|
116
|
+
# self.appendDataframe(dfArousal, eventResp, {"resp_event": "name"}, offsetStart=-0.5, offsetEnd=0.5)
|
|
97
117
|
|
|
98
|
-
self.registerData("eventDF",
|
|
118
|
+
self.registerData("eventDF", df)
|
|
99
119
|
|
|
100
120
|
return dfArousal
|
|
101
121
|
|
|
@@ -105,10 +125,10 @@ class ExtractEvents(Phase):
|
|
|
105
125
|
offsetStart = 0
|
|
106
126
|
if offsetEnd is None:
|
|
107
127
|
offsetEnd = 0
|
|
108
|
-
for e in tqdm.tqdm(df.iloc):
|
|
128
|
+
for e in tqdm.tqdm(df.iloc, total=len(df)):
|
|
109
129
|
start = e.start + offsetStart
|
|
110
130
|
end = e.end + offsetEnd
|
|
111
|
-
possibleEvents = dfToAppend.query("
|
|
131
|
+
possibleEvents = dfToAppend.query(f"recordId == '{e.recordId}' and start < {end} and end > {start}")
|
|
112
132
|
for name in cols:
|
|
113
133
|
cols[name].append(",".join([ev[colMap[name]] for ev in possibleEvents.iloc]))
|
|
114
134
|
for name in cols:
|
|
@@ -133,58 +153,62 @@ class ExtractEvents(Phase):
|
|
|
133
153
|
|
|
134
154
|
# df = self.getData("eventDF")
|
|
135
155
|
df = self.getData("eventDF", list)
|
|
156
|
+
|
|
157
|
+
df["site"] = df["recordId"].str[:5]
|
|
136
158
|
|
|
137
159
|
modelConfigString = self.project.getDataFromName("eventDF").getTagString()
|
|
138
160
|
logPath = f"eval/{modelConfigString}/"
|
|
139
161
|
self.logPath = logPath
|
|
140
162
|
plot = Plot(logPath)
|
|
141
163
|
plot.createLogFolder()
|
|
142
|
-
|
|
164
|
+
dfArousal = df.query("name.str.contains('arousal')")
|
|
165
|
+
dfLeg = df.query("name.str.contains('Leg')")
|
|
166
|
+
plot.plotClassDistribution(dfArousal.groupby("name")["recordId"].count(), "Arousal/Rera")
|
|
143
167
|
plot.save("arousal-rera")
|
|
144
|
-
plot.plotClassDistribution(df.query("name == 'arousal'").groupby("sleepStage")["recordIndex"].count(), "sleepStage Arousal")
|
|
145
|
-
plot.save("arousal-sleepStage")
|
|
146
|
-
plot.plotClassDistribution(df.query("name == 'arousal_rera'").groupby("sleepStage")["recordIndex"].count(), "sleepStage Rera")
|
|
147
|
-
plot.save("rara-sleepStage")
|
|
148
|
-
plot.plotClassDistribution(df.query("name == 'arousal_rera'").groupby("sleepStage")["recordIndex"].count(), "sleepStage")
|
|
149
|
-
plot.save("sleepStage")
|
|
168
|
+
# plot.plotClassDistribution(df.query("name == 'arousal'").groupby("sleepStage")["recordIndex"].count(), "sleepStage Arousal")
|
|
169
|
+
# plot.save("arousal-sleepStage")
|
|
170
|
+
# plot.plotClassDistribution(df.query("name == 'arousal_rera'").groupby("sleepStage")["recordIndex"].count(), "sleepStage Rera")
|
|
171
|
+
# plot.save("rara-sleepStage")
|
|
172
|
+
# plot.plotClassDistribution(df.query("name == 'arousal_rera'").groupby("sleepStage")["recordIndex"].count(), "sleepStage")
|
|
173
|
+
# plot.save("sleepStage")
|
|
150
174
|
|
|
151
|
-
df["type"] = df.apply(lambda row: ("L" if row["legmovements"] != "" else "") + ("R" if row["resp_event"] != "" else ""), axis=1)
|
|
152
|
-
df["first_resp"] = df.apply(lambda row: row["resp_event"].split(",")[0], axis=1)
|
|
175
|
+
# df["type"] = df.apply(lambda row: ("L" if row["legmovements"] != "" else "") + ("R" if row["resp_event"] != "" else ""), axis=1)
|
|
176
|
+
# df["first_resp"] = df.apply(lambda row: row["resp_event"].split(",")[0], axis=1)
|
|
153
177
|
|
|
154
|
-
plot.plotClassDistribution(df.groupby("type")["recordIndex"].count(), "type")
|
|
155
|
-
plot.save("type")
|
|
178
|
+
# plot.plotClassDistribution(df.groupby("type")["recordIndex"].count(), "type")
|
|
179
|
+
# plot.save("type")
|
|
156
180
|
|
|
157
|
-
plot.plotClassDistribution(df.query("first_resp != ''").groupby("first_resp")["recordIndex"].count())
|
|
158
|
-
plot.save("first_resp")
|
|
181
|
+
# plot.plotClassDistribution(df.query("first_resp != ''").groupby("first_resp")["recordIndex"].count())
|
|
182
|
+
# plot.save("first_resp")
|
|
159
183
|
|
|
160
184
|
|
|
161
|
-
plot.plotClassDistribution(df.groupby("resp_event")["recordIndex"].count(), "resp_events")
|
|
162
|
-
plot.save("resp_events")
|
|
185
|
+
# plot.plotClassDistribution(df.groupby("resp_event")["recordIndex"].count(), "resp_events")
|
|
186
|
+
# plot.save("resp_events")
|
|
163
187
|
|
|
164
188
|
|
|
165
189
|
# df["duration"].plot(kind='density')
|
|
166
|
-
self.plotArousalHist(
|
|
190
|
+
self.plotArousalHist(dfArousal, "duration", "Duration")
|
|
167
191
|
plt.text(10 + 2.5, plt.ylim()[1]*0.9, "10", color='red', ha='center', va='bottom')
|
|
168
192
|
plt.axvline(x=10, color='red')
|
|
169
193
|
plot.save("duration")
|
|
170
194
|
|
|
171
|
-
self.plotArousalHist(
|
|
195
|
+
self.plotArousalHist(dfArousal.query("name == 'arousal_rera'"), "duration", "Rera Duration")
|
|
172
196
|
plt.text(10 + 2.5, plt.ylim()[1]*0.9, "10", color='red', ha='center', va='bottom')
|
|
173
197
|
plt.axvline(x=10, color='red')
|
|
174
198
|
plot.save("rera-duration")
|
|
175
199
|
|
|
176
|
-
self.plotArousalHist(
|
|
200
|
+
self.plotArousalHist(dfArousal.query("name == 'arousal'"), "duration", "Arousal Duration")
|
|
177
201
|
plt.text(10 + 2.5, plt.ylim()[1]*0.9, "10", color='red', ha='center', va='bottom')
|
|
178
202
|
plt.axvline(x=10, color='red')
|
|
179
203
|
plot.save("arousal-duration")
|
|
180
204
|
|
|
181
|
-
self.plotArousalHist(
|
|
205
|
+
self.plotArousalHist(dfArousal.query("name == 'arousal'"), "start", "Arousal Start")
|
|
182
206
|
plot.save("arousal-start")
|
|
183
207
|
|
|
184
|
-
self.plotArousalHist(
|
|
208
|
+
self.plotArousalHist(dfArousal.query("name == 'arousal_rera'"), "start", "Rera Start")
|
|
185
209
|
plot.save("rera-start")
|
|
186
210
|
|
|
187
|
-
self.plotArousalHist(
|
|
211
|
+
self.plotArousalHist(dfArousal, "start", "Start")
|
|
188
212
|
plt.gca().xaxis.set_major_formatter(ticker.FuncFormatter(lambda x, pos: f"{x/3600:.1f}h"))
|
|
189
213
|
plot.save("start")
|
|
190
214
|
|
|
@@ -166,7 +166,7 @@ class ExtractFeatures(Phase):
|
|
|
166
166
|
featureConfigs = self.getConfig("featureExtraction.features", [])
|
|
167
167
|
if featureNameDict != {}:
|
|
168
168
|
featureNames = list(featureNameDict.keys())
|
|
169
|
-
featureConfigs = [f for f in featureConfigs if f["name"] in featureNames]
|
|
169
|
+
featureConfigs = [f for f in featureConfigs if f["name"] + "_checked" in featureNames]
|
|
170
170
|
|
|
171
171
|
recordLoader = RecordLoader.get()
|
|
172
172
|
recordSignal = None
|
|
@@ -181,9 +181,10 @@ class ExtractFeatures(Phase):
|
|
|
181
181
|
|
|
182
182
|
|
|
183
183
|
for featureConfig in featureConfigs:
|
|
184
|
+
checkedCol = featureConfig["name"] + "_checked"
|
|
184
185
|
if recordSignal is None:
|
|
185
186
|
self.features.loc[recordId, "recordId"] = recordId
|
|
186
|
-
self.features.loc[recordId,
|
|
187
|
+
self.features.loc[recordId, checkedCol] = True
|
|
187
188
|
self.registerData("features", self.features)
|
|
188
189
|
return {}
|
|
189
190
|
|
|
@@ -198,7 +199,7 @@ class ExtractFeatures(Phase):
|
|
|
198
199
|
|
|
199
200
|
self.features.loc[idx, "recordId"] = recordId
|
|
200
201
|
self.features.loc[idx, "channel"] = c
|
|
201
|
-
self.features.loc[idx,
|
|
202
|
+
self.features.loc[idx, checkedCol] = True
|
|
202
203
|
|
|
203
204
|
self.registerData("features", self.features)
|
|
204
205
|
idx = (recordId, channel) if channel is not None else recordId
|
|
@@ -211,7 +212,7 @@ class ExtractFeatures(Phase):
|
|
|
211
212
|
recordIds = metadata["recordId"]
|
|
212
213
|
forceCalculation = self.getConfig("featureExtraction.force", False)
|
|
213
214
|
|
|
214
|
-
reqFeatures = {f["name"]: True for f in self.getConfig("featureExtraction.features", [])}
|
|
215
|
+
reqFeatures = {f["name"] + "_checked": True for f in self.getConfig("featureExtraction.features", [])}
|
|
215
216
|
|
|
216
217
|
recordId = self.getConfig("recordId", False)
|
|
217
218
|
recordIds = [recordId] if recordId else recordIds.iloc
|
|
@@ -11,9 +11,7 @@ from pyPhasesRecordloader.RecordLoader import ChannelsNotPresent
|
|
|
11
11
|
from SleePyPhases.DataManipulation import DataManipulation
|
|
12
12
|
from SleePyPhases.FeatureExtraction import FeatureExtraction
|
|
13
13
|
from SleePyPhases.PreManipulation import PreManipulation
|
|
14
|
-
from SleePyPhases.PSGEventManager import PSGEventManager
|
|
15
14
|
from SleePyPhases.SignalPreprocessing import SignalPreprocessing
|
|
16
|
-
from SleePyPhases.phases.Extract import RecordProcessor
|
|
17
15
|
|
|
18
16
|
|
|
19
17
|
class Predict(Phase):
|
|
@@ -129,11 +127,12 @@ class Predict(Phase):
|
|
|
129
127
|
|
|
130
128
|
return recordSignal
|
|
131
129
|
|
|
132
|
-
def preprocessSignal(self, recordSignal):
|
|
130
|
+
def preprocessSignal(self, recordSignal, targetSignals=None):
|
|
133
131
|
"""Preprocess the signal for model input."""
|
|
134
132
|
preprocessingConfig = self.getConfig("preprocessing")
|
|
135
133
|
targetFrequency = preprocessingConfig["targetFrequency"]
|
|
136
|
-
targetSignals = preprocessingConfig["targetChannels"]
|
|
134
|
+
targetSignals = preprocessingConfig["targetChannels"] if targetSignals is None else None
|
|
135
|
+
|
|
137
136
|
featureChannels = preprocessingConfig.get("featureChannels", [])
|
|
138
137
|
|
|
139
138
|
recordSignal.targetFrequency = targetFrequency
|
|
@@ -165,31 +164,40 @@ class Predict(Phase):
|
|
|
165
164
|
|
|
166
165
|
# Load and preprocess the signal
|
|
167
166
|
recordSignal = self.getSignalFromFile(inputFile, channelMapping)
|
|
167
|
+
|
|
168
|
+
return self.predictRecordSignal(recordSignal)
|
|
169
|
+
|
|
170
|
+
def predictRecordSignal(self, recordSignal, dummyY=None):
|
|
168
171
|
processedSignal = self.preprocessSignal(recordSignal)
|
|
169
172
|
|
|
170
|
-
# Prepare input for model
|
|
171
|
-
if len(processedSignal.shape) == 2:
|
|
172
|
-
|
|
173
|
+
# # Prepare input for model
|
|
174
|
+
# if len(processedSignal.shape) == 2:
|
|
175
|
+
# processedSignal = processedSignal.reshape([1] + list(processedSignal.shape))
|
|
173
176
|
|
|
174
177
|
# Apply segment manipulation for evaluation
|
|
175
178
|
manipulationSteps = self.getConfig("segmentManipulationEval", [])
|
|
176
179
|
da = DataManipulation.getInstance(manipulationSteps, "test", self.project.config)
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
180
|
+
dummyY = np.array([0]).reshape(1, -1, 1) if dummyY is None else dummyY
|
|
181
|
+
processedSignal, dummyY = da((processedSignal, dummyY)) # Dummy label
|
|
182
|
+
|
|
183
|
+
with self.project:
|
|
184
|
+
overwriteConfig = self.getConfig("predict", {})
|
|
185
|
+
self.project.updateConfig(overwriteConfig)
|
|
186
|
+
|
|
187
|
+
# Load model with optional custom weights
|
|
188
|
+
weightsPath = self.getConfig("predict.weights", False)
|
|
189
|
+
model = self.getModel(weightsPath)
|
|
190
|
+
|
|
191
|
+
# Run prediction
|
|
192
|
+
prediction = model.predict(processedSignal, returnNumpy=True)
|
|
193
|
+
|
|
194
|
+
# Apply post-prediction manipulation if configured
|
|
195
|
+
manipulationAfterPredict = self.getConfig("manipulationAfterPredict", False)
|
|
196
|
+
if manipulationAfterPredict:
|
|
197
|
+
threshold = self.getConfig("fixedThreshold", False)
|
|
198
|
+
threshold = threshold or self.getData("threshold", list, generate=False)
|
|
199
|
+
da = DataManipulation.getInstance(manipulationAfterPredict, "test", self.project.config, threshold=threshold)
|
|
200
|
+
prediction, _ = da((prediction, dummyY))
|
|
193
201
|
|
|
194
202
|
return prediction
|
|
195
203
|
|
|
@@ -2,7 +2,7 @@ from pathlib import Path
|
|
|
2
2
|
|
|
3
3
|
from pyPhases import Phase
|
|
4
4
|
from pyPhasesML import DataversionManager, Scorer
|
|
5
|
-
from pyPhasesRecordloader import RecordLoader
|
|
5
|
+
from pyPhasesRecordloader import RecordLoader, RecordLoaderNotFound
|
|
6
6
|
|
|
7
7
|
|
|
8
8
|
|
|
@@ -18,8 +18,11 @@ class Setup(Phase):
|
|
|
18
18
|
self.setConfig("segmentManipulationEval", defaultValidationbatchManipulation)
|
|
19
19
|
|
|
20
20
|
preprocessingConfig = self.getConfig("preprocessing", {})
|
|
21
|
-
if "combineChannels" in preprocessingConfig:
|
|
22
|
-
|
|
21
|
+
if "combineChannels" in preprocessingConfig and self.getConfig("useLoader", False):
|
|
22
|
+
try:
|
|
23
|
+
RecordLoader.get().addCombinedChannels(preprocessingConfig["combineChannels"])
|
|
24
|
+
except RecordLoaderNotFound as e:
|
|
25
|
+
self.logError(e)
|
|
23
26
|
|
|
24
27
|
|
|
25
28
|
# default fold is 0 if none is set
|
|
@@ -101,8 +104,6 @@ class Setup(Phase):
|
|
|
101
104
|
dm.addSplitByRemaining("validation", valSplit) # , remainingSplit=trainingSlice
|
|
102
105
|
dm.addSplitByRemaining("training", 1)
|
|
103
106
|
|
|
104
|
-
|
|
105
|
-
|
|
106
107
|
if "test" not in dm.splits:
|
|
107
108
|
dm.addSplitByRemaining("test", testSplit)
|
|
108
109
|
|
|
@@ -5,7 +5,7 @@ with open("README.md", "r") as fh:
|
|
|
5
5
|
|
|
6
6
|
setuptools.setup(
|
|
7
7
|
name="SleePyPhases",
|
|
8
|
-
version="v0.6.
|
|
8
|
+
version="v0.6.2"[1:],
|
|
9
9
|
author="Franz Ehrlich",
|
|
10
10
|
author_email="fehrlichd@gmail.com",
|
|
11
11
|
description="A framwork for creating deep learning pipelines for sleep data",
|
|
@@ -3,7 +3,7 @@
|
|
|
3
3
|
# import numpy as np
|
|
4
4
|
# from pyPhasesRecordloader import Event
|
|
5
5
|
|
|
6
|
-
# from
|
|
6
|
+
# from SleepHarmonizer.PSGEventManager import PSGEventManager
|
|
7
7
|
# from SleePyPhases.RecordFeatureExtraction import RecordFeatureExtraction
|
|
8
8
|
|
|
9
9
|
|
|
@@ -1,137 +0,0 @@
|
|
|
1
|
-
from pyPhasesRecordloader import EventManager
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
class PSGEventManager(EventManager):
|
|
5
|
-
INDEX_WAKE = 2
|
|
6
|
-
INDEX_NREM1 = 4
|
|
7
|
-
INDEX_NREM2 = 8
|
|
8
|
-
INDEX_NREM3 = 16
|
|
9
|
-
INDEX_REM = 32
|
|
10
|
-
|
|
11
|
-
INDEX_APNEA_OBSTRUCTIVE = 2
|
|
12
|
-
INDEX_APNEA_HYPO = 4
|
|
13
|
-
INDEX_APNEA_CENTRAL = 8
|
|
14
|
-
INDEX_APNEA_MIXED = 16
|
|
15
|
-
INDEX_APNEA_PARTIALOBSTRUCTIVE = 32
|
|
16
|
-
INDEX_APNEA_HYPOVENTILATION = 64
|
|
17
|
-
INDEX_APNEA_CHEYNESTOKESBREATH = 128
|
|
18
|
-
|
|
19
|
-
INDEX_AROUSAL_RERA = 2
|
|
20
|
-
INDEX_AROUSAL_SPONTANEOUS = 4
|
|
21
|
-
INDEX_AROUSAL_PLM = 8
|
|
22
|
-
INDEX_AROUSAL_SNORE = 16
|
|
23
|
-
INDEX_AROUSAL_BRUXISM = 32
|
|
24
|
-
INDEX_AROUSAL_NOISE = 64
|
|
25
|
-
INDEX_AROUSAL = 128
|
|
26
|
-
INDEX_AROUSAL_ASDA = 256
|
|
27
|
-
INDEX_AROUSAL_CHIN = 512
|
|
28
|
-
INDEX_AROUSAL_LIMB = 8
|
|
29
|
-
|
|
30
|
-
INDEX_LEGMOVEMENT = 8
|
|
31
|
-
INDEX_LEGMOVEMENT_LEFT = 2
|
|
32
|
-
INDEX_LEGMOVEMENT_RIGHT = 4
|
|
33
|
-
|
|
34
|
-
def __init__(self):
|
|
35
|
-
eventGroups = {
|
|
36
|
-
"sleepStage": ["undefined", "W", "N1", "N2", "N3", "R"],
|
|
37
|
-
"bodyposition": [
|
|
38
|
-
"undefinedBodyPosition",
|
|
39
|
-
"Up",
|
|
40
|
-
"Supine",
|
|
41
|
-
"Left",
|
|
42
|
-
"Prone",
|
|
43
|
-
"Right",
|
|
44
|
-
],
|
|
45
|
-
"apnea": [
|
|
46
|
-
"noneApnea",
|
|
47
|
-
"resp_obstructiveapnea",
|
|
48
|
-
"resp_hypopnea",
|
|
49
|
-
"resp_centralapnea",
|
|
50
|
-
"resp_mixedapnea",
|
|
51
|
-
"resp_partialobstructive",
|
|
52
|
-
"resp_hypoventilation",
|
|
53
|
-
"resp_cheynestokesbreath",
|
|
54
|
-
"resp_hypopnea_obstructive",
|
|
55
|
-
"resp_hypopnea_central",
|
|
56
|
-
"resp_rera" # flow limitation (>10 sec)
|
|
57
|
-
],
|
|
58
|
-
"arousal": [
|
|
59
|
-
"noneArousal",
|
|
60
|
-
"arousal_rera", # the actual arousal
|
|
61
|
-
"arousal_spontaneous",
|
|
62
|
-
"arousal_plm",
|
|
63
|
-
"arousal_snore",
|
|
64
|
-
"arousal_bruxism",
|
|
65
|
-
"arousal_noise",
|
|
66
|
-
"arousal",
|
|
67
|
-
"arousal_asda", # shhs/mros: American Sleep Disorders Association
|
|
68
|
-
"arousal_chin", # shhs
|
|
69
|
-
"arousal_limb",
|
|
70
|
-
"arousal_respiratory",
|
|
71
|
-
],
|
|
72
|
-
"grapho": [
|
|
73
|
-
"no-grapho",
|
|
74
|
-
"AlphaActivity-eeg",
|
|
75
|
-
"RapidEyeMovement",
|
|
76
|
-
"SlowEyeMovement",
|
|
77
|
-
"Spindle-eeg",
|
|
78
|
-
],
|
|
79
|
-
"fail": [ # maximal 31 for 32bit integer
|
|
80
|
-
"workingSignal",
|
|
81
|
-
"fail-channel0",
|
|
82
|
-
"fail-channel1",
|
|
83
|
-
"fail-channel2",
|
|
84
|
-
"fail-channel3",
|
|
85
|
-
"fail-channel4",
|
|
86
|
-
"fail-channel5",
|
|
87
|
-
"fail-channel6",
|
|
88
|
-
"fail-channel7",
|
|
89
|
-
"fail-channel8",
|
|
90
|
-
"fail-channel9",
|
|
91
|
-
"fail-channel10",
|
|
92
|
-
"fail-channel11",
|
|
93
|
-
"fail-channel12",
|
|
94
|
-
"fail-channel13",
|
|
95
|
-
"fail-channel14",
|
|
96
|
-
"fail-channel15",
|
|
97
|
-
"fail-channel16",
|
|
98
|
-
"fail-channel17",
|
|
99
|
-
"fail-channel18",
|
|
100
|
-
"fail-channel19",
|
|
101
|
-
"fail-channel20",
|
|
102
|
-
"fail-channel21",
|
|
103
|
-
"fail-channel22",
|
|
104
|
-
"fail-channel23",
|
|
105
|
-
"fail-channel24",
|
|
106
|
-
"fail-channel25",
|
|
107
|
-
"fail-channel26",
|
|
108
|
-
"fail-channel27",
|
|
109
|
-
"fail-channel28",
|
|
110
|
-
"fail-channel29",
|
|
111
|
-
],
|
|
112
|
-
"limb": [
|
|
113
|
-
"noneLegmovement",
|
|
114
|
-
"LegMovement-Left",
|
|
115
|
-
"LegMovement-Right",
|
|
116
|
-
"LegMovement",
|
|
117
|
-
"PLM",
|
|
118
|
-
"PLM-Left",
|
|
119
|
-
"PLM-Right",
|
|
120
|
-
],
|
|
121
|
-
"spo2": [
|
|
122
|
-
"spo2_desaturation",
|
|
123
|
-
"spo2_meanPerEpoch",
|
|
124
|
-
"spo2_minPerEpoch",
|
|
125
|
-
"spo2_Lower90",
|
|
126
|
-
"spo2_Lower80",
|
|
127
|
-
],
|
|
128
|
-
"cardiac": [
|
|
129
|
-
"hr_rise",
|
|
130
|
-
"sinus_tachycardia",
|
|
131
|
-
],
|
|
132
|
-
"light": [
|
|
133
|
-
"lightOff",
|
|
134
|
-
"lightOn",
|
|
135
|
-
],
|
|
136
|
-
}
|
|
137
|
-
super().__init__(eventGroups)
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|