SleePyPhases 0.6.2__tar.gz → 0.7.0__tar.gz

{sleepyphases-0.6.2 → sleepyphases-0.7.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: SleePyPhases
-Version: 0.6.2
+Version: 0.7.0
 Summary: A framwork for creating deep learning pipelines for sleep data
 Home-page: https://gitlab.com/sleep-is-all-you-need/sleepyphases
 Author: Franz Ehrlich

{sleepyphases-0.6.2 → sleepyphases-0.7.0}/SleePyPhases/MultiScorer.py

@@ -2,7 +2,7 @@ import numpy as np
 from pyPhasesML.scorer.Scorer import Scorer
 
 class MultiScorer(Scorer):
-    def __init__(self, numClasses, metrics, scorerNames=[], ignoreClasses=None, classNames=None, scorerClasses=None) -> None:
+    def __init__(self, numClasses, metrics, scorerNames=[], ignoreClasses=None, classNames=None, scorerClasses=None, multilabel=None) -> None:
 
         ignoreClasses = ignoreClasses or [-1]
 
@@ -13,11 +13,17 @@ class MultiScorer(Scorer):
         self.scorerNames = scorerNames
         self.metricDefinitions = {}
 
+        # Normalise multilabel to a per-scorer list
+        if multilabel is None:
+            multilabel = [False] * len(numClasses)
+        elif isinstance(multilabel, bool):
+            multilabel = [multilabel] * len(numClasses)
+
         metrics = metrics if isinstance(metrics[0], list) else [metrics for _ in range(len(numClasses))]
 
         for i, (numClasses, scorerName) in enumerate(zip(numClasses, scorerNames)):
             scorerClass = scorerClasses[i] if scorerClasses is not None else Scorer
-            s = scorerClass(numClasses)
+            s = scorerClass(numClasses, multilabel=multilabel[i])
             s.trace = True
             s.ignoreClasses = ignoreClasses
             s.numClasses = numClasses if numClasses > 1 else 2
@@ -96,29 +102,39 @@ class MultiScorer(Scorer):
 
         return scorer.score(truth, prediction, trace=trace)
     
-    def prettyPrintConfusionMatrix(self, confusion_matrix):
-        num_rows, num_cols = confusion_matrix.shape
-        max_value_length = max(len(str(confusion_matrix.max())), len(str(confusion_matrix.min())))
-        separator = "-" * ((max_value_length + 2) * num_cols)
-
+    def _print2DConfusionMatrix(self, cm):
+        cm = np.array(cm)
+        num_rows, num_cols = cm.shape
+        max_value_length = max(len(str(int(cm.max()))), len(str(int(cm.min()))))
+        separator = "-" * ((max_value_length + 3) * num_cols)
         rows = []
         for i in range(num_rows):
-            row_data = [f"{int(confusion_matrix[i, j]):>{max_value_length}}" for j in range(num_cols)]
+            row_data = [f"{int(cm[i, j]):>{max_value_length}}" for j in range(num_cols)]
             rows.append(" | ".join(row_data))
-
         print(separator)
         print("\n".join(rows))
         print(separator)
+
+    def prettyPrintConfusionMatrix(self, confusion_matrix, classNames=None):
+        cm = np.array(confusion_matrix)
+        if cm.ndim == 3:
+            # multilabel: shape (n_classes, 2, 2)
+            for i, cm_i in enumerate(cm):
+                label = classNames[i] if classNames is not None else f"Class {i}"
+                print(f"  [{label}]")
+                self._print2DConfusionMatrix(cm_i)
+        else:
+            self._print2DConfusionMatrix(cm)
         
     def scoreAllRecords(self):
         results = {}
         self.recordResult = {}
         for name, s in self.scorer.items():
             r = s.scoreAllRecords()
-            results.update({f"{m}_{name}": v for m,v in r.items()})
+            results.update({f"{m}_{name}": v for m, v in r.items()})
             if "confusion" in r:
                 print(f"######{name}########")
-                self.prettyPrintConfusionMatrix(r["confusion"] )
+                self.prettyPrintConfusionMatrix(r["confusion"], classNames=s.classNames)
             self.recordResult[name] = s.recordResult
         self.results = results
 

{sleepyphases-0.6.2 → sleepyphases-0.7.0}/SleePyPhases/Plugin.py

@@ -24,6 +24,8 @@ from pyPhasesML.exporter.MemmapRecordExporter import MemmapRecordExporter
 
 
 class Plugin(PluginAdapter):
+    defaultConfig = "config.yaml"
+
     def __init__(self, project: Project, options=None):
         super().__init__(project, options)
 

sleepyphases-0.7.0/SleePyPhases/SleePyPhases.py

@@ -0,0 +1,34 @@
+from phases import DefaultProject
+from pyPhases import Project
+from pyPhasesRecordloader import RecordSignal
+
+
+class SleePyPhases(Project):
+    """
+    This is the main class for the SleePyPhases project. It defines the phases that will be executed in the project.
+    """
+
+    @staticmethod
+    def create(configFiles=None, plugins=None):
+        if plugins is None:
+            plugins = []
+
+        defaultPlugins = [
+            "SleePyPhases",
+            "pyPhasesML",
+            "pyPhasesRecordloader",
+        ]
+        project = DefaultProject.create(configFiles, plugins + defaultPlugins, project=SleePyPhases())
+
+        return project
+    
+
+    def predictRecordSignal(self, recordSignal: RecordSignal):
+        from SleePyPhases.phases.Predict import Predict
+        predict: Predict = self.getPhase("Predict")
+        return predict.predictRecordSignal(recordSignal)
+    
+    def predictFromFile(self, filepath: str):
+        from SleePyPhases.phases.Predict import Predict
+        predict: Predict = self.getPhase("Predict")
+        return predict.predict(filepath)

sleepyphases-0.7.0/SleePyPhases/config.yaml

@@ -0,0 +1,167 @@
+# isFullConfig: True
+
+# phases:
+#   - name: Setup
+#     description: Prepare the project and compute neccessary config values
+#     exports:
+#       - dataversionmanager
+#   - name: Extract
+#     description: Preprocess and save the raw data
+#     exports:
+#       - removedRecordIds
+#       - data-processed
+#       - data-features
+#       - record-preprocessed
+#   - name: BuildDataset
+#     description: Combine datasets and manipulate the data live for the model
+#     exports:
+#       # - dataset-trainval 
+#       - dataset-training
+#       - dataset-validation
+#       - dataset-test
+#       - dataset-bySplit
+#       - data-records
+#       - record-data
+#   - name: Training
+#     description: Train my model
+#     exports:
+#       - modelState
+#       - modelStateConfig
+#   - name: Validation
+#     description: Run Model on validation data for optimization
+#   - name: ThresholdOptimisation
+#     description: Threshold optimisation by optimising the threshold metric on validation data
+#     exports:
+#       - threshold
+#       - validationResult
+#   - name: Eval
+#     description: Evalutate my model against the test data
+#     exports:
+#       - evalResults
+#       - eventResults
+#   - name: EvalReport
+#     description: generate evaluation report
+#   - name: DataAnalysis
+#     description: Get class distribution and other statistics
+#     exports:
+#       - datasetstats
+#   - name: DataReport
+#   - name: ExtractEvents
+#     exports:
+#       - events
+#       - eventDF
+#   - name: ExportMetadataToMeticalDB
+#   - name: ExtractFeatures
+#     exports:
+#     - features
+#   - name: Predict
+#     description: Execute trained model on a given EDF file for prediction
+#     exports:
+#       - prediction
+
+
+# data:
+#   # - name: data-records
+#   - name: features
+#     dependsOn:
+#     - metadata
+#   - name: events
+#     dependsOn:
+#       - allDBRecordIds
+#   - name: eventDF
+#     dependsOn:
+#       - allDBRecordIds
+#   - name: data-processed
+#     dependsOn:
+#       - allDBRecordIds
+#       - preprocessing
+#       - labelChannels
+#   - name: record-preprocessed
+#     dependsOn: 
+#       - recordId
+#       - data-processed
+#   - name: record-data
+#   - name: data-features
+#     dependsOn:
+#       - data-processed
+#   - name: dataversionmanager
+#     dependsOn:
+#       - data-processed
+#       - datafold
+#       - dataversion
+#   - name: removedRecordIds
+#     dependsOn:
+#       - data-processed
+#   - name: modelState
+#     dependsOn:
+#       - dataset-bySplit
+#       - modelName
+#       - model
+#       - inputShape
+#       - segmentManipulation
+#       - trainingParameter
+#       - fold
+#   - name: modelStateConfig
+#     dependsOn:
+#       - modelState
+#   - name: threshold
+#     dependsOn:
+#       - modelState
+#       - eventEval
+#       - thresholdMetric
+#       - fixedThreshold
+#   - name: validationResult
+#     dependsOn:
+#       - modelState
+#       - segmentManipulationEval
+#       - thresholdMetric
+#       - optimizeOn
+#   - name: evalResults
+#     dependsOn:
+#       - threshold
+#       - evalOn
+#       - manipulationAfterPredict
+#       - segmentManipulationEval # is applied to the test dataset
+#   - name: eventResults
+#     dependsOn:
+#       - evalResults
+#       - eventEval
+#       - eval.clinicalMetrics
+#   - name: dataset-bySplit
+#     dependsOn:
+#       # does not require the split, since its encoded in the name and will not be saved
+#       - dataversionmanager
+#   - name: dataset-training
+#     dependsOn:
+#       - dataversionmanager
+#   - name: dataset-validation
+#     dependsOn:
+#       - dataversionmanager
+#   # - name: dataset-trainval
+#   #   dependsOn:
+#   #     - dataset-bySplit
+#   #     - labelChannels
+#   #     - fold
+#   - name: dataset-test
+#     dependsOn:
+#       - dataversionmanager
+#       - evalOn
+#   - name: datasetstats
+#     dependsOn:
+#       - data-processed
+#       - data-features
+#   - name: prediction
+#     dependsOn:
+#       - predict.inputFile
+#       - predict.weights
+#       - predict.recordLoader
+#       - predict.channelMapping
+
+# config:
+
+predict:
+  inputFile: null
+  channelMapping: {}
+  weights: null
+
+inputShape: null

{sleepyphases-0.6.2 → sleepyphases-0.7.0}/SleePyPhases/phases/BuildDataset.py

@@ -20,10 +20,11 @@ class BuildDataset(Phase):
         splitName = self.getConfig("datasetSplit")
 
         datasetVersions = self.getConfig("datafold.datasets", {"config": {}})
-        datasetVersions = datasetVersions if isinstance(datasetVersions, list) else [datasetVersions]
+        datasetVersions = datasetVersions if isinstance(datasetVersions, list) else [{"config": {}}]
 
         seed = self.getConfig("datafold.seed", 2024)
-        multiSrcMap = MultiSourceMap(seed=None)
+        shouldShuffle = self.getConfig("datafold.shuffle", False)
+        multiSrcMap = MultiSourceMap(seed=seed)
         recordMetadata = []
         for sourceVersion in datasetVersions:
             overwriteConfig = sourceVersion["config"]
@@ -46,7 +47,7 @@ class BuildDataset(Phase):
 
                 recordIds = dm.getRecordsForSplit(splitName)
                 recordIndexes = [allRecordIds.index(r) for r in recordIds]
-                recordMetadata = recordMetadata + [metadataMap[r] for r in recordIds]
+                recordMetadata = [metadataMap[r] for r in allRecordIds] # unshuffled
 
                 # handle segment wise data
                 recordLengthsInSegments = None
@@ -67,40 +68,34 @@ class BuildDataset(Phase):
                     )
 
                 dataset = DatasetXY(dataExporterSignals, dataExporterFeatures)
-                recordSet = RecordMap(dataset, recordIndexes, mappingLengths=recordLengthsInSegments)
 
-                multiSrcMap.addRecordMap(recordSet)
+                useConfigName = "segmentManipulationEval" if splitName != "training" else "segmentManipulation"
+                manipulationSteps = self.getConfig(useConfigName)
+                segmentManipulation = DataManipulation.getInstance(
+                    manipulationSteps, splitName, self.project.config, recordMetadata=recordMetadata
+                )
 
-        # shuffle recordings or segments
-        if self.getConfig("datafold.shuffle", False):
-            multiSrcMap.shuffle()
-            # use same seed to shuffle the metadata
-            np.random.seed(seed)
-            np.random.shuffle(recordMetadata)
+                manipulator = Manipulator(dataset, segmentManipulation)
+                
+                recordSet = RecordMap(manipulator, recordIndexes, mappingLengths=recordLengthsInSegments)
 
-        return multiSrcMap, recordMetadata
+                multiSrcMap.addRecordMap(recordSet)
 
-    def buildDatasetManipulatorBySplit(self):
-        split = self.getConfig("datasetSplit", "training")
+        if shouldShuffle:
+            multiSrcMap.shuffle()
 
-        dataRecords, recordMetaData = self.buildDataRecords()
+        return multiSrcMap
 
-        useConfigName = "segmentManipulationEval" if split != "training" else "segmentManipulation"
-        manipulationSteps = self.getConfig(useConfigName)
-        segmentManipulation = DataManipulation.getInstance(
-            manipulationSteps, split, self.project.config, recordMetadata=recordMetaData
-        )
-        return Manipulator(dataRecords, segmentManipulation)
 
     def buildDatasetBySplit(self):
         split = self.getConfig("datasetSplit")
         batchSize = self.getConfig("trainingParameter.batchSize") if split != "test" else self.getConfig("eval.batchSize")
         buildConfig = self.getConfig("BuildDataset", {"useMultiThreading": True})
 
-        manipulator = self.buildDatasetManipulatorBySplit()
+        datasets = self.buildDataRecords()
 
         dataloader = DataLoader.build(
-            manipulator,
+            datasets,
             preload=buildConfig["useMultiThreading"],
             batchSize=batchSize,
             shuffle=self.getConfig("trainingParameter.shuffle", False),

{sleepyphases-0.6.2 → sleepyphases-0.7.0}/SleePyPhases/phases/Extract.py

@@ -93,7 +93,7 @@ class RecordProcessor:
             signal = signal.reshape(-1, 1)
             labelChannels.append(signal)
 
-        return np.concatenate(labelChannels, axis=1)
+        return np.concatenate(labelChannels, axis=1) if labelChannels else np.zeros((0, 0))
 
     @staticmethod
     def tailorToSleepScoring(recordSignal, events, useSPT=False, fillLastSleepstage=False, addOffset=None, cutToAnnotation=False):
@@ -169,10 +169,10 @@ class RecordProcessor:
         try:
             recordSignal, events = self.recordLoader.loadRecord(recordId)
 
-            targetSignals = self.preProcessingConfig["targetChannels"]
-            targetFrequency = self.preProcessingConfig["targetFrequency"]
-            labelFrequency = self.preProcessingConfig["labelFrequency"]
-            featureChannels = self.preProcessingConfig["featureChannels"]
+            targetSignals = self.preProcessingConfig["targetChannels"] if "targetChannels" in self.preProcessingConfig else []
+            targetFrequency = self.preProcessingConfig["targetFrequency"] if "targetFrequency" in self.preProcessingConfig else []
+            labelFrequency = self.preProcessingConfig["labelFrequency"] if "labelFrequency" in self.preProcessingConfig else []
+            featureChannels = self.preProcessingConfig["featureChannels"] if "featureChannels" in self.preProcessingConfig else []
             recordSignal.targetFrequency = targetFrequency
 
             # tailor to targetsignals
@@ -233,7 +233,7 @@ class Extract(Phase):
             preProcessingConfig=preprocessingConfig,
             signalProcessing=SignalPreprocessing.getInstance(preprocessingConfig),
             eventManager=PSGEventManager(),
-            labelChannels=self.getConfig("labelChannels"),
+            labelChannels=self.getConfig("labelChannels", []),
             featureExtraction=featureExtraction,
             preDataManipulation=preDataManipulation,
             project=self.project,
@@ -243,7 +243,7 @@ class Extract(Phase):
     def extractRecord(self, recordId):
         
         processRecord = self.getRecordProcessor()
-        result = processRecord(recordId)      
+        result = processRecord(recordId)
         if result is None:
             raise ChannelsNotPresent(processRecord.channelNotPresent)
         return result

{sleepyphases-0.6.2 → sleepyphases-0.7.0}/SleePyPhases/phases/Predict.py

@@ -98,7 +98,7 @@ class Predict(Phase):
         # Apply channel mapping if provided
         if channelMapping:
             recordLoader.chanelNameAliasMap = {v: k for k, v in channelMapping.items()}
-        
+        recordLoader.targetSignals = expectedChannels
         try:
             recordSignal = recordLoader.loadSignal(inputFile)
         except ChannelsNotPresent as e:
@@ -170,26 +170,22 @@ class Predict(Phase):
     def predictRecordSignal(self, recordSignal, dummyY=None):
         processedSignal = self.preprocessSignal(recordSignal)
 
-        # # Prepare input for model
-        # if len(processedSignal.shape) == 2:
-        #     processedSignal = processedSignal.reshape([1] + list(processedSignal.shape))
-
-        # Apply segment manipulation for evaluation
-        manipulationSteps = self.getConfig("segmentManipulationEval", [])
-        da = DataManipulation.getInstance(manipulationSteps, "test", self.project.config)
-        dummyY = np.array([0]).reshape(1, -1, 1) if  dummyY is None else dummyY
-        processedSignal, dummyY = da((processedSignal, dummyY))  # Dummy label
-
         with self.project:
             overwriteConfig = self.getConfig("predict", {})
             self.project.updateConfig(overwriteConfig)
 
+            # Apply segment manipulation for evaluation
+            manipulationSteps = self.getConfig("segmentManipulationEval", [])
+            da = DataManipulation.getInstance(manipulationSteps, "test", self.project.config)
+            dummyY = np.array([0]).reshape(1, -1, 1) if  dummyY is None else dummyY
+            processedSignal, dummyY = da((processedSignal, dummyY))  # Dummy label
+
             # Load model with optional custom weights
             weightsPath = self.getConfig("predict.weights", False)
             model = self.getModel(weightsPath)
 
             # Run prediction
-            prediction = model.predict(processedSignal, returnNumpy=True)
+            prediction = model.predict(processedSignal, returnNumpy=False)
 
             # Apply post-prediction manipulation if configured
             manipulationAfterPredict = self.getConfig("manipulationAfterPredict", False)

{sleepyphases-0.6.2 → sleepyphases-0.7.0}/SleePyPhases/phases/Setup.py

@@ -71,7 +71,7 @@ class Setup(Phase):
         
         # create non manual splits, if not specified
         valSplit = self.getConfig("validationSplit", 0.2)
-        testSplit = self.getConfig("testSplit", 0.2)
+        testSplit = self.getConfig("testSplit", 0)
 
         dm = DataversionManager(groupedRecords, seed=seed)
 
@@ -104,7 +104,7 @@ class Setup(Phase):
             dm.addSplitByRemaining("validation", valSplit) # , remainingSplit=trainingSlice
             dm.addSplitByRemaining("training", 1)
 
-        if "test" not in dm.splits:
+        if "test" not in dm.splits and testSplit > 0:
             dm.addSplitByRemaining("test", testSplit)
 
         if "validation" not in dm.splits:

{sleepyphases-0.6.2 → sleepyphases-0.7.0}/SleePyPhases.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: SleePyPhases
-Version: 0.6.2
+Version: 0.7.0
 Summary: A framwork for creating deep learning pipelines for sleep data
 Home-page: https://gitlab.com/sleep-is-all-you-need/sleepyphases
 Author: Franz Ehrlich

{sleepyphases-0.6.2 → sleepyphases-0.7.0}/SleePyPhases.egg-info/SOURCES.txt

@@ -12,8 +12,10 @@ SleePyPhases/PreManipulation.py
 SleePyPhases/RecordFeatureExtraction.py
 SleePyPhases/Reporter.py
 SleePyPhases/SignalPreprocessing.py
+SleePyPhases/SleePyPhases.py
 SleePyPhases/SleepMetaData.py
 SleePyPhases/__init__.py
+SleePyPhases/config.yaml
 SleePyPhases.egg-info/PKG-INFO
 SleePyPhases.egg-info/SOURCES.txt
 SleePyPhases.egg-info/dependency_links.txt
@@ -46,6 +48,7 @@ tests/conftest.py
 tests/unit/__init__.py
 tests/unit/test_BuildDataset.py
 tests/unit/test_BuildDatasetFolded.py
+tests/unit/test_BuildDatasetMetadataSync.py
 tests/unit/test_BuildDatasetSegments.py
 tests/unit/test_DataManipulation.py
 tests/unit/test_Eval.py

{sleepyphases-0.6.2 → sleepyphases-0.7.0}/setup.py

@@ -5,7 +5,7 @@ with open("README.md", "r") as fh:
 
 setuptools.setup(
     name="SleePyPhases",
-    version="v0.6.2"[1:],
+    version="v0.7.0"[1:],
     author="Franz Ehrlich",
     author_email="fehrlichd@gmail.com",
     description="A framwork for creating deep learning pipelines for sleep data",

{sleepyphases-0.6.2 → sleepyphases-0.7.0}/tests/__init__.py

@@ -1,4 +1,15 @@
 # Setup project for unittest discovery
+import sys
+import os
+
+# Ensure SleePyPhases is imported as a package before loadClass adds
+# the inner directory to sys.path (which would shadow the package with SleePyPhases.py)
+_cwd = os.getcwd()
+_inner = os.path.join(_cwd, "SleePyPhases")
+if _inner in sys.path:
+    sys.path.remove(_inner)
+import SleePyPhases  # noqa: E402,F401 — force package registration
+
 from phases.commands.test import Test as PhasesTest
 from pyPhases.test import TestCase
 

sleepyphases-0.7.0/tests/unit/test_BuildDatasetMetadataSync.py

@@ -0,0 +1,261 @@
+"""
+Acceptance tests: verify that recordMetadata stays in sync with the actual
+data items throughout the full pyPhases pipeline – including datafold with
+multiple datasets, shuffle, and deterministic re-loading.
+
+Scenario
+--------
+- 3 datasets via ``datafold.datasets``, each contributing 5 recordings
+  (recordIds "d0-0"…"d0-4", "d1-0"…"d1-4", "d2-0"…"d2-4").
+- Each recording's X data embeds the *flat record index* so we can tell
+  which record a data item came from.
+- A custom DataManipulation subclass (injected via Swappable.setClass)
+  adds a ``checkMetadata`` step that, for every item, records what
+  ``self.recordMetadata[self.currentIndex]`` returns versus the recordId
+  derivable from the data payload.
+- We pull data through ``project.getData("dataset-training")`` — the
+  same code path the real Training phase uses.
+"""
+
+from unittest.mock import MagicMock
+
+import numpy as np
+import pandas as pd
+
+from pyPhases.test.Mocks import OverwriteConfig
+from pyPhases.test.TestCase import TestCase
+from pyPhasesRecordloader import RecordLoader
+
+from SleePyPhases.DataManipulation import DataManipulation
+from SleePyPhases.phases.BuildDataset import BuildDataset
+
+
+# ---------------------------------------------------------------------------
+# Helpers
+# ---------------------------------------------------------------------------
+
+# Accumulates (index, metadataRecordId, dataValue) triples that the custom
+# manipulation step records while the pipeline iterates.
+_metadata_checks: list = []
+
+
+class CheckingDataManipulation(DataManipulation):
+    """DataManipulation subclass with an extra ``checkMetadata`` step.
+
+    For every item it reads ``self.recordMetadata[self.currentIndex]`` and
+    compares the ``recordId`` to the flat-index value embedded in the X
+    data (after ``addBatchDimension`` + ``fixedSize`` centering to pos 5).
+    """
+
+    def checkMetadata(self, X, Y):
+        meta = self.recordMetadata[self.currentIndex]
+        metadataRecordId = meta["recordId"]
+        dataValue = int(X[0, 5, 0])   # flat index encoded in the signal
+        _metadata_checks.append(
+            (self.currentIndex, metadataRecordId, dataValue)
+        )
+        return X, Y
+
+
+# Per-dataset record ids: 3 datasets × 5 records
+ALL_IDS = [[f"d{d}-{r}" for r in range(7)] for d in range(3)]
+FLAT_IDS = [rid for ds in ALL_IDS for rid in ds]          # 15 ids in flat order
+
+
+# ---------------------------------------------------------------------------
+# Test class
+# ---------------------------------------------------------------------------
+
+class TestBuildDatasetMetadataSync(TestCase):
+    phase = BuildDataset()
+
+    def config(self):
+        self.project.setConfig("dataversion.split", {})
+        return {
+            "useLoader": "test",
+            "dataversion": {
+                "seed": 2023,
+                "recordIds": FLAT_IDS,
+                "split": {"training": ["0:15"]},
+                "groupBy": None,
+            },
+            "segmentManipulation": [
+                {"name": "addBatchDimension"},
+                {"name": "fixedSize", "position": "center", "size": 10},
+                {"name": "checkMetadata"},
+            ],
+            "segmentManipulationEval": [
+                {"name": "addBatchDimension"},
+                {"name": "fixedSize", "position": "center", "size": 10},
+                {"name": "checkMetadata"},
+            ],
+            "evalOn": None,
+            "datafold": {
+                "shuffle": False,
+                "seed": 42,
+                "datasets": [
+                    {"config": {"dataversion": {"recordIds": ALL_IDS[0], "seed": 1, "split": {"training": ["0:5"], "validation": ["5:7"]}, "groupBy": None}}},
+                    {"config": {"dataversion": {"recordIds": ALL_IDS[1], "seed": 2, "split": {"training": ["0:5"], "validation": ["5:7"]}, "groupBy": None}}},
+                    {"config": {"dataversion": {"recordIds": ALL_IDS[2], "seed": 3, "split": {"training": ["0:5"], "validation": ["5:7"]}, "groupBy": None}}},
+                ],
+            },
+        }
+
+    def setUp(self):
+        super().setUp()
+        DataManipulation.setClass(CheckingDataManipulation)
+
+        metadata = pd.DataFrame([{"recordId": rid} for rid in FLAT_IDS])
+
+        RecordLoader.getRecordList = MagicMock(return_value=FLAT_IDS)
+
+        # Register data for each datafold dataset context — mirrors what
+        # buildDataRecords() does with ``with self.project:`` + updateConfig.
+        # Each dataset gets its own data arrays (like separate memmaps in prod).
+        datasets = self.project.getConfig("datafold.datasets", [{"config": {}}])
+        for dsIdx, ds in enumerate(datasets):
+            overwriteConfig = ds["config"]
+            with self.project:
+                if "dataversion" in overwriteConfig:
+                    self.project.setConfig("dataversion", {})
+                self.project.updateConfig(overwriteConfig)
+
+                dsRecordIds = self.project.getConfig("dataversion.recordIds")
+                # Each dataset's memmap only contains its own records.
+                # Values are the *global flat index* so we can trace which
+                # record a data item originally belonged to.
+                dsStart = dsIdx * 7
+                dsProcessed = np.arange(dsStart, dsStart + len(dsRecordIds)).reshape(-1, 1, 1)
+                dsFeatures = np.arange(dsStart, dsStart + len(dsRecordIds)).reshape(-1, 1, 1)
+
+                self.project.registerData("allDBRecordIds", {rid: [rid] for rid in dsRecordIds})
+                self.project.registerData("metadata", metadata)
+                self.project.registerData("data-processed", dsProcessed)
+                self.project.registerData("data-features", dsFeatures)
+
+    def tearDown(self):
+        DataManipulation.setClass(None)
+        super().tearDown()
+
+    # -- helpers ----------------------------------------------------------
+
+    def _drain(self, dataset):
+        """Iterate the full dataset, return list of (x, y)."""
+        return [item for item in dataset]
+
+    def _data_order(self, items):
+        """Return the list of flat-index values read from the X payloads."""
+        return [int(x[0, 5, 0]) for x, _ in items]
+
+    def _assert_metadata_matches_data(self, label=""):
+        """Assert every recorded (metadata, data) pair agrees."""
+        mismatches = []
+        for idx, metaId, dataVal in _metadata_checks:
+            expected_flat = FLAT_IDS.index(metaId)
+            if expected_flat != dataVal:
+                mismatches.append(
+                    f"  index {idx}: metadata={metaId} (flat {expected_flat}), data={dataVal}"
+                )
+        if mismatches:
+            self.fail(
+                f"{label}{len(mismatches)} metadata/data mismatches:\n"
+                + "\n".join(mismatches)
+            )
+
+    # =====================================================================
+    # 1) No shuffle — metadata must match data for every item
+    # =====================================================================
+
+    def testNoShuffle_MetadataMatchesData(self):
+        """Without shuffle the metadata recordId at each index must equal
+        the recordId derivable from the data payload."""
+        _metadata_checks.clear()
+        items = self._drain(self.getData("dataset-training"))
+        self.assertEqual(len(items), 15)
+        self._assert_metadata_matches_data("no-shuffle: ")
+
+    def testNoShuffle_MetadataMatchesDataValidation(self):
+        """Without shuffle the metadata recordId at each index must equal
+        the recordId derivable from the data payload."""
+        _metadata_checks.clear()
+        items = self._drain(self.getData("dataset-validation"))
+        self.assertEqual(len(items), 6)
+        self._assert_metadata_matches_data("no-shuffle: ")
+
+    # =====================================================================
+    # 2) No shuffle — deterministic across two loads
+    # =====================================================================
+
+    def testNoShuffle_DeterministicOrder(self):
+        """Two consecutive loads without shuffle must return data in
+        identical order."""
+        items1 = self._drain(self.getData("dataset-training"))
+        order1 = self._data_order(items1)
+
+        items2 = self._drain(self.getData("dataset-training"))
+        order2 = self._data_order(items2)
+
+        self.assertEqual(order1, order2)
+
+    def testNoShuffle_DeterministicOrderValidation(self):
+        """Two consecutive loads without shuffle must return validation data in
+        identical order."""
+        items1 = self._drain(self.getData("dataset-validation"))
+        order1 = self._data_order(items1)
+
+        items2 = self._drain(self.getData("dataset-validation"))
+        order2 = self._data_order(items2)
+
+        self.assertEqual(order1, order2)
+
+    # =====================================================================
+    # 3) With shuffle — load twice, verify deterministic order via seed
+    # =====================================================================
+
+    @OverwriteConfig({"datafold": {"shuffle": True, "seed": 42}})
+    def testShuffle_DeterministicOrder(self):
+        """Loading the dataset twice with the same seed must yield
+        identical item order both times."""
+        items1 = self._drain(self.getData("dataset-training"))
+        order1 = self._data_order(items1)
+
+        items2 = self._drain(self.getData("dataset-training"))
+        order2 = self._data_order(items2)
+
+        self.assertEqual(order1, order2,
+                         "Same seed must produce the same item order")
+
+    @OverwriteConfig({"datafold": {"shuffle": True, "seed": 42}})
+    def testShuffle_DeterministicOrderValidation(self):
+        """Loading the validation dataset twice with the same seed must yield
+        identical item order both times."""
+        items1 = self._drain(self.getData("dataset-validation"))
+        order1 = self._data_order(items1)
+
+        items2 = self._drain(self.getData("dataset-validation"))
+        order2 = self._data_order(items2)
+
+        self.assertEqual(order1, order2,
+                         "Same seed must produce the same item order")
+
+    # =====================================================================
+    # 4) With shuffle — metadata must still match data for every item
+    # =====================================================================
+
+    @OverwriteConfig({"datafold": {"shuffle": True, "seed": 42}})
+    def testShuffle_MetadataMatchesData(self):
+        """After shuffle, recordMetadata[i].recordId must describe the
+        record that data item i actually contains."""
+        _metadata_checks.clear()
+        items = self._drain(self.getData("dataset-training"))
+        self.assertEqual(len(items), 15)
+        self._assert_metadata_matches_data("shuffled: ")
+
+    @OverwriteConfig({"datafold": {"shuffle": True, "seed": 42}})
+    def testShuffle_MetadataMatchesDataValidation(self):
+        """After shuffle, recordMetadata[i].recordId must describe the
+        record that validation data item i actually contains."""
+        _metadata_checks.clear()
+        items = self._drain(self.getData("dataset-validation"))
+        self.assertEqual(len(items), 6)
+        self._assert_metadata_matches_data("shuffled-validation: ")

{sleepyphases-0.6.2 → sleepyphases-0.7.0}/tests/unit/test_DataManipulation.py

@@ -207,7 +207,7 @@ class TestDataManipulation(unittest.TestCase):
 
     def test_deleteIgnored(self):
         testY = np.array([1, 2, 3, -1, 2, 2, 3, -1, 2, -1, 2]).reshape(1, -1, 1)
-        testX = np.array([1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11]).reshape(1, -1, 1)
+        testX = np.array([1, 2, 3,  4, 5, 6, 7,  8, 9, 10, 11]).reshape(1, -1, 1)
         newX, newY = self.getDA().deleteIgnored(testX, testY)
         npt.assert_equal(newY.reshape(-1), [1, 2, 3, 2, 2, 3, 2, 2])
         npt.assert_equal(newX.reshape(-1), [1, 2, 3, 5, 6, 7, 9, 11])