dbdicom 0.3.1__py3-none-any.whl → 0.3.3__py3-none-any.whl

dbdicom/dbd.py

@@ -1,5 +1,7 @@
 import os
 from datetime import datetime
+import json
+from typing import Union
 
 from tqdm import tqdm
 import numpy as np
@@ -8,9 +10,9 @@ import vreg
 from pydicom.dataset import Dataset
 
 import dbdicom.utils.arrays
-import dbdicom.utils.files as filetools
-import dbdicom.utils.dcm4che as dcm4che
+
 import dbdicom.dataset as dbdataset
+import dbdicom.database as dbdatabase
 import dbdicom.register as register
 import dbdicom.const as const
 
@@ -32,7 +34,8 @@ class DataBaseDicom():
         file = self._register_file()
         if os.path.exists(file):
             try:
-                self.register = pd.read_pickle(file)
+                with open(file, 'r') as f:
+                    self.register = json.load(f)
             except:
                 # If the file is corrupted, delete it and load again
                 os.remove(file)
@@ -44,75 +47,45 @@ class DataBaseDicom():
     def read(self):
         """Read the DICOM folder again
         """
-
-        files = filetools.all_files(self.path)
-        self.register = dbdataset.read_dataframe(
-            files, 
-            register.COLUMNS + ['NumberOfFrames','SOPClassUID'], 
-            path=self.path, 
-            images_only = True)
-        self.register['removed'] = False
-        self.register['created'] = False
-        # No support for multiframe data at the moment
-        self._multiframe_to_singleframe()
+        self.register = dbdatabase.read(self.path)
         # For now ensure all series have just a single CIOD
-        self._split_series()
+        # Leaving this out for now until the issue occurs again
+        # self._split_series()
         return self
-    
 
-    def close(self): 
-        """Close the DICOM folder
-        
-        This also saves changes in the header file to disk.
-        """
+    
 
-        created = self.register.created & (self.register.removed==False) 
-        removed = self.register.removed
-        created = created[created].index
-        removed = removed[removed].index
+    def delete(self, entity):
+        """Delete a DICOM entity from the database
 
+        Args:
+            entity (list): entity to delete
+        """
+        removed = register.index(self.register, entity)
         # delete datasets marked for removal
         for index in removed.tolist():
             file = os.path.join(self.path, index)
             if os.path.exists(file): 
                 os.remove(file)
         # and drop then from the register
-        self.register.drop(index=removed, inplace=True)
-
-        # for new or edited data, mark as saved.
-        self.register.loc[created, 'created'] = False
-
-        # save register
-        file = self._register_file()
-        self.register.to_pickle(file)
+        self.register = register.drop(removed)
         return self
     
 
-    def restore(self): 
-        """Restore the DICOM folder to the last saved state.""" 
-
-        created = self.register.created 
-        removed = self.register.removed & (self.register.created==False)
-        created = created[created].index
-        removed = removed[removed].index
-
-        # permanently delete newly created datasets
-        for index in created.tolist():
-            file = os.path.join(self.path, index)
-            if os.path.exists(file): 
-                os.remove(file)
-
-        # and drop then from the register
-        self.register.drop(index=created, inplace=True)
-
-        # Restore those that were marked for removal
-        self.register.loc[removed, 'removed'] = False
-
-        # save register
+    def close(self): 
+        """Close the DICOM folder
+        
+        This also saves changes in the header file to disk.
+        """
+        # Save df as pkl
         file = self._register_file()
-        self.register.to_pickle(file)
-        return self    
+        with open(file, 'w') as f:
+            json.dump(self.register, f, indent=4)
+        return self
 
+    def _register_file(self):
+        return os.path.join(self.path, 'dbtree.json') 
+    
 
     def summary(self):
         """Return a summary of the contents of the database.
@@ -122,6 +95,7 @@ class DataBaseDicom():
         """
         return register.summary(self.register)
     
+
     def print(self):
         """Print the contents of the DICOM folder
         """
@@ -170,6 +144,11 @@ class DataBaseDicom():
             for patient in self.patients():
                 studies += self.studies(patient, name, contains, isin)
             return studies
+        elif len(entity)==1:
+            studies = []
+            for patient in self.patients():
+                studies += self.studies(patient, name, contains, isin)
+            return studies
         else:
             return register.studies(self.register, entity, name, contains, isin)
     
@@ -199,6 +178,11 @@ class DataBaseDicom():
             for study in self.studies(entity):
                 series += self.series(study, name, contains, isin)
             return series
+        elif len(entity)==1:
+            series = []
+            for study in self.studies(entity):
+                series += self.series(study, name, contains, isin)
+            return series            
         elif len(entity)==2:
             series = []
             for study in self.studies(entity):
@@ -208,20 +192,20 @@ class DataBaseDicom():
             return register.series(self.register, entity, name, contains, isin)
 
 
-    def volume(self, series:list, dims:list=None, multislice=False) -> vreg.Volume3D:
+    def volume(self, series:list, dims:list=None) -> vreg.Volume3D:
         """Read a vreg.Volume3D from a DICOM series
 
         Args:
             series (list): DICOM series to read
             dims (list, optional): Non-spatial dimensions of the volume. Defaults to None.
-            multislice (bool, optional): Whether the data are to be read 
-                as multislice or not. In multislice data the voxel size 
-                is taken from the slice gap rather thsan the slice thickness. Defaults to False.
 
         Returns:
             vreg.Volume3D: vole read from the series.
         """
-
+        if isinstance(series, str): # path to folder
+            return [self.volume(s, dims) for s in self.series(series)]
+        if len(series) < 4: # folder, patient or study
+            return [self.volume(s, dims) for s in self.series(series)]
         if dims is None:
             dims = []
         elif isinstance(dims, str):
@@ -238,7 +222,7 @@ class DataBaseDicom():
         for f in tqdm(files, desc='Reading volume..'):
             ds = dbdataset.read_dataset(f)  
             values.append(dbdataset.get_values(ds, dims))
-            volumes.append(dbdataset.volume(ds, multislice))
+            volumes.append(dbdataset.volume(ds))
 
         # Format as mesh
         coords = np.stack(values, axis=-1)
@@ -256,9 +240,21 @@ class DataBaseDicom():
                     "firstslice=True, the coordinates of the lowest "
                     "slice will be assigned to the whole volume."     
                 )
+            
+        # Infer spacing between slices from slice locations
+        # Technically only necessary if SpacingBetweenSlices not set or incorrect
+        vols = infer_slice_spacing(vols)
 
         # Join 2D volumes into 3D volumes
-        vol = vreg.join(vols)
+        try:
+            vol = vreg.join(vols)
+        except ValueError:
+            # some vendors define the slice vector as -cross product 
+            # of row and column vector. Check if that solves the issue.
+            for v in vols.reshape(-1):
+                v.affine[:3,2] = -v.affine[:3,2]
+                # Then try again
+            vol = vreg.join(vols)
         if vol.ndim > 3:
             vol.set_coords(c0)
             vol.set_dims(dims[1:])
@@ -266,21 +262,21 @@ class DataBaseDicom():
 
     
     def write_volume(
-            self, vol:vreg.Volume3D, series:list, 
-            ref:list=None, multislice=False,
+            self, vol:Union[vreg.Volume3D, tuple], series:list, 
+            ref:list=None, 
         ):
         """Write a vreg.Volume3D to a DICOM series
 
         Args:
             vol (vreg.Volume3D): Volume to write to the series.
             series (list): DICOM series to read
-            dims (list, optional): Non-spatial dimensions of the volume. Defaults to None.
-            multislice (bool, optional): Whether the data are to be read 
-                as multislice or not. In multislice data the voxel size 
-                is taken from the slice gap rather thsan the slice thickness. Defaults to False.
+            ref (list): Reference series
         """
+        if isinstance(vol, tuple):
+            vol = vreg.volume(vol[0], vol[1])
         if ref is None:
             ds = dbdataset.new_dataset('MRImage')
+            #ds = dbdataset.new_dataset('ParametricMap')
         else:
             if ref[0] == series[0]:
                 ref_mgr = self
@@ -293,26 +289,24 @@ class DataBaseDicom():
         attr = self._attributes(series)
         n = self._max_instance_number(attr['SeriesInstanceUID'])
 
-        new_instances = {}
         if vol.ndim==3:
             slices = vol.split()
             for i, sl in tqdm(enumerate(slices), desc='Writing volume..'):
-                dbdataset.set_volume(ds, sl, multislice)
-                self._write_dataset(ds, attr, n + 1 + i, new_instances)
+                dbdataset.set_volume(ds, sl)
+                self._write_dataset(ds, attr, n + 1 + i)
         else:
             i=0
             vols = vol.separate().reshape(-1)
             for vt in tqdm(vols, desc='Writing volume..'):
                 for sl in vt.split():
-                    dbdataset.set_volume(ds, sl, multislice)
+                    dbdataset.set_volume(ds, sl)
                     dbdataset.set_value(ds, sl.dims, sl.coords[:,...])
-                    self._write_dataset(ds, attr, n + 1 + i, new_instances)
+                    self._write_dataset(ds, attr, n + 1 + i)
                     i+=1
-            return self
-        self._update_register(new_instances)
+        return self
 
 
-    def to_nifti(self, series:list, file:str, dims=None, multislice=False):
+    def to_nifti(self, series:list, file:str, dims=None):
         """Save a DICOM series in nifti format.
 
         Args:
@@ -320,34 +314,31 @@ class DataBaseDicom():
             file (str): file path of the nifti file.
             dims (list, optional): Non-spatial dimensions of the volume. 
                 Defaults to None.
-            multislice (bool, optional): Whether the data are to be read 
-                as multislice or not. In multislice data the voxel size 
-                is taken from the slice gap rather thaan the slice thickness. Defaults to False.
         """
-        vol = self.volume(series, dims, multislice)
+        vol = self.volume(series, dims)
         vreg.write_nifti(vol, file)
         return self
 
-    def from_nifti(self, file:str, series:list, ref:list=None, multislice=False):
+    def from_nifti(self, file:str, series:list, ref:list=None):
         """Create a DICOM series from a nifti file.
 
         Args:
             file (str): file path of the nifti file.
             series (list): DICOM series to create
             ref (list): DICOM series to use as template.
-            multislice (bool, optional): Whether the data are to be written
-                as multislice or not. In multislice data the voxel size 
-                is written in the slice gap rather thaan the slice thickness. Defaults to False.
         """
         vol = vreg.read_nifti(file)
-        self.write_volume(vol, series, ref, multislice)
+        self.write_volume(vol, series, ref)
         return self
     
     def pixel_data(self, series:list, dims:list=None, coords=False, include=None) -> np.ndarray:
         """Read the pixel data from a DICOM series
 
         Args:
-            series (list): DICOM series to read
+            series (list or str): DICOM series to read. This can also 
+                be a path to a folder containing DICOM files, or a 
+                patient or study to read all series in that patient or 
+                study. In those cases a list is returned.
             dims (list, optional): Dimensions of the array.
             coords (bool): If set to Trye, the coordinates of the 
                 arrays are returned alongside the pixel data
@@ -360,8 +351,12 @@ class DataBaseDicom():
                 coords is set these are returned too as an array with 
                 coordinates of the slices according to dims. If include 
                 is provide the values are returned as a dictionary in the last 
-                return value.
+                return value. 
         """
+        if isinstance(series, str): # path to folder
+            return [self.pixel_data(s, dims, coords, include) for s in self.series(series)]
+        if len(series) < 4: # folder, patient or study
+            return [self.pixel_data(s, dims, coords, include) for s in self.series(series)]
         if coords:
             if dims is None:
                 raise ValueError(
@@ -488,16 +483,6 @@ class DataBaseDicom():
             f"Cannot copy {from_entity} to {to_entity}. "
         )
     
-    def delete(self, entity):
-        """Delete a DICOM entity from the database
-
-        Args:
-            entity (list): entity to delete
-        """
-        index = register.index(self.register, entity)
-        self.register.loc[index,'removed'] = True
-        return self
-
     def move(self, from_entity, to_entity):
         """Move a DICOM entity
 
@@ -510,15 +495,15 @@ class DataBaseDicom():
 
     def _values(self, attributes:list, entity:list):
         # Create a np array v with values for each instance and attribute
-        if set(attributes) <= set(self.register.columns):
-            index = register.index(self.register, entity)
-            v = self.register.loc[index, attributes].values
-        else:
-            files = register.files(self.register, entity)
-            v = np.empty((len(files), len(attributes)), dtype=object)
-            for i, f in enumerate(files):
-                ds = dbdataset.read_dataset(f)
-                v[i,:] = dbdataset.get_values(ds, attributes)
+        # if set(attributes) <= set(dbdatabase.COLUMNS):
+        #     index = register.index(self.register, entity)
+        #     v = self.register.loc[index, attributes].values
+        # else:
+        files = register.files(self.register, entity)
+        v = np.empty((len(files), len(attributes)), dtype=object)
+        for i, f in enumerate(files):
+            ds = dbdataset.read_dataset(f)
+            v[i,:] = dbdataset.get_values(ds, attributes)
         return v
 
     def _copy_patient(self, from_patient, to_patient):
@@ -562,30 +547,34 @@ class DataBaseDicom():
         n = self._max_instance_number(attr['SeriesInstanceUID'])
         
         # Copy the files to the new series 
-        new_instances = {}
         for i, f in tqdm(enumerate(files), total=len(files), desc=f'Copying series {to_series[1:]}'):
             # Read dataset and assign new properties
             ds = dbdataset.read_dataset(f)
-            self._write_dataset(ds, attr, n + 1 + i, new_instances)
-        self._update_register(new_instances)
-
-
+            self._write_dataset(ds, attr, n + 1 + i)
+
+    def _max_study_id(self, patient_id):
+        for pt in self.register:
+            if pt['PatientID'] == patient_id:
+                n = [int(st['StudyID']) for st in pt['studies']]
+                return int(np.amax(n))
+        return 0
+    
     def _max_series_number(self, study_uid):
-        df = self.register
-        df = df[(df.StudyInstanceUID==study_uid) & (df.removed==False)]
-        n = df['SeriesNumber'].values
-        n = n[n != -1]
-        max_number=0 if n.size==0 else np.amax(n)  
-        return max_number 
+        for pt in self.register:
+            for st in pt['studies']:
+                if st['StudyInstanceUID'] == study_uid:
+                    n = [sr['SeriesNumber'] for sr in st['series']]
+                    return int(np.amax(n))
+        return 0
 
     def _max_instance_number(self, series_uid):
-        df = self.register
-        df = df[(df.SeriesInstanceUID==series_uid) & (df.removed==False)]
-        n = df['InstanceNumber'].values
-        n = n[n != -1]
-        max_number=0 if n.size==0 else np.amax(n)  
-        return max_number 
-
+        for pt in self.register:
+            for st in pt['studies']:
+                for sr in st['series']:
+                    if sr['SeriesInstanceUID'] == series_uid:
+                        n = list(sr['instances'].keys())
+                        return int(np.amax([int(i) for i in n]))
+        return 0
 
     def _attributes(self, entity):
         if len(entity)==4:
@@ -606,7 +595,8 @@ class DataBaseDicom():
         except:
             # If the patient does not exist, generate values
             attr = ['PatientID', 'PatientName']
-            patient_id = dbdataset.new_uid()
+            #patient_id = dbdataset.new_uid()
+            patient_id = patient[-1] if isinstance(patient[-1], str) else f"{patient[-1][0]}_{patient[-1][1]}"
             patient_name = patient[-1] if isinstance(patient[-1], str) else patient[-1][0]
             vals = [patient_id, patient_name]
         return {attr[i]:vals[i] for i in range(len(attr)) if vals[i] is not None}
@@ -621,12 +611,18 @@ class DataBaseDicom():
             ds = dbdataset.read_dataset(files[0])
             vals = dbdataset.get_values(ds, attr)
         except:
-            # If the study does not exist, generate values
-            attr = ['StudyInstanceUID', 'StudyDescription', 'StudyDate']
-            study_id = dbdataset.new_uid()
+            # If the study does not exist or is empty, generate values
+            try:
+                patient_id = register.uid(self.register, study[:-1])
+            except:
+                study_id = 1
+            else:
+                study_id = 1 + self._max_study_id(patient_id)
+            attr = ['StudyInstanceUID', 'StudyDescription', 'StudyDate', 'StudyID']
+            study_uid = dbdataset.new_uid()
             study_desc = study[-1] if isinstance(study[-1], str) else study[-1][0]
             study_date = datetime.today().strftime('%Y%m%d')
-            vals = [study_id, study_desc, study_date]
+            vals = [study_uid, study_desc, study_date, str(study_id)]
         return patient_attr | {attr[i]:vals[i] for i in range(len(attr)) if vals[i] is not None}
 
 
@@ -647,90 +643,114 @@ class DataBaseDicom():
             else:
                 series_number = 1 + self._max_series_number(study_uid)
             attr = ['SeriesInstanceUID', 'SeriesDescription', 'SeriesNumber']
-            series_id = dbdataset.new_uid()
+            series_uid = dbdataset.new_uid()
             series_desc = series[-1] if isinstance(series[-1], str) else series[-1][0]
-            vals = [series_id, series_desc, series_number]
+            vals = [series_uid, series_desc, int(series_number)]
         return study_attr | {attr[i]:vals[i] for i in range(len(attr)) if vals[i] is not None}
 
         
-    def _write_dataset(self, ds:Dataset, attr:dict, instance_nr:int, register:dict):
+    def _write_dataset(self, ds:Dataset, attr:dict, instance_nr:int):
         # Set new attributes 
         attr['SOPInstanceUID'] = dbdataset.new_uid()
-        attr['InstanceNumber'] = instance_nr
+        attr['InstanceNumber'] = str(instance_nr)
         dbdataset.set_values(ds, list(attr.keys()), list(attr.values()))
         # Save results in a new file
-        rel_path = os.path.join('dbdicom', dbdataset.new_uid() + '.dcm') 
+        rel_dir = os.path.join(
+            f"patient_{attr['PatientID']}", 
+            f"study_[{attr['StudyID']}]_{attr['StudyDescription']}", 
+            f"series_[{attr['SeriesNumber']}]_{attr['SeriesDescription']}",
+        )
+        os.makedirs(os.path.join(self.path, rel_dir), exist_ok=True)
+        rel_path = os.path.join(rel_dir, dbdataset.new_uid() + '.dcm')
         dbdataset.write(ds, os.path.join(self.path, rel_path))
-        # Add a row to the register
-        register[rel_path] = dbdataset.get_values(ds, self.register.columns)
-
+        # Add an entry in the register
+        register.add_instance(self.register, attr, rel_path)
+
+
+
+    # def _split_series(self):
+    #     """
+    #     Split series with multiple SOP Classes.
+
+    #     If a series contain instances from different SOP Classes, 
+    #     these are separated out into multiple series with identical SOP Classes.
+    #     """
+    #     # For each series, check if there are multiple
+    #     # SOP Classes in the series and split them if yes.
+    #     for series in self.series():
+    #         series_index = register.index(self.register, series)
+    #         df_series = self.register.loc[series_index]
+    #         sop_classes = df_series.SOPClassUID.unique()
+    #         if len(sop_classes) > 1:
+    #             # For each sop_class, create a new series and move all
+    #             # instances of that sop_class to the new series
+    #             desc = series[-1] if isinstance(series, str) else series[0]
+    #             for i, sop_class in tqdm(enumerate(sop_classes[1:]), desc='Splitting series with multiple SOP Classes.'):
+    #                 df_sop_class = df_series[df_series.SOPClassUID == sop_class]
+    #                 relpaths = df_sop_class.index.tolist()
+    #                 sop_class_files = [os.path.join(self.path, p) for p in relpaths]
+    #                 sop_class_series = series[:-1] + [desc + f' [{i+1}]']
+    #                 self._files_to_series(sop_class_files, sop_class_series)
+    #                 # Delete original files permanently
+    #                 self.register.drop(relpaths)
+    #                 for f in sop_class_files:
+    #                     os.remove(f)
+    #     self.register.drop('SOPClassUID', axis=1, inplace=True)
+
+
+def infer_slice_spacing(vols):
+    # In case spacing between slices is not (correctly) encoded in 
+    # DICOM it can be inferred from the slice locations.
+
+    shape = vols.shape
+    vols = vols.reshape((shape[0], -1))
+    slice_spacing = np.zeros(vols.shape[-1])
+
+    for d in range(vols.shape[-1]):
+
+        # For single slice volumes there is nothing to do
+        if vols[:,d].shape[0]==1:
+            continue
+
+        # Get a normal slice vector from the first volume.
+        mat = vols[0,d].affine[:3,:3]
+        normal = mat[:,2]/np.linalg.norm(mat[:,2])
+
+        # Get slice locations by projection on the normal.
+        pos = [v.affine[:3,3] for v in vols[:,d]]
+        slice_loc = [np.dot(p, normal) for p in pos]
+
+        # Sort slice locations and take consecutive differences.
+        slice_loc = np.sort(slice_loc)
+        distances = slice_loc[1:] - slice_loc[:-1]
+
+        # Round to micrometer and check if unique
+        distances = np.around(distances, 3)
+        slice_spacing_d = np.unique(distances)
+
+        # Check if unique - otherwise this is not a volume
+        if len(slice_spacing_d) > 1:
+            raise ValueError(
+                'Cannot build a volume - spacings between slices are not unique.'
+            )
+        else:
+            slice_spacing_d= slice_spacing_d[0]
+        
+        # Set correct slice spacing in all volumes
+        for v in vols[:,d]:
+            v.affine[:3,2] = normal * abs(slice_spacing_d)
 
-    def _update_register(self, new_instances:dict):
-        # A new instances to the register
-        df = pd.DataFrame.from_dict(new_instances, orient='index', columns=self.register.columns)
-        df['removed'] = False
-        df['created'] = True
-        self.register = pd.concat([self.register, df])
+        slice_spacing[d] = slice_spacing_d
 
+    # Check slice_spacing is the same across dimensions
+    slice_spacing = np.unique(slice_spacing)
+    if len(slice_spacing) > 1:
+        raise ValueError(
+            'Cannot build a volume - spacings between slices are not unique.'
+        )    
 
-    def _register_file(self):
-        filename = os.path.basename(os.path.normpath(self.path)) + ".pkl"
-        return os.path.join(self.path, filename) 
-    
+    return vols.reshape(shape)
 
-    def _multiframe_to_singleframe(self):
-        """Converts all multiframe files in the folder into single-frame files.
-        
-        Reads all the multi-frame files in the folder,
-        converts them to singleframe files, and delete the original multiframe file.
-        """
-        singleframe = self.register.NumberOfFrames.isnull() 
-        multiframe = singleframe == False
-        nr_multiframe = multiframe.sum()
-        if nr_multiframe != 0: 
-            for relpath in tqdm(self.register[multiframe].index.values, desc="Converting multiframe file " + relpath):
-                filepath = os.path.join(self.path, relpath)
-                singleframe_files = dcm4che.split_multiframe(filepath) 
-                if singleframe_files != []:            
-                    # add the single frame files to the dataframe
-                    df = dbdataset.read_dataframe(singleframe_files, self.register.columns, path=self.path)
-                    df['removed'] = False
-                    df['created'] = False
-                    self.register = pd.concat([self.register, df])
-                    # delete the original multiframe 
-                    os.remove(filepath)
-                # drop the file also if the conversion has failed
-                self.register.drop(index=relpath, inplace=True)
-        self.register.drop('NumberOfFrames', axis=1, inplace=True)
-
-
-    def _split_series(self):
-        """
-        Split series with multiple SOP Classes.
 
-        If a series contain instances from different SOP Classes, 
-        these are separated out into multiple series with identical SOP Classes.
-        """
-        # For each series, check if there are multiple
-        # SOP Classes in the series and split them if yes.
-        for series in self.series():
-            series_index = register.index(self.register, series)
-            df_series = self.register.loc[series_index]
-            sop_classes = df_series.SOPClassUID.unique()
-            if len(sop_classes) > 1:
-                # For each sop_class, create a new series and move all
-                # instances of that sop_class to the new series
-                desc = series[-1] if isinstance(series, str) else series[0]
-                for i, sop_class in tqdm(enumerate(sop_classes[1:]), desc='Splitting series with multiple SOP Classes.'):
-                    df_sop_class = df_series[df_series.SOPClassUID == sop_class]
-                    relpaths = df_sop_class.index.tolist()
-                    sop_class_files = [os.path.join(self.path, p) for p in relpaths]
-                    sop_class_series = series[:-1] + [desc + f' [{i+1}]']
-                    self._files_to_series(sop_class_files, sop_class_series)
-                    # Delete original files permanently
-                    self.register.drop(relpaths)
-                    for f in sop_class_files:
-                        os.remove(f)
-        self.register.drop('SOPClassUID', axis=1, inplace=True)