spectre-core 0.0.11__py3-none-any.whl → 0.0.12__py3-none-any.whl

spectre_core/_file_io/file_handlers.py

@@ -10,10 +10,10 @@ from typing import Any, Optional
 
 class BaseFileHandler(ABC):
     def __init__(self, 
-                 parent_path: str, 
+                 parent_dir_path: str, 
                  base_file_name: str, 
                  extension: Optional[str] = None):
-        self._parent_path = parent_path
+        self._parent_dir_path = parent_dir_path
         self._base_file_name = base_file_name
         self._extension = extension
 
@@ -24,8 +24,8 @@ class BaseFileHandler(ABC):
  
 
     @property
-    def parent_path(self) -> str:
-        return self._parent_path
+    def parent_dir_path(self) -> str:
+        return self._parent_dir_path
     
 
     @property
@@ -45,7 +45,7 @@ class BaseFileHandler(ABC):
 
     @property
     def file_path(self) -> str:
-        return os.path.join(self._parent_path, self.file_name)
+        return os.path.join(self._parent_dir_path, self.file_name)
     
     
     @property
@@ -53,8 +53,8 @@ class BaseFileHandler(ABC):
         return os.path.exists(self.file_path) 
 
 
-    def make_parent_path(self) -> None:
-        os.makedirs(self.parent_path, exist_ok=True) 
+    def make_parent_dir_path(self) -> None:
+        os.makedirs(self.parent_dir_path, exist_ok=True) 
     
 
     def delete(self,
@@ -71,13 +71,13 @@ class BaseFileHandler(ABC):
 
 class JsonHandler(BaseFileHandler):
     def __init__(self, 
-                 parent_path: str, 
+                 parent_dir_path: str, 
                  base_file_name: str,
                  extension: str = "json",
                  **kwargs):
         
         self._dict = None # cache
-        super().__init__(parent_path, 
+        super().__init__(parent_dir_path, 
                          base_file_name, 
                          extension,
                          **kwargs)
@@ -91,7 +91,7 @@ class JsonHandler(BaseFileHandler):
     def save(self, 
              d: dict, 
              force: bool = False) -> None:
-        self.make_parent_path()
+        self.make_parent_dir_path()
 
         if self.exists:
             if force:
@@ -113,11 +113,11 @@ class JsonHandler(BaseFileHandler):
 
 class TextHandler(BaseFileHandler):
     def __init__(self, 
-                 parent_path: str,
+                 parent_dir_path: str,
                  base_file_name: str, 
                  extension: str = "txt",
                  **kwargs):
-        super().__init__(parent_path,
+        super().__init__(parent_dir_path,
                          base_file_name,
                          extension, 
                          **kwargs)

spectre_core/batches/__init__.py

@@ -0,0 +1,22 @@
+# SPDX-FileCopyrightText: © 2024 Jimmy Fitzpatrick <jcfitzpatrick12@gmail.com>
+# This file is part of SPECTRE
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+# register decorators are actioned on import
+from .library._fixed_center_frequency import _Batch
+from .library._swept_center_frequency import _Batch
+from .library._callisto import _Batch
+
+from ._base import BaseBatch, BatchFile
+from ._factory import get_batch_cls_from_tag
+from ._batches import Batches
+from .library._swept_center_frequency import SweepMetadata
+
+__all__ = [
+    "BaseBatch",
+    "BatchFile",
+    "get_batch_cls_from_tag",
+    "Batches",
+    "SweepMetadata"
+]
+

spectre_core/batches/_base.py

@@ -0,0 +1,146 @@
+# SPDX-FileCopyrightText: © 2024 Jimmy Fitzpatrick <jcfitzpatrick12@gmail.com>
+# This file is part of SPECTRE
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+from datetime import datetime
+from typing import Optional
+
+from spectre_core._file_io import BaseFileHandler
+from spectre_core.config import get_batches_dir_path, TimeFormats
+from spectre_core.exceptions import BatchFileNotFoundError
+
+
+class BatchFile(BaseFileHandler):
+    """A specific file in a given batch, uniquely identified in the batch by the file extension."""
+    def __init__(self, 
+                 batch_parent_dir_path: str, 
+                 batch_name: str, 
+                 extension: str):
+        super().__init__(batch_parent_dir_path, 
+                         batch_name, 
+                         extension)
+        self._start_time, self._tag = batch_name.split("_")
+        # computed if required
+        self._start_datetime: Optional[datetime] = None
+        
+        
+    @property
+    def start_time(self) -> str:
+        """The start time of the batch file, up to seconds precision."""
+        return self._start_time
+
+
+    @property
+    def start_datetime(self) -> datetime:
+        """The datetime of the batch file, up to seconds precision."""
+        if self._start_datetime is None:
+            self._start_datetime = datetime.strptime(self.start_time, TimeFormats.DATETIME)
+        return self._start_datetime
+    
+
+    @property
+    def tag(self) -> str:
+        """The tag identifier for the batch file."""
+        return self._tag
+    
+
+
+class BaseBatch:
+    """A group of one or more files which share a common start time and a tag identifier.
+    
+    All files belonging to the same batch will share a batch name, and differ
+    only in their file extension.
+    """
+    def __init__(self, 
+                 start_time: str,
+                 tag: str):
+        self._start_time = start_time
+        self._tag: str = tag
+        self._batch_files: dict[str, BatchFile] = {}
+        self._start_datetime = datetime.strptime(self.start_time, TimeFormats.DATETIME)
+        self._parent_dir_path = get_batches_dir_path(year  = self.start_datetime.year,
+                                                     month = self.start_datetime.month,
+                                                     day   = self.start_datetime.day)
+
+
+    @property
+    def start_time(self) -> str:
+        """The start time of the batch, up to seconds precision."""
+        return self._start_time
+
+
+    @property
+    def tag(self) -> str:
+        """The tag identifier of for the batch."""
+        return self._tag
+      
+
+    @property
+    def start_datetime(self) -> datetime:
+        """The datetime of the batch file, up to seconds precision."""
+        return self._start_datetime
+    
+
+    @property
+    def parent_dir_path(self) -> str:
+        """The parent directory for the batch."""
+        return self._parent_dir_path
+
+
+    @property
+    def name(self) -> str:
+        """The name of the batch."""
+        return f"{self._start_time}_{self._tag}"
+    
+    
+    @property
+    def extensions(self) -> list[str]:
+        """All defined file extensions for the batch."""
+        return list(self._batch_files.keys())
+    
+    @property
+    def batch_files(self) -> dict[str, BatchFile]:
+        """Map each file extension in the batch to the corresponding batch file instance."""
+        return self._batch_files
+    
+    
+    def add_file(self, batch_file: BatchFile) -> None:
+        """Add an instance of a batch file to the batch."""
+        self._batch_files[batch_file.extension] = batch_file
+    
+
+    def get_file(self, extension: str) -> BatchFile:
+        """Get a batch file instance from the batch, according to the file extension."""
+        try:
+            return self._batch_files[extension]
+        except KeyError:
+            raise BatchFileNotFoundError(f"No batch file found with extension '{extension}'")
+
+
+    def read_file(self, extension: str):
+        """Read a file from the batch, according to the file extension."""
+        batch_file = self.get_file(extension)
+        return batch_file.read()
+
+
+    def delete_file(self, extension: str) -> None:
+        """Delete a file from the batch, according to the file extension."""
+        batch_file = self.get_file(extension)
+        try:
+            batch_file.delete()
+        except FileNotFoundError as e:
+            raise BatchFileNotFoundError(str(e))
+
+
+    def has_file(self, extension: str) -> bool:
+        """Return true if a file exists in the batch with the input file extension."""
+        try:
+            # only return true if both
+            # -> the batch has the extension defined
+            # -> the file with that extension exists in the batch parent directory
+            batch_file = self.get_file(extension)
+            return batch_file.exists
+        except BatchFileNotFoundError:
+            return False
+
+

spectre_core/batches/_batches.py

@@ -0,0 +1,197 @@
+# SPDX-FileCopyrightText: © 2024 Jimmy Fitzpatrick <jcfitzpatrick12@gmail.com>
+# This file is part of SPECTRE
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+from logging import getLogger
+_LOGGER = getLogger(__name__)
+
+import os
+from typing import Optional
+from collections import OrderedDict
+import warnings
+from datetime import datetime
+
+from spectre_core.spectrograms import Spectrogram, time_chop, join_spectrograms
+from spectre_core.config import get_batches_dir_path, TimeFormats
+from spectre_core.exceptions import (
+    SpectrogramNotFoundError,
+    BatchNotFoundError
+)
+from ._base import BaseBatch
+from ._factory import get_batch_cls_from_tag
+
+class Batches:
+    """A collection of batches for a given day of the year."""
+    def __init__(self, 
+                 tag: str,
+                 year: Optional[int] = None, 
+                 month: Optional[int] = None, 
+                 day: Optional[int] = None):
+        self._tag = tag
+        self._Batch = get_batch_cls_from_tag(tag)
+        self._batch_map: dict[str, BaseBatch] = OrderedDict()
+        self.set_date(year, month, day)
+
+
+    @property
+    def tag(self) -> str:
+        """Tag identifier for each batch."""
+        return self._tag
+
+
+    @property
+    def year(self) -> int:
+        """The numeric year."""
+        return self._year
+
+
+    @property 
+    def month(self) -> int:
+        """The numeric month of the year."""
+        return self._month
+    
+
+    @property
+    def day(self) -> int:
+        """The numeric day of the year."""
+        return self._day
+    
+
+    @property
+    def batches_dir_path(self) -> str:
+        """The parent directory for all the batches."""
+        return get_batches_dir_path(self.year, self.month, self.day)
+    
+
+    @property
+    def batch_list(self) -> list[BaseBatch]:
+        """A list of all the batch instances."""
+        return  list(self._batch_map.values())
+    
+
+    @property
+    def start_times(self) -> list[str]:
+        """The start times of each batch."""
+        return list(self._batch_map.keys())
+
+
+    @property
+    def num_batches(self) -> int:
+        """The number of batches in the batch parent directory."""
+        return len(self.batch_list)
+
+
+    def set_date(self, 
+                 year: Optional[int],
+                 month: Optional[int],
+                 day: Optional[int]) -> None:
+        """Update the parent directory for the batches according to the numeric date."""
+        self._year = year
+        self._month = month
+        self._day = day
+        self._update_batch_map()
+
+
+    def _update_batch_map(self) -> None:
+        # reset cache
+        self._batch_map = OrderedDict() 
+        
+        # get a list of all batch file names in the batches directory path
+        batch_file_names = [f for (_, _, files) in os.walk(self.batches_dir_path) for f in files]
+        for batch_file_name in batch_file_names:
+            # strip the extension
+            batch_name, _ = os.path.splitext(batch_file_name)
+            start_time, tag = batch_name.split("_", 1)
+            if tag == self._tag:
+                self._batch_map[start_time] = self._Batch(start_time, tag)
+        
+        self._batch_map = OrderedDict(sorted(self._batch_map.items()))
+
+
+    def update(self) -> None:
+        """Public alias for setting batch map"""
+        self._update_batch_map()
+    
+
+    def __iter__(self):
+        """Iterate over the stored batch instances."""
+        yield from self.batch_list
+
+
+    def _get_from_start_time(self, 
+                             start_time: str) -> BaseBatch:
+        """Get the batch according to the input start time."""
+        try:
+            return self._batch_map[start_time]
+        except KeyError:
+            raise BatchNotFoundError(f"Batch with start time {start_time} could not be found within {self.batches_dir_path}")
+
+
+    def _get_from_index(self, 
+                        index: int) -> BaseBatch:
+        """Get the batch according to its index, where the batches are ordered in time."""
+        num_batches = len(self.batch_list)
+        if num_batches == 0:
+            raise BatchNotFoundError("No batches are available")
+        index = index % num_batches  # Use modulo to make the index wrap around. Allows the user to iterate over all the batches via index cyclically.
+        return self.batch_list[index]
+
+
+    def __getitem__(self, subscript: str | int):
+        if isinstance(subscript, str):
+            return self._get_from_start_time(subscript)
+        elif isinstance(subscript, int):
+            return self._get_from_index(subscript)
+    
+
+    def num_batch_files(self, 
+                        extension: str) -> int:
+        """Get the number of existing batch files with the given extension."""
+        return sum(1 for batch_file in self if batch_file.has_file(extension))
+
+
+    def get_spectrogram_from_range(self, 
+                                   start_time: str, 
+                                   end_time: str) -> Spectrogram:
+        """Return a spectrogram over the input time range."""
+        # Convert input strings to datetime objects
+        start_datetime = datetime.strptime(start_time, TimeFormats.DATETIME)
+        end_datetime   = datetime.strptime(end_time, TimeFormats.DATETIME)
+
+        if start_datetime.day != end_datetime.day:
+            warning_message = "Joining spectrograms across multiple days"
+            _LOGGER.warning(warning_message)
+            warnings.warn(warning_message, RuntimeWarning)
+
+        spectrograms = []
+        num_fits_batch_files = self.num_batch_files("fits")
+
+        for i, batch in enumerate(self):
+            # skip batches without fits files
+            if not batch.has_file("fits"):
+                continue
+            
+            # rather than reading all files to evaluate the actual upper bound to their time range (slow)
+            # place an upper bound by using the start datetime for the next batch
+            # this assumes that the batches are non-overlapping (reasonable assumption)
+            lower_bound = batch.start_datetime
+            if i < num_fits_batch_files:
+                next_batch = self[i + 1]
+                upper_bound = next_batch.start_datetime
+            # if there is no "next batch" then we do have to read the file
+            else:
+                fits_batch = batch.get_file("fits")
+                upper_bound = fits_batch.datetimes[-1]
+
+            # if the batch overlaps with the input time range, then read the fits file
+            if start_datetime <= upper_bound and lower_bound <= end_datetime:
+                spectrogram = batch.read_file("fits")
+                spectrogram = time_chop(spectrogram, start_time, end_time)
+                # if we have a non-empty spectrogram, append it to the list of spectrograms
+                if spectrogram:
+                    spectrograms.append(spectrogram)
+
+        if spectrograms:
+            return join_spectrograms(spectrograms)
+        else:
+            raise SpectrogramNotFoundError("No spectrogram data found for the given time range")

spectre_core/batches/_factory.py

@@ -0,0 +1,27 @@
+# SPDX-FileCopyrightText: © 2024 Jimmy Fitzpatrick <jcfitzpatrick12@gmail.com>
+# This file is part of SPECTRE
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+from spectre_core.capture_configs import CaptureConfig, PNames
+from spectre_core.exceptions import BatchNotFoundError
+from ._register import batch_map
+from ._base import BaseBatch
+
+
+def _get_batch_cls(batch_key: str) -> BaseBatch:
+    Batch = batch_map.get(batch_key)
+    if Batch is None:
+        valid_batch_keys = list(batch_map.keys())
+        raise BatchNotFoundError(f"No batch found for the batch key: {batch_key}. Valid batch keys are: {valid_batch_keys}")
+    return Batch
+
+
+def get_batch_cls_from_tag(tag: str) -> BaseBatch:
+    # if we are dealing with a callisto batch, the batch key is equal to the tag
+    if "callisto" in tag:
+        batch_key = "callisto"
+    # otherwise, we fetch the batch key from the capture config
+    else:
+        capture_config= CaptureConfig(tag)
+        batch_key = capture_config.get_parameter_value(PNames.BATCH_KEY)
+    return _get_batch_cls(batch_key)

spectre_core/{chunks → batches}/_register.py

@@ -3,13 +3,13 @@
 # SPDX-License-Identifier: GPL-3.0-or-later
 
 #  Global dictionaries to hold the mappings
-chunk_map = {}
+batch_map = {}
 
-# classes decorated with @register_chunk([CHUNK_KEY])
-# will be added to chunk_map
-def register_chunk(chunk_key: str):
+# classes decorated with @register_batch([BATCH_KEY])
+# will be added to batch_map
+def register_batch(batch_key: str):
     def decorator(cls):
-        chunk_map[chunk_key] = cls
+        batch_map[batch_key] = cls
         return cls
     return decorator
 

spectre_core/{chunks → batches}/library/_callisto.py

@@ -11,43 +11,46 @@ from astropy.io.fits.hdu.image import PrimaryHDU
 from astropy.io.fits.hdu.table import BinTableHDU
 from astropy.io.fits.hdu.hdulist import HDUList
 
-from spectre_core.spectrograms import Spectrogram
-from .._register import register_chunk
-from .._base import BaseChunk, ChunkFile
+from spectre_core.spectrograms import Spectrogram, SpectrumTypes
+from .._register import register_batch
+from .._base import BaseBatch, BatchFile
 
 
-@register_chunk('callisto')
-class _Chunk(BaseChunk):
-    def __init__(self, chunk_start_time: str, tag: str):
-        super().__init__(chunk_start_time, tag) 
-        self.add_file(FitsChunk(self.chunk_parent_path, self.chunk_name))
+@register_batch('callisto')
+class _Batch(BaseBatch):
+    def __init__(self,
+                 start_time: str,
+                 tag: str):
+        super().__init__(start_time, tag) 
+        self.add_file( FitsFile(self.parent_dir_path, self.name) )
 
 
-class FitsChunk(ChunkFile):
-    def __init__(self, chunk_parent_path: str, chunk_name: str):
-        super().__init__(chunk_parent_path, chunk_name, "fits")
+class FitsFile(BatchFile):
+    def __init__(self, 
+                 parent_dir_path: str, 
+                 base_file_name: str):
+        super().__init__(parent_dir_path, base_file_name, "fits")
 
 
     def read(self) -> Spectrogram:
         with fits.open(self.file_path, mode='readonly') as hdulist:
-            primary_hdu = self._get_primary_hdu(hdulist)
-            dynamic_spectra = self._get_dynamic_spectra(primary_hdu)
-            microsecond_correction = self._get_microsecond_correction(primary_hdu)
-            bintable_hdu = self._get_bintable_hdu(hdulist)
-            times, frequencies = self._get_time_and_frequency(bintable_hdu)
-            spectrum_type = self._get_spectrum_type(primary_hdu)
-
-            if spectrum_type == "digits":
+            primary_hdu                = self._get_primary_hdu(hdulist)
+            dynamic_spectra            = self._get_dynamic_spectra(primary_hdu)
+            spectrogram_start_datetime = self._get_spectrogram_start_datetime(primary_hdu)
+            bintable_hdu               = self._get_bintable_hdu(hdulist)
+            times, frequencies         = self._get_time_and_frequency(bintable_hdu)
+            spectrum_type              = self._get_spectrum_type(primary_hdu)
+
+            if spectrum_type == SpectrumTypes.DIGITS:
                 dynamic_spectra_linearised = self._convert_units_to_linearised(dynamic_spectra)
                 return Spectrogram(dynamic_spectra_linearised[::-1, :], # reverse the spectra along the frequency axis
                         times, 
                         frequencies[::-1], # sort the frequencies in ascending order
                         self.tag, 
-                        chunk_start_time=self.chunk_start_time, 
-                        microsecond_correction = microsecond_correction,
-                        spectrum_type = spectrum_type)
+                        spectrogram_start_datetime,
+                        spectrum_type)
             else:
-                raise NotImplementedError(f"SPECTRE does not currently support spectrum type with BUNITS {spectrum_type}")
+                raise NotImplementedError(f"SPECTRE does not currently support spectrum type with BUNITS '{spectrum_type}'")
 
 
     @property
@@ -55,7 +58,7 @@ class FitsChunk(ChunkFile):
         with fits.open(self.file_path, mode='readonly') as hdulist:
             bintable_data = hdulist[1].data
             times = bintable_data['TIME'][0]
-            return [self.chunk_start_datetime + timedelta(seconds=t) for t in times]        
+            return [self.start_datetime + timedelta(seconds=t) for t in times]        
 
 
     def _get_primary_hdu(self, hdulist: HDUList) -> PrimaryHDU:
@@ -66,11 +69,10 @@ class FitsChunk(ChunkFile):
         return primary_hdu.data
 
 
-    def _get_microsecond_correction(self, primary_hdu: PrimaryHDU) -> int:
-        date_obs = primary_hdu.header.get('DATE-OBS', None)
-        time_obs = primary_hdu.header.get('TIME-OBS', None)
-        datetime_obs = datetime.strptime(f"{date_obs}T{time_obs}", "%Y/%m/%dT%H:%M:%S.%f")
-        return datetime_obs.microsecond
+    def _get_spectrogram_start_datetime(self, primary_hdu: PrimaryHDU) -> datetime:
+        date_obs = primary_hdu.header['DATE-OBS']
+        time_obs = primary_hdu.header['TIME-OBS']
+        return datetime.strptime(f"{date_obs}T{time_obs}", "%Y-%m-%dT%H:%M:%S.%f")
 
 
     def _get_bintable_hdu(self, hdulist: HDUList) -> BinTableHDU:
@@ -85,10 +87,6 @@ class FitsChunk(ChunkFile):
         return times, frequencies
 
 
-    def _get_spectrum_type(self, primary_hdu: PrimaryHDU) -> str:
-        return primary_hdu.header.get('BUNIT', None)
-
-
     def _convert_units_to_linearised(self, dynamic_spectra: np.ndarray) -> np.ndarray:
         digits_floats = np.array(dynamic_spectra, dtype='float')
         # conversion as per ADC specs [see email from C. Monstein]