spectre-core 0.0.1__py3-none-any.whl

spectre_core/__init__.py

@@ -0,0 +1,3 @@
+# SPDX-FileCopyrightText: © 2024 Jimmy Fitzpatrick <jcfitzpatrick12@gmail.com>
+# This file is part of SPECTRE
+# SPDX-License-Identifier: GPL-3.0-or-later

spectre_core/cfg.py

@@ -0,0 +1,116 @@
+# SPDX-FileCopyrightText: © 2024 Jimmy Fitzpatrick <jcfitzpatrick12@gmail.com>
+# This file is part of SPECTRE
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+import os
+
+SPECTRE_DATA_DIR_PATH = os.environ.get("SPECTRE_DATA_DIR_PATH")
+if SPECTRE_DATA_DIR_PATH is None:
+    raise ValueError("The environment variable SPECTRE_DATA_DIR_PATH has not been set")
+
+CHUNKS_DIR_PATH = os.environ.get("SPECTRE_CHUNKS_DIR_PATH", 
+                                 os.path.join(SPECTRE_DATA_DIR_PATH, 'chunks'))
+os.makedirs(CHUNKS_DIR_PATH, 
+            exist_ok=True)
+
+LOGS_DIR_PATH = os.environ.get("SPECTRE_LOGS_DIR_PATH",
+                               os.path.join(SPECTRE_DATA_DIR_PATH, 'logs'))
+os.makedirs(LOGS_DIR_PATH, 
+            exist_ok=True)
+
+JSON_CONFIGS_DIR_PATH = os.environ.get("SPECTRE_JSON_CONFIGS_DIR_PATH",
+                                       os.path.join(SPECTRE_DATA_DIR_PATH, "configs"))
+os.makedirs(JSON_CONFIGS_DIR_PATH, 
+            exist_ok=True)
+
+DEFAULT_TIME_FORMAT = "%H:%M:%S"
+DEFAULT_DATE_FORMAT = "%Y-%m-%d"
+DEFAULT_DATETIME_FORMAT = f"{DEFAULT_DATE_FORMAT}T{DEFAULT_TIME_FORMAT}"
+
+CALLISTO_INSTRUMENT_CODES = [
+    "ALASKA-ANCHORAGE",
+    "ALASKA-COHOE",
+    "ALASKA-HAARP",
+    "ALGERIA-CRAAG",
+    "ALMATY",
+    "AUSTRIA-Krumbach",
+    "AUSTRIA-MICHELBACH",
+    "AUSTRIA-OE3FLB",
+    "AUSTRIA-UNIGRAZ",
+    "Australia-ASSA",
+    "BIR",
+    "Croatia-Visnjan",
+    "DENMARK",
+    "EGYPT-Alexandria",
+    "EGYPT-SpaceAgency",
+    "FINLAND-Siuntio",
+    "Finland-Kempele",
+    "GERMANY-DLR",
+    "GLASGOW",
+    "GREENLAND",
+    "HUMAIN",
+    "HURBANOVO",
+    "INDIA-GAURI",
+    "INDIA-OOTY",
+    "INDIA-UDAIPUR",
+    "JAPAN-IBARAKI",
+    "KASI",
+    "MEXART",
+    "MEXICO-FCFM-UANL",
+    "MEXICO-LANCE-A",
+    "MEXICO-LANCE-B",
+    "MONGOLIA-UB",
+    "MRO",
+    "MRT3",
+    "Malaysia-Banting",
+    "NORWAY-EGERSUND",
+    "NORWAY-NY-AALESUND",
+    "NORWAY-RANDABERG",
+    "POLAND-Grotniki",
+    "ROMANIA",
+    "ROSWELL-NM",
+    "SPAIN-PERALEJOS",
+    "SSRT",
+    "SWISS-HB9SCT",
+    "SWISS-HEITERSWIL",
+    "SWISS-IRSOL",
+    "SWISS-Landschlacht",
+    "SWISS-MUHEN",
+    "TRIEST",
+    "TURKEY",
+    "UNAM",
+    "URUGUAY",
+    "USA-BOSTON",
+]
+
+def _get_date_based_dir_path(base_dir: str, year: int = None, 
+                             month: int = None, day: int = None
+) -> str:
+    if day and not (year and month):
+        raise ValueError("A day requires both a month and a year")
+    if month and not year:
+        raise ValueError("A month requires a year")
+    
+    date_dir_components = []
+    if year:
+        date_dir_components.append(f"{year:04}")
+    if month:
+        date_dir_components.append(f"{month:02}")
+    if day:
+        date_dir_components.append(f"{day:02}")
+    
+    return os.path.join(base_dir, *date_dir_components)
+
+
+def get_chunks_dir_path(year: int = None, 
+                        month: int = None, 
+                        day: int = None
+) -> str:
+    return _get_date_based_dir_path(CHUNKS_DIR_PATH, year, month, day)
+
+
+def get_logs_dir_path(year: int = None, 
+                      month: int = None, 
+                      day: int = None
+) -> str:
+    return _get_date_based_dir_path(LOGS_DIR_PATH, year, month, day)

spectre_core/chunks/__init__.py

@@ -0,0 +1,206 @@
+# SPDX-FileCopyrightText: © 2024 Jimmy Fitzpatrick <jcfitzpatrick12@gmail.com>
+# This file is part of SPECTRE
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+from logging import getLogger
+_LOGGER = getLogger(__name__)
+
+import os
+from typing import Optional
+from collections import OrderedDict
+import warnings
+from datetime import datetime
+
+# dynamically import all chunks
+import spectre_core.chunks.library
+from spectre_core.chunks.factory import get_chunk_from_tag
+from spectre_core.chunks.base import BaseChunk
+from spectre_core.spectrograms.spectrogram import Spectrogram
+from spectre_core.spectrograms import transform
+from spectre_core.cfg import (
+    DEFAULT_DATETIME_FORMAT,
+    get_chunks_dir_path
+)
+from spectre_core.exceptions import (
+    SpectrogramNotFoundError,
+    ChunkNotFoundError
+)
+
+class Chunks:
+    def __init__(self, 
+                 tag: str,
+                 year: Optional[int] = None, 
+                 month: Optional[int] = None, 
+                 day: Optional[int] = None):
+        self._tag = tag
+        self._Chunk = get_chunk_from_tag(tag)
+        self._chunk_map: dict[str, BaseChunk] = OrderedDict()
+        self._chunk_list: list[BaseChunk] = []
+        self._chunk_names: list[str] = []
+        self.set_date(year, month, day)
+
+
+    @property
+    def year(self) -> int:
+        return self._year
+
+
+    @property 
+    def month(self) -> int:
+        return self._month
+    
+
+    @property
+    def day(self) -> int:
+        return self._day
+
+
+    @property
+    def chunks_dir_path(self) -> str:
+        return get_chunks_dir_path(self.year, self.month, self.day)
+    
+
+    @property 
+    def chunk_map(self) -> dict[str, BaseChunk]:
+        return self._chunk_map
+    
+
+    @property
+    def chunk_list(self) -> list[BaseChunk]:
+        return  self._chunk_list
+    
+
+    @property
+    def chunk_names(self) -> list[str]:
+        return self._chunk_names
+
+
+    @property
+    def num_chunks(self) -> int:
+        return len(self.chunk_list)
+
+
+    def set_date(self, 
+                 year: Optional[int],
+                 month: Optional[int],
+                 day: Optional[int]) -> None:
+        self._year = year
+        self._month = month
+        self._day = day
+        self._update_chunk_map()
+
+
+    def _update_chunk_map(self) -> None:
+        self._chunk_map = OrderedDict() # reset cache
+        self._chunk_list = [] # reset cache
+        self._chunk_names = [] # reset cache
+
+        chunk_files = [f for (_, _, files) in os.walk(self.chunks_dir_path) for f in files]
+        
+        if len(chunk_files) == 0:
+            warning_message = "No chunks found, setting chunk map with empty dictionary."
+            _LOGGER.warning(warning_message)
+            warnings.warn(warning_message)
+            return
+        
+        for chunk_file in chunk_files:
+            file_name, _ = os.path.splitext(chunk_file)
+            chunk_start_time, tag = file_name.split("_", 1)
+            if tag == self._tag:
+                self._chunk_map[chunk_start_time] = self._Chunk(chunk_start_time, tag)
+        
+        self._chunk_map = OrderedDict(sorted(self._chunk_map.items()))
+        self._chunk_names = list(self._chunk_map.keys())
+        self._chunk_list = list(self._chunk_map.values())
+
+
+    def update(self) -> None:
+        """Public alias for setting chunk map"""
+        self._update_chunk_map()
+    
+
+    def __iter__(self):
+        yield from self.chunk_list
+
+
+    def _get_chunk_by_chunk_start_time(self, 
+                                      chunk_start_time: str) -> BaseChunk:
+        try:
+            return self.chunk_map[chunk_start_time]
+        except KeyError:
+            raise ChunkNotFoundError(f"Chunk with chunk start time {chunk_start_time} could not be found within {self.chunks_dir_path}")
+
+
+    def _get_chunk_by_index(self, 
+                           chunk_index: int) -> BaseChunk:
+        num_chunks = len(self.chunk_map)
+        if num_chunks == 0:
+            raise ChunkNotFoundError("No chunks are available")
+        index = chunk_index % num_chunks  # Use modulo to make the index wrap around. Allows the user to iterate over all the chunks via index cyclically.
+        return self.chunk_list[index]
+
+
+    def __getitem__(self, subscript: str | int):
+        if isinstance(subscript, str):
+            return self._get_chunk_by_chunk_start_time(subscript)
+        elif isinstance(subscript, int):
+            return self._get_chunk_by_index(subscript)
+
+
+    def get_index_by_chunk(self, 
+                           chunk_to_match: BaseChunk) -> int:
+        for i, chunk in enumerate(self):
+            if chunk.chunk_start_time == chunk_to_match.chunk_start_time:
+                return i
+        raise ChunkNotFoundError(f"No matching chunk found for chunk {chunk_to_match.chunk_name}")
+    
+
+    def count_chunk_files(self, 
+                          extension: str) -> int:
+        return sum(1 for chunk_file in self if chunk_file.has_file(extension))
+
+
+    def get_spectrogram_from_range(self, 
+                                   start_time: str, 
+                                   end_time: str) -> Spectrogram:
+        # Convert input strings to datetime objects
+        start_datetime = datetime.strptime(start_time, DEFAULT_DATETIME_FORMAT)
+        end_datetime = datetime.strptime(end_time, DEFAULT_DATETIME_FORMAT)
+
+        if start_datetime.day != end_datetime.day:
+            warning_message = "Joining spectrograms across multiple days"
+            _LOGGER.warning(warning_message)
+            warnings.warn(warning_message, RuntimeWarning)
+
+        spectrograms = []
+        num_fits_chunks = self.count_chunk_files("fits")
+
+        for i, chunk in enumerate(self):
+            # skip chunks without fits files
+            if not chunk.has_file("fits"):
+                continue
+            
+            # rather than reading all files to evaluate the actual upper bound to their time range (slow)
+            # place an upper bound by using the chunk start datetime for the next chunk
+            # this assumes that the chunks are non-overlapping (reasonable assumption)
+            lower_bound = chunk.chunk_start_datetime
+            if i < num_fits_chunks:
+                next_chunk = self[i + 1]
+                upper_bound = next_chunk.chunk_start_datetime
+            # if there is no "next chunk" then we do have to read the file
+            else:
+                fits_chunk = chunk.get_file("fits")
+                upper_bound = fits_chunk.datetimes[-1]
+
+            # if the chunk overlaps with the input time range, then read the fits file
+            if start_datetime <= upper_bound and lower_bound <= end_datetime:
+                spectrogram = chunk.read_file("fits")
+                spectrogram = transform.time_chop(spectrogram, start_time, end_time)
+                # if we have a non-empty spectrogram, append it to the list of spectrograms
+                if spectrogram:
+                    spectrograms.append(spectrogram)
+
+        if spectrograms:
+            return transform.join_spectrograms(spectrograms)
+        else:
+            raise SpectrogramNotFoundError("No spectrogram data found for the given time range")

spectre_core/chunks/base.py

@@ -0,0 +1,160 @@
+# SPDX-FileCopyrightText: © 2024 Jimmy Fitzpatrick <jcfitzpatrick12@gmail.com>
+# This file is part of SPECTRE
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+from datetime import datetime
+from abc import abstractmethod
+from typing import Optional
+
+from scipy.signal import ShortTimeFFT, get_window
+
+from spectre_core.file_handlers.base import BaseFileHandler
+from spectre_core.cfg import get_chunks_dir_path
+from spectre_core.file_handlers.configs import CaptureConfig
+from spectre_core.spectrograms.spectrogram import Spectrogram
+from spectre_core.cfg import DEFAULT_DATETIME_FORMAT
+from spectre_core.exceptions import ChunkFileNotFoundError
+
+class ChunkFile(BaseFileHandler):
+    def __init__(self, 
+                 chunk_parent_path: str, 
+                 chunk_name: str, 
+                 extension: str, 
+                 **kwargs):
+        self._chunk_start_time, self._tag = chunk_name.split("_")
+        self._chunk_start_datetime: Optional[datetime] = None
+        super().__init__(chunk_parent_path, 
+                         chunk_name, 
+                         extension = extension, 
+                         **kwargs)
+
+
+    @property
+    def chunk_start_time(self) -> str:
+        return self._chunk_start_time
+    
+
+    @property
+    def chunk_start_datetime(self) -> datetime:
+        if self._chunk_start_datetime is None:
+            self._chunk_start_datetime = datetime.strptime(self.chunk_start_time, DEFAULT_DATETIME_FORMAT)
+        return self._chunk_start_datetime
+    
+
+    @property
+    def tag(self) -> str:
+        return self._tag
+    
+
+
+class BaseChunk:
+    def __init__(self, 
+                 chunk_start_time: str, 
+                 tag: str):
+        self._chunk_start_time: str = chunk_start_time
+        self._tag: str = tag
+        self._chunk_files: dict[str, ChunkFile] = {}
+        self._chunk_start_datetime: Optional[datetime] = None
+        self.chunk_parent_path: str = get_chunks_dir_path(year = self.chunk_start_datetime.year,
+                                                          month = self.chunk_start_datetime.month,
+                                                          day = self.chunk_start_datetime.day)
+        self._chunk_name: str = f"{self.chunk_start_time}_{self.tag}"
+
+
+    @property
+    def chunk_start_time(self) -> str:
+        return self._chunk_start_time
+    
+
+    @property
+    def chunk_start_datetime(self) -> datetime:
+        if self._chunk_start_datetime is None:
+            self._chunk_start_datetime = datetime.strptime(self.chunk_start_time, DEFAULT_DATETIME_FORMAT)
+        return self._chunk_start_datetime
+    
+
+    @property
+    def tag(self) -> str:
+        return self._tag
+
+
+    @property
+    def chunk_name(self) -> str:
+        return f"{self._chunk_start_time}_{self._tag}"
+    
+
+    @property
+    def extensions(self) -> list[str]:
+        return list(self._chunk_files.keys())
+    
+    
+    def add_file(self, chunk_file: ChunkFile) -> None:
+        self._chunk_files[chunk_file.extension] = chunk_file
+    
+
+    def get_file(self, extension: str) -> ChunkFile:
+        try:
+            return self._chunk_files[extension]
+        except KeyError:
+            raise ChunkFileNotFoundError(f"No chunk file found with extension '{extension}'")
+
+
+    def read_file(self, extension: str):
+        chunk_file = self.get_file(extension)
+        return chunk_file.read()
+
+
+    def delete_file(self, extension: str, **kwargs):
+        chunk_file = self.get_file(extension)
+        try:
+            chunk_file.delete(**kwargs)
+        except FileNotFoundError as e:
+            raise ChunkFileNotFoundError(str(e))
+
+
+    def has_file(self, extension: str) -> bool:
+        try:
+            chunk_file = self.get_file(extension)
+            return chunk_file.exists
+        except ChunkFileNotFoundError:
+            return False
+
+
+class SPECTREChunk(BaseChunk):
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+
+        self._capture_config = CaptureConfig(self._tag)
+        self._SFT = None # cache
+
+
+    @abstractmethod
+    def build_spectrogram(self) -> Spectrogram:
+        """Create a spectrogram object derived from chunk files for this chunk."""
+        pass
+
+
+    @property
+    def capture_config(self) -> CaptureConfig:
+        return self._capture_config
+    
+
+    @property
+    def SFT(self) -> ShortTimeFFT:
+        if self._SFT is None:
+            self._SFT = self.__get_SFT_instance()
+        return self._SFT
+    
+
+    def __get_SFT_instance(self) -> ShortTimeFFT:
+        window_type = self.capture_config.get("window_type")
+        window_params = self.capture_config.get("window_kwargs").values()
+        window_size = self.capture_config.get("window_size")
+        window = get_window((window_type, 
+                             *window_params), 
+                             window_size)
+        samp_rate = self.capture_config.get("samp_rate")
+        return ShortTimeFFT(window, 
+                            fs=samp_rate, 
+                            fft_mode="centered", 
+                            **self.capture_config.get("STFFT_kwargs"))

spectre_core/chunks/chunk_register.py

@@ -0,0 +1,15 @@
+# SPDX-FileCopyrightText: © 2024 Jimmy Fitzpatrick <jcfitzpatrick12@gmail.com>
+# This file is part of SPECTRE
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+#  Global dictionaries to hold the mappings
+chunk_map = {}
+
+# classes decorated with @register_chunk([CHUNK_KEY])
+# will be added to chunk_map
+def register_chunk(chunk_key: str):
+    def decorator(cls):
+        chunk_map[chunk_key] = cls
+        return cls
+    return decorator
+

spectre_core/chunks/factory.py

@@ -0,0 +1,26 @@
+# SPDX-FileCopyrightText: © 2024 Jimmy Fitzpatrick <jcfitzpatrick12@gmail.com>
+# This file is part of SPECTRE
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+# after we decorate all chunks, we can import the chunk_key -> chunk maps
+from spectre_core.chunks.base import BaseChunk
+from spectre_core.file_handlers.configs import CaptureConfig
+from spectre_core.exceptions import ChunkNotFoundError
+
+
+def get_chunk(chunk_key: str) -> BaseChunk:
+    Chunk = chunk_map.get(chunk_key)
+    if Chunk is None:
+        valid_chunk_keys = list(chunk_map.keys())
+        raise ChunkNotFoundError(f"No chunk found for the chunk key: {chunk_key}. Valid chunk keys are: {valid_chunk_keys}")
+    return Chunk
+
+def get_chunk_from_tag(tag: str) -> BaseChunk:
+    # if we are dealing with a callisto chunk, the chunk key is equal to the tag
+    if "callisto" in tag:
+        chunk_key = "callisto"
+    # otherwise, we fetch the chunk key from the capture config
+    else:
+        capture_config= CaptureConfig(tag)
+        chunk_key = capture_config.get('chunk_key')
+    return get_chunk(chunk_key)

spectre_core/chunks/library/__init__.py

@@ -0,0 +1,8 @@
+# SPDX-FileCopyrightText: © 2024 Jimmy Fitzpatrick <jcfitzpatrick12@gmail.com>
+# This file is part of SPECTRE
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+from spectre_core.dynamic_imports import import_target_modules
+
+import_target_modules(__file__, __name__, "chunk")
+

spectre_core/chunks/library/callisto/chunk.py

@@ -0,0 +1,101 @@
+# SPDX-FileCopyrightText: © 2024 Jimmy Fitzpatrick <jcfitzpatrick12@gmail.com>
+# This file is part of SPECTRE
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+from datetime import datetime, timedelta
+from typing import Tuple
+
+import numpy as np
+from astropy.io import fits
+from astropy.io.fits.hdu.image import PrimaryHDU
+from astropy.io.fits.hdu.table import BinTableHDU
+from astropy.io.fits.hdu.hdulist import HDUList
+
+from spectre_core.chunks.chunk_register import register_chunk
+from spectre_core.spectrograms.spectrogram import Spectrogram
+from spectre_core.chunks.base import (
+    BaseChunk, 
+    ChunkFile
+)
+
+
+@register_chunk('callisto')
+class Chunk(BaseChunk):
+    def __init__(self, chunk_start_time: str, tag: str):
+        super().__init__(chunk_start_time, tag) 
+        self.add_file(FitsChunk(self.chunk_parent_path, self.chunk_name))
+
+
+class FitsChunk(ChunkFile):
+    def __init__(self, chunk_parent_path: str, chunk_name: str):
+        super().__init__(chunk_parent_path, chunk_name, "fits")
+
+
+    def read(self) -> Spectrogram:
+        with fits.open(self.file_path, mode='readonly') as hdulist:
+            primary_hdu = self._get_primary_hdu(hdulist)
+            dynamic_spectra = self._get_dynamic_spectra(primary_hdu)
+            microsecond_correction = self._get_microsecond_correction(primary_hdu)
+            bintable_hdu = self._get_bintable_hdu(hdulist)
+            times, frequencies = self._get_time_and_frequency(bintable_hdu)
+            spectrum_type = self._get_spectrum_type(primary_hdu)
+
+            if spectrum_type == "digits":
+                dynamic_spectra_linearised = self._convert_units_to_linearised(dynamic_spectra)
+                return Spectrogram(dynamic_spectra_linearised[::-1, :], # reverse the spectra along the frequency axis
+                        times, 
+                        frequencies[::-1], # sort the frequencies in ascending order
+                        self.tag, 
+                        chunk_start_time=self.chunk_start_time, 
+                        microsecond_correction = microsecond_correction,
+                        spectrum_type = spectrum_type)
+            else:
+                raise NotImplementedError(f"SPECTRE does not currently support spectrum type with BUNITS {spectrum_type}")
+
+
+    @property
+    def datetimes(self) -> np.ndarray:
+        with fits.open(self.file_path, mode='readonly') as hdulist:
+            bintable_data = hdulist[1].data
+            times = bintable_data['TIME'][0]
+            return [self.chunk_start_datetime + timedelta(seconds=t) for t in times]        
+
+
+    def _get_primary_hdu(self, hdulist: HDUList) -> PrimaryHDU:
+        return hdulist['PRIMARY']
+
+
+    def _get_dynamic_spectra(self, primary_hdu: PrimaryHDU) -> np.ndarray:
+        return primary_hdu.data
+
+
+    def _get_microsecond_correction(self, primary_hdu: PrimaryHDU) -> int:
+        date_obs = primary_hdu.header.get('DATE-OBS', None)
+        time_obs = primary_hdu.header.get('TIME-OBS', None)
+        datetime_obs = datetime.strptime(f"{date_obs}T{time_obs}", "%Y/%m/%dT%H:%M:%S.%f")
+        return datetime_obs.microsecond
+
+
+    def _get_bintable_hdu(self, hdulist: HDUList) -> BinTableHDU:
+        return hdulist[1]
+
+
+    def _get_time_and_frequency(self, bintable_hdu: BinTableHDU) -> Tuple[np.ndarray, np.ndarray]:
+        data = bintable_hdu.data
+        times = data['TIME'][0]
+        frequencies_MHz = data['FREQUENCY'][0]
+        frequencies = frequencies_MHz * 1e6 # convert to Hz
+        return times, frequencies
+
+
+    def _get_spectrum_type(self, primary_hdu: PrimaryHDU) -> str:
+        return primary_hdu.header.get('BUNIT', None)
+
+
+    def _convert_units_to_linearised(self, dynamic_spectra: np.ndarray) -> np.ndarray:
+        digits_floats = np.array(dynamic_spectra, dtype='float')
+        # conversion as per ADC specs [see email from C. Monstein]
+        dB = (digits_floats / 255) * (2500 / 25)
+        return 10 ** (dB / 10)
+
+