PyDIET 0.9.3__py3-none-any.whl

pydiet/server/datafiles.py

@@ -0,0 +1,367 @@
+"""
+Functions that gather data from files.
+"""
+# Copyright CFHT/CNRS/CEA/UParisSaclay
+# Licensed under the MIT licence
+
+from os import PathLike, scandir
+from os.path import basename, exists, isabs, join
+from pathlib import Path
+
+# Manage TOML library for Python versions < 3.11
+import sys
+if sys.version_info >= (3, 11):
+    import tomllib
+else:
+    import tomli as tomllib
+import warnings
+
+from typing import Any, IO, Optional
+from astropy.table import QTable  #type: ignore[import-untyped]
+from astropy import units as u  #type: ignore[import-untyped]
+from astropy.modeling.models import Const1D  #type: ignore[import-untyped]
+from astropy.utils.exceptions import AstropyUserWarning  #type: ignore[import-untyped]
+from pydantic import BaseModel, Field
+from specutils import Spectrum  #type: ignore[import-untyped]
+from synphot import (  #type: ignore[import-untyped]
+    SourceSpectrum,
+    SpectralElement,
+    ThermalSpectralElement
+) #type: ignore[import-untyped]
+
+from .. import package
+from .config import override, settings
+from .models.dataconfig import (
+    DataConfigModel,
+    DetectorConfigModel,
+    EmissionConfigModel,
+    FiltersConfigModel,
+    OpticsConfigModel,
+    TransmissionConfigModel
+)
+from .models.instrument import (
+    DetectorModel,
+    FiltersModel,
+    InstrumentModel,
+    OpticsModel,
+    SBSEDModel,
+    SiteModel,
+    TelescopeModel,
+    TransmissionModel
+)
+
+def add_trans(self, other):
+    """Add ``self`` with ``other``."""
+    self._validate_other_mul_div(other)
+    result = self.__class__(self.model + other.model)
+    self._merge_meta(self, other, result)
+    return result
+
+
+SpectralElement.__add__ = add_trans
+
+
+def get_data_config(data_config: Optional[str] = None) -> DataConfigModel:
+    data_config = override("data_config", data_config)
+    assert data_config is not None # This is to make mypy happy
+    with open(data_config, "rb") as f:
+        data = tomllib.load(f)
+    assert data is not None # This is to make mypy happy
+    # if "path" value is not absolute, assume it is relative to pkg root dir
+    if not isabs(data['path']):
+        data['path'] = join(package.root_dir, data['path'])
+    data_config_model = DataConfigModel.model_validate(data)
+    return data_config_model
+
+
+def get_data_file(filename: IO[bytes] | PathLike | str):
+    return QTable.read(filename)
+
+
+def get_default(d: dict) -> Any:
+    lst = [val for val in d.values() if val.default]
+    return lst[0] if len(lst) > 0 else list(d.values())[0]
+
+
+def get_detector(
+        parent_dir: str,
+        detector: DetectorConfigModel) -> DetectorModel:
+    # Instantiate the model
+    transmissions = get_transmissions(
+            join(parent_dir, detector.path),
+            detector.transmission
+    )
+    emissions = get_emissions(
+            join(parent_dir, detector.path),
+            detector.emission
+    )
+    return DetectorModel(
+        gain = detector.gain,
+        ron = detector.ron,
+        scale = detector.scale,
+        transmissions = transmissions,
+        emissions = emissions
+    )
+
+
+def get_emission(
+        file: str,
+        id: str,
+        name: str="",
+        description: str="",
+        vars: dict[str, float | str]={},
+    ) -> SBSEDModel:
+    data = get_data_file(file)
+    wave = u.Quantity(data['WAVELENGTH'])
+    sed = u.Quantity(data['PHOTLAM']).to(
+        u.Jy / u.arcsec**2,
+        equivalencies=u.spectral_density(wave)
+    )
+    # Instantiate the model
+    with warnings.catch_warnings():
+        warnings.filterwarnings(
+            "ignore",
+            message=".*negative flux or throughput.*",
+            category=AstropyUserWarning,
+        )
+        emission = SBSEDModel(
+            id = id,
+            name = name,
+            description = description,
+            vars = vars,
+            # We drop the surface part as Spectrum does cannot deal with SBs.
+            spectral = SourceSpectrum.from_spectrum1d(
+                Spectrum(
+                    spectral_axis = wave,
+                    flux = sed * u.arcsec**2
+                ),
+                keep_neg=False
+            )
+        )
+    return emission
+
+
+def get_emission_from_transmission(
+        transmission: TransmissionModel,
+        temperature: u.Quantity['temperature'],  #type: ignore[name-defined]
+        id: str) -> SBSEDModel:
+    flat = SpectralElement(Const1D, amplitude=1.)
+    assert transmission.spectral != None    # Make mypy happy
+    emission = SBSEDModel(
+        id = id,
+        name = f"{transmission.name} emission",
+        description = f"Blackbody emission at {temperature.to(u.K).value:.1f} K",
+        # Thermal source spectral flux with Blackbody spectrum over 1 arcsec2
+        spectral = ThermalSpectralElement(
+            flat + (-1.) * transmission.spectral, # Apply emissivity
+            temperature=temperature,
+            beam_fill_factor=1.0
+        ).thermal_source()
+    )
+    return emission
+
+
+"""
+def get_emission_from_transmission(
+        transmission: TransmissionModel,
+        temperature: u.Quantity['temperature'],  #type: ignore[name-defined]
+        area: u.Quantity['area'],  #type: ignore[name-defined]
+        id: str) -> SBSEDModel:
+    # Thermal source spectral flux with Blackbody spectrum over 1 arcsec2
+    bb = ThermalSpectralElement(
+        BlackBody1D,
+        temperature=temperature
+    ).thermal_source() * area.to(u.m**2).value
+    emission = SBSEDModel(
+        id = id,
+        name = f"{transmission.name} emission",
+        description = f"Blackbody emission at {temperature.to(u.K).value:.1f} K",
+        # Apply emissivity
+        spectral = bb - bb * transmission.spectral
+    )
+    return emission
+"""
+
+
+def get_emissions(
+        parent_dir: str,
+        emission_config: EmissionConfigModel,
+        transmissions: dict[str, TransmissionModel] | None = None
+        ) -> dict[str, SBSEDModel]:
+    emissions : dict[str, SBSEDModel] = {}
+    for file_config in emission_config.files:
+        key = file_config.id if file_config.id != '' else str(len(emissions))
+        emissions[key] = get_emission(
+            file=join(parent_dir, emission_config.path, file_config.file),
+            id=key,
+            name=file_config.name,
+            description = file_config.description,
+            vars = file_config.vars
+        )
+    # No emission files: we use a blackbody with emissivity from transmission
+    if len(emission_config.files) == 0 and transmissions is not None:
+        temperatures = emission_config.temperatures
+        for t, key in enumerate(transmissions):
+            temperature = temperatures[t] if t < len(temperatures) \
+                else temperatures[-1]
+            emissions[key] = get_emission_from_transmission(
+                transmissions[key],
+                temperature=temperature,
+                id=key
+            )
+    return emissions
+
+
+def get_filters(
+        parent_dir: str,
+        filters_config: FiltersConfigModel) -> FiltersModel:
+    path = join(parent_dir, filters_config.path)
+    transmissions = get_transmissions(path, filters_config.transmission)
+    # For emissions we may have to use transmission curves
+    emissions = get_emissions(
+        path,
+        filters_config.emission,
+        transmissions
+    )
+    return FiltersModel(
+        transmissions=transmissions,
+        emissions=emissions
+    )
+
+
+def get_instruments(
+        data_config: DataConfigModel) -> dict:
+    # Start by gathering the provided sites and telescopes
+    sites = get_sites(data_config)
+    telescopes = get_telescopes(data_config)
+    instruments = {}
+    for instrument in data_config.instruments:
+        path = join(data_config.path, instrument.path)
+        # Instantiate the model
+        instruments[instrument.id] = InstrumentModel(
+            id = instrument.id,
+            name = instrument.name,
+            description = instrument.description,
+            wavelength_range = instrument.wavelength_range,
+            obstruction_area = instrument.obstruction_area,
+            overhead = instrument.overhead,
+            optics = get_optics(path, instrument.optics),
+            filters = get_filters(path, instrument.filters),
+            detector = get_detector(path, instrument.detector),
+            telescope = telescopes[instrument.telescope_id],
+            site = sites[instrument.site_id],
+            default = instrument.default
+        )
+    return instruments
+
+
+def get_optics(
+        parent_dir: str,
+        optics_config: OpticsConfigModel) -> OpticsModel:
+    path = join(parent_dir, optics_config.path)
+    transmissions = get_transmissions(path, optics_config.transmission)
+    # For emissions we may have to use transmission curves
+    emissions = get_emissions(
+        path,
+        optics_config.emission,
+        transmissions
+    )
+    return OpticsModel(
+        transmissions=transmissions,
+        emissions=emissions
+    )
+
+
+def get_sites(data_config: DataConfigModel) -> dict[str, SiteModel]:
+    sites = {}
+    for site in data_config.sites:
+        path = join(data_config.path, site.path)
+        # Instantiate the model
+        sites[site.id] = SiteModel(
+            id = site.id,
+            name = site.name,
+            description = site.description,
+            sky_transmissions = get_transmissions(path, site.transmission),
+            sky_emissions = get_emissions(path, site.emission),
+            default = site.default
+        )
+    return sites
+
+
+def get_telescopes(data_config: DataConfigModel) -> dict[str, TelescopeModel]:
+    telescopes = {}
+    for telescope in data_config.telescopes:
+        path = join(data_config.path, telescope.path)
+        # Instantiate the model
+        transmissions = get_transmissions(path, telescope.transmission)
+        emissions = get_emissions(path, telescope.emission, transmissions)
+        telescopes[telescope.id] = TelescopeModel(
+            id = telescope.id,
+            name = telescope.name,
+            description = telescope.description,
+            collecting_area = telescope.collecting_area,
+            obstruction_area = telescope.obstruction_area,
+            transmissions = transmissions,
+            emissions = emissions,
+            default = telescope.default
+        )
+    return telescopes
+
+
+def get_transmission(
+        file: IO | PathLike | str,
+        id: str,
+        name: str="",
+        description: str="",
+        vars: dict[str, float | str]={}
+    ) -> TransmissionModel:
+    data = get_data_file(file)
+    # Instantiate the model
+    wave = u.Quantity(data['WAVELENGTH'])
+    response = u.Quantity(data['THROUGHPUT'])
+    with warnings.catch_warnings():
+        warnings.filterwarnings(
+            "ignore",
+            message=".*negative flux or throughput.*",
+            category=AstropyUserWarning,
+        )
+        transmission = TransmissionModel(
+            id = id,
+            name = name,
+            description = description,
+            vars = vars,
+            # Apply tapering to filters to avoid possible spurious spectral leaks
+            spectral = SpectralElement.from_spectrum1d(
+                Spectrum(spectral_axis=wave, flux=response),
+                keep_neg=False
+            ).taper()
+        )
+    return transmission
+
+
+def get_transmissions(
+        parent_dir: str,
+        transmission_config: TransmissionConfigModel) -> dict[str, TransmissionModel]:
+    transmissions : dict[str, TransmissionModel] = {}
+    for file_config in transmission_config.files:
+        key = file_config.id if file_config.id != '' else str(len(transmissions))
+        transmissions[key] = get_transmission(
+            file=join(parent_dir, transmission_config.path, file_config.file),
+            id=file_config.id,
+            name=file_config.name,
+            description=file_config.description,
+            vars=file_config.vars
+        )
+    return transmissions
+
+
+def get_webapi_instruments(instruments: dict[str, InstrumentModel]) -> dict[str, InstrumentModel]:
+    winstruments = {}
+    for instrument in instruments:
+        winstruments[instrument] = instruments[instrument].copy(exclude={
+            'site': {'sky_emissions', 'sky_transmissions'},
+            'transmissions' : True,
+            'emissions_ct' : True
+        })
+    return winstruments
+

pydiet/server/image.py

@@ -0,0 +1,342 @@
+"""
+Image simulation module
+"""
+# Copyright CFHT
+# Licensed under the MIT licence
+
+from typing import Literal, Tuple
+
+from astropy import units as u  #type: ignore[import-untyped]
+from base64 import b64encode
+from io import BytesIO
+from PIL.Image import fromarray
+import numpy as np
+from scipy.optimize import brentq, minimize_scalar  #type: ignore[import-untyped]
+from synphot import Observation, SpectralElement  #type: ignore[import-untyped]
+
+from .models.types import PhotometryID, SourceID
+
+
+
+class Image(object):
+    """
+    Raster image generation class for computing exposure times and SNRs
+
+    Examples
+    --------
+    >>> from astropy import units as u
+
+    >>> img = Image(
+    ...     source='point_source',
+    ...     psf_fwhm=0.8 * u.arcsec,
+    ...     psf_beta=3.2,
+    ...     pixel=(0.186 * u.arcsec, 0.186 * u.arcsec),
+    ...     rate=42.,
+    ...     bkg_rate=10.,
+    ...     ron=4.,
+    ...     gain=1.65,
+    ...     photometry='model_fitting'
+    ... )
+
+   >>> # Compute SNR from exposure time
+   >>> print(f"{img.snr(etime=10.):.1f}")
+   4.6
+
+   >>> # Compute Exposure time from SNR
+   >>> print(f"{img.etime(snr=10.):.1f}")
+   43.0
+
+
+    Parameters
+    ----------
+    source: Literal['point_source', 'galaxy', 'extended'], optional
+        Source type.
+    psf_fwhm: ~astropy.units.Quantity['angle'], optional
+        Full Width at Half Maximum of the Point Spread Function.
+    psf_beta: float, optional
+        Moffat beta parameter of the Point Spread Function.
+    sersic_radius: ~astropy.units.Quantity['angle'], optional
+        Half-light radius for Sersic galaxy profiles.
+    sersic_index: float, optional
+        Sersic index for galaxy profiles.
+    pixel: ~astropy.units.Quantity['angle'], optional
+        Pixel scale on each axis.
+    image_size: Tuple[int, int], optional
+        Image size in pixels on each axis.
+    rate: float, optional
+        Number of photons per second.
+    bkg_rate: float, optional
+        Total background photon rate in photons per second per pixel.
+    ron: float, optional
+        Detector read out noise standard deviation in electrons.
+    gain: float, optional
+        Detector conversion factor in e-/ADU.
+    full_well: float, optional
+        Detector full well in electrons.
+    range: int, optional
+        Digital range, in analog-to-digital converter steps.
+    bias: float, optional
+        Detector bias in ADUs.
+    photometry: Literal['model_fitting', 'fixed_aperture', 'optimal_aperture', 'large_aperture']
+        Photometric measurement type.
+    aperture: float, optional
+        Aperture diameter in pixels for fixed aperture photometry.
+    oversamp: int, optional
+        Number of oversampling sub pixels on each axis.
+    max_etime: ~astropy.units.Quantity['time'], optional
+        Maximum possible exposure time in output
+    """
+    def __init__(
+            self,
+            source: SourceID='point_source',
+            psf_fwhm: u.Quantity['angle']=1.*u.arcsec, #type: ignore[name-defined]
+            psf_beta: float=3.2,
+            sersic_radius: u.Quantity['angle']=1.*u.arcsec, #type: ignore[name-defined]
+            sersic_index: float=1.,
+            pixel: u.Quantity['angle']=(0.2, 0.2)*u.arcsec, #type: ignore[name-defined]
+            image_size: Tuple[int, int]=(64, 64),
+            rate: float=1.,
+            bkg_rate: float=0.,
+            ron: float=0.,
+            gain: float=1.,
+            full_well: float=1e6,
+            range: int=65536,
+            bias: float=0.,
+            photometry: PhotometryID='model_fitting',
+            aperture: float=3.,
+            oversamp: int=1,
+            max_etime: u.Quantity['time'] = 1e9 * u.s) -> None:  #type: ignore[name-defined]
+
+        self.source = source
+        self.pixel = pixel
+        self.rate = rate
+        self.bkg_rate = bkg_rate
+        self.ron = ron
+        self.var_rate = rate
+        self.var_bkg_rate = bkg_rate
+        self.var_ron = ron*ron
+        self.gain = gain
+        self.oversamp = oversamp
+        self.photometry = photometry
+        self.saturation = min(range - 1. - bias, full_well / gain)
+        self.max_etime = max_etime.to(u.s).value
+
+        # Create image coordinate rasters
+        raster_size = [image_size[0] * oversamp, image_size[1] * oversamp]
+        yx = np.mgrid[
+            -raster_size[0]//2:raster_size[0] - raster_size[0]//2,
+            -raster_size[1]//2:raster_size[1] - raster_size[1]//2
+        ].astype(np.float32)
+        r2 = yx[0]**2 + yx[1]**2
+        self.r2 = r2
+
+        # Create truncation disk
+        self.mask_r2 = r2[0, raster_size[1]//2]
+        self.mask = r2 <= self.mask_r2
+
+        self.pixel_area = (pixel[0] * pixel[1]) / oversamp**2 * u.pix**2
+
+        if source == 'extended':
+            self.image = self.extended()
+            return
+
+        # Rasterize the PSF
+        if psf_beta <= 1.:
+            raise ValueError("Moffat beta must be > 1.")
+
+        # Compute the square of the alpha parameter from the FWHM
+        alpha2 = u.Quantity(psf_fwhm)**2 / (4. * (2.**(1./psf_beta) - 1.)) \
+            / self.pixel_area
+
+        # Create PSF raster
+        moffat = np.power(1. + r2 / alpha2, -psf_beta) 
+
+        # Truncate inside a disk
+        moffat *= self.mask
+        self.psf = moffat / moffat.sum()
+
+        # Generate star or galaxy image
+        self.image = self.sersic(
+            re=sersic_radius,
+            n=sersic_index
+        ) if source == 'galaxy' else self.psf
+
+        # Create photometry measurement aperture
+        if self.photometry != 'model_fitting':
+            if self.photometry == 'fixed_aperture':
+                # User-provided aperture diameter
+                r2max = aperture**2 * u.arcsec**2 / self.pixel_area
+            elif self.photometry == 'large_aperture':
+                # Aperture enclosing 96% of the flux
+                r2max = brentq(
+                    f = lambda r2: np.sum((self.r2 <= r2) * self.image) - 0.96,
+                    a=0.,
+                    b=self.mask_r2,
+                    xtol=1e-3,
+                    maxiter=100
+                ) 
+            elif self.photometry == 'optimal_aperture':
+                r2max = 0.
+            self.aperture = r2 < r2max
+
+
+    def delta_snr2(self, etime: float, snr: float) -> float:
+        return self.snr(etime=etime)**2 - snr**2
+
+
+    def etime(self, snr: float) -> float:
+        return brentq(
+            f=self.delta_snr2,
+            a=0.,
+            b=self.etime_max(snr),
+            args=(snr),
+            xtol=1e-6,
+            maxiter=100
+        ) if self.rate > 0. else self.max_etime
+
+
+    def etime_bkg_sat(self) -> float:
+        return self.saturation / self.bkg_rate if self.bkg_rate > 0. else self.max_etime
+
+
+    def etime_max(self, snr:float) -> float:
+        # Find exposure time range for root finding
+        t_high = 1.
+        while (tsnr:=self.snr(t_high)) < snr and tsnr < 1.e12:
+            t_high *= 2.
+        return t_high
+
+
+    def etime_source_sat(self) -> float:
+        return self.saturation / self.max() if self.rate > 0. and self.bkg_rate > 0. \
+            else self.max_etime
+
+
+    def extended(self) -> np.ndarray:
+        return self.mask * self.pixel_area.to(u.arcsec**2).value
+
+
+    def gif(self, etime: float, exposures: int=1, frames: int=10) -> str:
+        # Initialize random generator
+        rng = np.random.default_rng()
+        # Use the PSF as a template image and generate a noiseless image
+        noiseless = (self.rate * np.array([self.image] * frames) + self.bkg_rate) * etime
+        # Generate Poisson + Gaussian noise realizations
+        # We add a 3 sigma offset above the background to prevent negative values
+        sigmas = 3.*(self.ron*self.ron + self.bkg_rate * etime)**0.5 \
+          / np.sqrt(exposures)
+        offset = sigmas - self.bkg_rate * etime
+        nmax = (noiseless.max() + offset + sigmas)  / self.gain
+        noisy = np.round(
+            exposures * (
+                rng.poisson(lam=noiseless*exposures) / exposures + rng.normal(
+                    loc=offset,
+                    scale=self.ron / np.sqrt(exposures),
+                    size=noiseless.shape
+                )
+            ) / self.gain
+        ) / exposures
+        # Normalize to a max of 1
+        noisy[noisy < 0.] = 0.
+        noisy /= nmax
+        noisy[noisy > 1.] = 1.
+        # Apply sRGB gamma correction and convert to 0...255 unsigned integers
+        noisy = (
+            np.where(
+                noisy <= 0.0031308,
+                12.92 * noisy,
+                1.055 * np.power(noisy, 1./2.4) - 0.055
+            ) * 255.
+        ).astype(np.uint8)
+        
+        # Create image buffer
+        buffer = BytesIO()
+        # Save GIF to buffer and append the rest of the animation
+        fromarray(noisy[0], mode="L").save(
+            buffer,
+            format='GIF',
+            save_all=True,
+            append_images=[fromarray(im) for im in noisy[1:]],
+            duration=100,
+            loop=0
+        )
+        buffer.seek(0)
+        # Encode GIF as base64
+        gif_base64 = b64encode(buffer.read()).decode("utf-8")
+        return f"data:image/gif;base64,{gif_base64}"
+
+
+    def max(self) -> float:
+        return self.rate * self.image.max() + self.bkg_rate
+
+
+    def sersic(
+            self,
+            re: u.Quantity['angle']=1.*u.arcsec,  #type: ignore[name-defined]
+            n: float=1.) -> np.ndarray:
+        # Model validity limits
+        if n < 0.36:
+            n = 0.36
+        # R_e normalization from Ciotti & Bertin 1999
+        bn = 2.* n - 0.333333 + 9.8765e-3*n**(-1) + 1.8029e-3 * n**(-2) \
+            + 1.1409e-4 * n**(-3) - 7.151e-5 * n**(-4)
+        inv2n = 0.5 / n
+        invre2 = (self.pixel_area / re**2).value
+        sersic = np.exp(-bn * (np.power(self.r2 * invre2, inv2n) - 1.))
+        sersic = np.fft.irfft2(
+            np.fft.rfft2(sersic) * np.fft.rfft2(np.fft.fftshift(self.psf))
+        ) * self.mask
+        return sersic / np.sum(sersic)
+
+
+    def snr(self, etime: float=1.) -> float:
+        # First treat special case of extended source
+        if self.source == 'extended':
+            # Compute pixel area in arcsec2
+            invarea = 1. / self.pixel_area.to(u.arcsec**2).value
+            return self.rate * etime / np.sqrt(
+                (self.var_bkg_rate * invarea + self.var_rate) * etime \
+                + self.var_ron * invarea
+            )
+        # Compute the "noise variance image"
+        img2 = self.image**2
+        var_tot = self.var_ron + (
+            self.var_bkg_rate + self.var_rate * self.image
+        ) * etime
+        if self.photometry == 'optimal_aperture':
+            # (Re-)compute optimal aperture
+            res = minimize_scalar(
+                fun = lambda r2: - self.snr_aper(
+                    self.rate* etime,
+                    self.image,
+                    var_tot,
+                    self.r2 < r2,
+                ),
+                bounds=(0., self.mask_r2),
+                method='bounded'
+            )
+            # Return SNR at optimal aperture
+            return -res.fun
+        elif self.photometry == 'model_fitting':
+            # Return model-fitting SNR
+            return self.rate * etime * np.sqrt(
+                np.sum(img2 / var_tot + img2 / (2. * var_tot**2))
+            )
+        else:
+            # Return SNR for a predefined aperture
+            return self.snr_aper(
+                self.rate * etime,
+                self.image,
+                var_tot,
+                self.aperture
+            )
+
+    def snr_aper(
+            self,
+            photons: float,
+            obj: np.ndarray,
+            var: np.ndarray,
+            aper: np.ndarray) -> float:
+        return photons * np.sum(obj * aper) / np.sqrt(np.sum(var * aper))
+
+