pyobs-core 2.0.0.dev67__tar.gz → 2.0.0.dev70__tar.gz

{pyobs_core-2.0.0.dev67 → pyobs_core-2.0.0.dev70}/.gitignore

@@ -160,3 +160,9 @@ cython_debug/
 #  and can be added to the global gitignore or merged into this file.  For a more nuclear
 #  option (not recommended) you can uncomment the following to ignore the entire idea folder.
 .idea/
+
+# graphify local build cache and machine-specific state (outputs are tracked)
+graphify-out/cache/
+graphify-out/[0-9][0-9][0-9][0-9]-[0-9][0-9]-[0-9][0-9]/
+graphify-out/.graphify_python
+graphify-out/.graphify_root

{pyobs_core-2.0.0.dev67 → pyobs_core-2.0.0.dev70}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: pyobs-core
-Version: 2.0.0.dev67
+Version: 2.0.0.dev70
 Summary: robotic telescope software
 Author-email: Tim-Oliver Husser <thusser@uni-goettingen.de>
 License-Expression: MIT

{pyobs_core-2.0.0.dev67 → pyobs_core-2.0.0.dev70}/pyobs/images/processors/calibration/_ccddata_calibrator.py

@@ -8,6 +8,11 @@ from pyobs.images import Image
 
 class _CCDDataCalibrator:
     def __init__(self, image: Image, bias: Image | None = None, dark: Image | None = None, flat: Image | None = None):
+        # to_ccddata() below never carries a catalog through anyway, so drop it before trim() to
+        # avoid its stale-catalog guard rejecting an image whose catalog wouldn't survive regardless.
+        if image.safe_catalog is not None:
+            image = image.copy()
+            image.catalog = None
         self._image = image.trim()
         self._bias = self._optional_to_ccddata(bias)
         self._dark = self._optional_to_ccddata(dark)

{pyobs_core-2.0.0.dev67 → pyobs_core-2.0.0.dev70}/pyobs/images/processors/photometry/aperture_photometry.py

@@ -1,5 +1,6 @@
 import asyncio
 import logging
+from abc import abstractmethod
 from typing import Any
 
 from pyobs.images import Image
@@ -12,10 +13,12 @@ log = logging.getLogger(__name__)
 
 
 class AperturePhotometry(Photometry):
-    __module__ = "pyobs.images.processors.photometry"
+    """Base class for aperture photometry processors -- not meant to be used directly,
+    see PhotUtilsPhotometry/SepPhotometry for concrete calculators."""
 
     APERTURE_RADII = range(1, 9)
 
+    @abstractmethod
     def __init__(self, calculator: _PhotometryCalculator, **kwargs: Any) -> None:
         self._calculator = calculator
         super().__init__(**kwargs)

{pyobs_core-2.0.0.dev67 → pyobs_core-2.0.0.dev70}/pyobs/mixins/pipeline.py

@@ -1,12 +1,15 @@
 from __future__ import annotations
 
 import logging
-from typing import Any
+from typing import TYPE_CHECKING, Any
 
 from pyobs.images import Image, ImageProcessor
 from pyobs.object import Object
 from pyobs.utils.exceptions import ImageError
 
+if TYPE_CHECKING:
+    from pyobs.robotic.utils.archive import Archive
+
 log = logging.getLogger(__name__)
 
 
@@ -15,21 +18,42 @@ class PipelineMixin:
 
     __module__ = "pyobs.mixins"
 
-    def __init__(self, steps: list[dict[str, Any] | ImageProcessor] | None = None, **kwargs: Any):
+    def __init__(
+        self,
+        steps: list[dict[str, Any] | ImageProcessor] | None = None,
+        archive: dict[str, Any] | Archive | None = None,
+        **kwargs: Any,
+    ):
         """Initializes the mixin.
 
         Args:
             steps: Pipeline steps to run on images.
+            archive: Default archive config/object for steps that accept one (e.g.
+                Calibration) and don't already specify their own -- lets a pipeline's
+                steps share the archive it was itself given, instead of repeating the
+                same archive config in every step that needs one. Steps that don't
+                declare an archive parameter just absorb and drop it via their own
+                **kwargs, the same as any other unrecognized Object kwarg.
         """
 
         # store
         if isinstance(self, Object):
             steps = [] if steps is None else steps
-            self.__pipeline_steps = [self.add_child_object(step, ImageProcessor) for step in steps]
+            self.__pipeline_steps = [
+                self.add_child_object(self._with_default_archive(step, archive), ImageProcessor) for step in steps
+            ]
 
         else:
             raise ValueError("This class is no Object.")
 
+    @staticmethod
+    def _with_default_archive(
+        step: dict[str, Any] | ImageProcessor, archive: dict[str, Any] | Archive | None
+    ) -> dict[str, Any] | ImageProcessor:
+        if archive is not None and isinstance(step, dict) and "archive" not in step:
+            return {**step, "archive": archive}
+        return step
+
     async def reset_pipeline(self) -> None:
         """Resets all previous state of the involved image processors."""
         for step in self.__pipeline_steps:

{pyobs_core-2.0.0.dev67 → pyobs_core-2.0.0.dev70}/pyobs/robotic/utils/archive/local_archive.py

@@ -166,7 +166,13 @@ class LocalArchive(Archive):
         return headers
 
     async def upload_frames(self, images: list[Image]) -> None:
-        pass
+        self._root_path.mkdir(parents=True, exist_ok=True)
+        for image in images:
+            filename = image.header.get("FNAME")
+            if not filename:
+                raise ValueError("Image has no FNAME header set, cannot determine output filename.")
+            image.writeto(str(self._root_path / filename), overwrite=True)
+        self._update_root()  # refresh the in-memory index so newly-written frames are queryable
 
 
 __all__ = ["LocalArchive"]

pyobs_core-2.0.0.dev70/pyobs/utils/pipeline/__init__.py

@@ -0,0 +1,14 @@
+from .pipeline import Pipeline
+from .progress import MasterCalibCreated, ProgressCallback, ProgressEvent, ScienceFrameProcessed
+from .reduction import Reduction
+from .reduction_base import ReductionBase
+
+__all__ = [
+    "Reduction",
+    "ReductionBase",
+    "Pipeline",
+    "MasterCalibCreated",
+    "ScienceFrameProcessed",
+    "ProgressEvent",
+    "ProgressCallback",
+]

{pyobs_core-2.0.0.dev67 → pyobs_core-2.0.0.dev70}/pyobs/utils/pipeline/pipeline.py

@@ -22,14 +22,22 @@ log = logging.getLogger(__name__)
 class Pipeline(Object, PipelineMixin):
     """Pipeline based on the astropy package ccdproc."""
 
-    def __init__(self, steps: list[dict[str, Any] | ImageProcessor], **kwargs: Any):
+    def __init__(
+        self,
+        steps: list[dict[str, Any] | ImageProcessor],
+        archive: dict[str, Any] | Archive | None = None,
+        **kwargs: Any,
+    ):
         """Pipeline for science images.
 
         Args:
             steps: List of pipeline steps to perform.
+            archive: Default archive config/object for steps that accept one (e.g.
+                Calibration) and don't already specify their own. See
+                PipelineMixin.__init__.
         """
         Object.__init__(self, **kwargs)
-        PipelineMixin.__init__(self, steps)
+        PipelineMixin.__init__(self, steps, archive=archive)
 
     @staticmethod
     def _combine_calib_images(

pyobs_core-2.0.0.dev70/pyobs/utils/pipeline/progress.py

@@ -0,0 +1,40 @@
+from __future__ import annotations
+
+from collections.abc import Callable
+from dataclasses import dataclass
+from typing import Literal
+
+from pyobs.utils.enums import ImageType
+
+
+@dataclass
+class MasterCalibCreated:
+    """A master calibration frame (BIAS/DARK/SKYFLAT) was created and stored/uploaded."""
+
+    image_type: ImageType
+    instrument: str
+    binning: str
+    filter_name: str | None
+    filename: str
+
+
+@dataclass
+class ScienceFrameProcessed:
+    """One science frame finished processing (successfully or not).
+
+    index/total are cumulative across the whole reduction run, not just the current
+    instrument/binning/filter batch.
+    """
+
+    index: int
+    total: int
+    filename: str | None
+    status: Literal["ok", "error"]
+    error: str | None = None
+
+
+ProgressEvent = MasterCalibCreated | ScienceFrameProcessed
+ProgressCallback = Callable[[ProgressEvent], None]
+
+
+__all__ = ["MasterCalibCreated", "ScienceFrameProcessed", "ProgressEvent", "ProgressCallback"]

pyobs_core-2.0.0.dev67/pyobs/utils/pipeline/night.py → pyobs_core-2.0.0.dev70/pyobs/utils/pipeline/reduction.py

@@ -1,17 +1,19 @@
 from __future__ import annotations
 
 import logging
+import os
 import os.path
 from typing import Any
 
 from pyobs.images import Image
 from pyobs.object import get_object
-from pyobs.robotic.utils.archive import Archive
+from pyobs.robotic.utils.archive import Archive, FrameInfo
 from pyobs.utils.enums import ImageType
 from pyobs.utils.fits import FilenameFormatter
-from pyobs.utils.time import Time
 
 from .pipeline import Pipeline
+from .progress import MasterCalibCreated, ProgressCallback, ScienceFrameProcessed
+from .reduction_base import ReductionBase
 
 log = logging.getLogger(__name__)
 
@@ -19,87 +21,53 @@ log = logging.getLogger(__name__)
 FILENAME = "{SITEID}{TELID}-{INSTRUME}-{DAY-OBS|date:}-{IMAGETYP}-{XBINNING}x{YBINNING}{FILTER|filter}.fits"
 
 
-class Night:
+class Reduction(ReductionBase):
     def __init__(
         self,
         archive: dict[str, Any] | Archive,
         pipeline: dict[str, Any] | Pipeline,
-        worker_procs: int = 4,
         filenames_calib: str = FILENAME,
         min_flats: int = 10,
-        store_local: str | None = None,
+        output: str | dict[str, Any] | Archive | None = None,
         create_calibs: bool = True,
         calib_science: bool = True,
-        **kwargs: Any,
+        progress_callback: ProgressCallback | None = None,
     ):
-        """Creates a Night object for reducing a given night.
+        """Creates a Reduction object for reducing a given observation period.
 
         Args:
-            archive: Archive to fetch images from and write results to.
+            archive: Archive to fetch raw and calibration frames from. Also the output
+                destination, unless output is set.
             pipeline: Science pipeline.
-            worker_procs: Number of worker processes.
             filenames_calib: Filename pattern for master calibration files.
             min_flats: Minimum number of raw frames to create flat field.
-            store_local: If True, files are stored in given local directory instead of uploaded to archive.
+            output: Where to write results. A string is a local directory path; a dict or
+                Archive is an output archive. If not set, results are written back to archive.
             create_calibs: If False, no calibration files are created for night.
             calib_science: If False, no science frames are calibrated.
+            progress_callback: See ReductionBase.
         """
+        super().__init__(archive=archive, pipeline=pipeline, min_flats=min_flats, progress_callback=progress_callback)
 
-        # get archive and science pipeline
-        self._archive = get_object(archive, Archive)
-        self._pipeline = get_object(pipeline, Pipeline, archive=archive)
-
-        # stuff
-        self._worker_processes = worker_procs
-        self._min_flats = min_flats
-        self._store_local = store_local
+        self._store_local: str | None = output if isinstance(output, str) else None
+        self._output_archive = (
+            self._archive if output is None or isinstance(output, str) else get_object(output, Archive)
+        )
         self._create_calibs = create_calibs
         self._calib_science = calib_science
 
-        # cache for master calibration frames
-        self._master_frames: dict[tuple[ImageType, str, str, str | None], Image] = {}
+        # make sure the local output directory exists
+        if self._store_local:
+            os.makedirs(self._store_local, exist_ok=True)
 
         # default filename patterns
         self._fmt_calib = FilenameFormatter(filenames_calib)
 
-    async def _find_master(
-        self,
-        night: str,
-        image_type: ImageType,
-        instrument: str,
-        binning: str,
-        filter_name: str | None = None,
-        max_days: float = 30.0,
-    ) -> Image | None:
-        """Find master calibration frame for given parameters using a cache.
-
-        Args:
-            image_type: image type.
-            instrument: Instrument name.
-            binning: Binning.
-            filter_name: Name of filter.
-            max_days: Maximum number of days from DATE-OBS to find frames.
-
-        Returns:
-            Image or None
-        """
-
-        # is in cache?
-        if (image_type, instrument, binning, filter_name) in self._master_frames:
-            return self._master_frames[image_type, instrument, binning, filter_name]
-
-        # try to download one
-        midnight = Time(night + " 23:59:59")
-        image = await self._pipeline.find_master(
-            self._archive, image_type, midnight, instrument, binning, filter_name, max_days=max_days
-        )
-        if image is not None:
-            # store and return it
-            self._master_frames[image_type, instrument, binning, filter_name] = image
-            return image
-        else:
-            # still nothing
-            return None
+        # populated by _count_science_frames() at the start of __call__, consumed by _calib_data()
+        # so science frames are only listed from the archive once per combination, not twice
+        self._science_frame_cache: dict[tuple[str, str, str], list[FrameInfo]] = {}
+        self._frames_done = 0
+        self._frames_total = 0
 
     async def _create_master_calib(
         self, night: str, instrument: str, image_type: ImageType, binning: str, filter_name: str | None = None
@@ -128,6 +96,7 @@ class Night:
         images = await self._archive.download_frames(infos)
         if len(images) < 3:
             log.warning("Too few (%d) frames found, skipping...", len(infos))
+            return None
 
         # create master
         calib: Image | None = None
@@ -191,21 +160,43 @@ class Night:
             calib.writeto(path, overwrite=True)
         else:
             log.info("Uploading master calibration frame as %s...", calib.header["FNAME"])
-            await self._archive.upload_frames([calib])
+            await self._output_archive.upload_frames([calib])
+
+        self._report_progress(
+            MasterCalibCreated(
+                image_type=image_type,
+                instrument=instrument,
+                binning=binning,
+                filter_name=filter_name,
+                filename=calib.header["FNAME"],
+            )
+        )
 
         # finished
         return calib
 
+    async def _count_science_frames(self, night: str, options: dict[str, list[Any]]) -> int:
+        """Pre-pass: list (not download) OBJECT frames for every instrument/binning/filter
+        combination once, caching the results so _calib_data() doesn't have to list them
+        again, and return the total count across the whole night for progress reporting."""
+        total = 0
+        for instrument in options["instruments"]:
+            for binning in options["binnings"]:
+                for filter_name in options["filters"]:
+                    infos = await self._archive.list_frames(
+                        night=night,
+                        instrument=instrument,
+                        image_type=ImageType.OBJECT,
+                        binning=binning,
+                        filter_name=filter_name,
+                        rlevel=0,
+                    )
+                    self._science_frame_cache[instrument, binning, filter_name] = infos
+                    total += len(infos)
+        return total
+
     async def _calib_data(self, night: str, instrument: str, binning: str, filter_name: str) -> None:
-        # get all frames
-        infos = await self._archive.list_frames(
-            night=night,
-            instrument=instrument,
-            image_type=ImageType.OBJECT,
-            binning=binning,
-            filter_name=filter_name,
-            rlevel=0,
-        )
+        infos = self._science_frame_cache.get((instrument, binning, filter_name), [])
         total = len(infos)
         if total == 0:
             return
@@ -230,10 +221,27 @@ class Night:
                     calibrated.writeto(path, overwrite=True)
                 else:
                     log.info("(%d/%d) Uploading calibrated images as %s...", i, total, calibrated.header["FNAME"])
-                    await self._archive.upload_frames([calibrated])
+                    await self._output_archive.upload_frames([calibrated])
+
+                self._frames_done += 1
+                self._report_progress(
+                    ScienceFrameProcessed(
+                        index=self._frames_done, total=self._frames_total, filename=info.filename, status="ok"
+                    )
+                )
 
-            except Exception:
+            except Exception as e:
                 log.exception("(%d/%d) Error processing image %s.", i, total, info.filename)
+                self._frames_done += 1
+                self._report_progress(
+                    ScienceFrameProcessed(
+                        index=self._frames_done,
+                        total=self._frames_total,
+                        filename=info.filename,
+                        status="error",
+                        error=str(e),
+                    )
+                )
 
     async def __call__(self, site: str, night: str) -> None:
         """Reduces all data im this night."""
@@ -248,6 +256,11 @@ class Night:
             len(options["filters"]),
         )
 
+        # pre-pass: know the total science-frame count up front for progress reporting, and
+        # cache the per-combination frame lists so _calib_data() doesn't list them again
+        if self._calib_science:
+            self._frames_total = await self._count_science_frames(night, options)
+
         # loop instruments
         for instrument in options["instruments"]:
             log.info("Reducing data for instrument %s...", instrument)
@@ -256,18 +269,32 @@ class Night:
             for binning in options["binnings"]:
                 # create bias and dark
                 if self._create_calibs:
-                    await self._create_master_calib(night, instrument, ImageType.BIAS, binning)
-                    await self._create_master_calib(night, instrument, ImageType.DARK, binning)
+                    try:
+                        await self._create_master_calib(night, instrument, ImageType.BIAS, binning)
+                    except Exception:
+                        log.exception("Error creating master bias for instrument %s, binning %s.", instrument, binning)
+                    try:
+                        await self._create_master_calib(night, instrument, ImageType.DARK, binning)
+                    except Exception:
+                        log.exception("Error creating master dark for instrument %s, binning %s.", instrument, binning)
 
                 # loop filters
                 for filter_name in options["filters"]:
                     # create flat
                     if self._create_calibs:
-                        await self._create_master_calib(night, instrument, ImageType.SKYFLAT, binning, filter_name)
+                        try:
+                            await self._create_master_calib(night, instrument, ImageType.SKYFLAT, binning, filter_name)
+                        except Exception:
+                            log.exception(
+                                "Error creating master flat for instrument %s, binning %s, filter %s.",
+                                instrument,
+                                binning,
+                                filter_name,
+                            )
 
                     # calibrate science data
                     if self._calib_science:
                         await self._calib_data(night, instrument, binning, filter_name)
 
 
-__all__ = ["Night"]
+__all__ = ["Reduction"]

pyobs_core-2.0.0.dev70/pyobs/utils/pipeline/reduction_base.py

@@ -0,0 +1,104 @@
+from __future__ import annotations
+
+import logging
+from abc import ABC, abstractmethod
+from typing import Any
+
+from pyobs.images import Image
+from pyobs.object import get_object
+from pyobs.robotic.utils.archive import Archive
+from pyobs.utils.enums import ImageType
+from pyobs.utils.time import Time
+
+from .pipeline import Pipeline
+from .progress import ProgressCallback, ProgressEvent
+
+log = logging.getLogger(__name__)
+
+
+class ReductionBase(ABC):
+    """Common base for classes that reduce one observation period's worth of data.
+
+    Holds the archive/pipeline plumbing, the master-calibration-frame cache and lookup
+    (shared by any reduction strategy that needs BIAS/DARK/SKYFLAT masters), and the
+    progress-callback mechanism. Concrete subclasses implement __call__.
+    """
+
+    def __init__(
+        self,
+        archive: dict[str, Any] | Archive,
+        pipeline: dict[str, Any] | Pipeline,
+        min_flats: int = 10,
+        progress_callback: ProgressCallback | None = None,
+    ):
+        """
+        Args:
+            archive: Archive to fetch raw and calibration frames from.
+            pipeline: Science pipeline.
+            min_flats: Minimum number of raw frames to create flat field.
+            progress_callback: Optional callable invoked with a ProgressEvent whenever a
+                master calibration frame is created or a science frame finishes processing.
+                Exceptions raised by the callback are logged and otherwise ignored.
+        """
+        self._archive = get_object(archive, Archive)
+        self._pipeline = get_object(pipeline, Pipeline, archive=archive)
+        self._min_flats = min_flats
+        self._progress_callback = progress_callback
+
+        # cache for master calibration frames
+        self._master_frames: dict[tuple[ImageType, str, str, str | None], Image] = {}
+
+    def _report_progress(self, event: ProgressEvent) -> None:
+        if self._progress_callback is None:
+            return
+        try:
+            self._progress_callback(event)
+        except Exception:
+            log.exception("Error in progress callback.")
+
+    async def _find_master(
+        self,
+        night: str,
+        image_type: ImageType,
+        instrument: str,
+        binning: str,
+        filter_name: str | None = None,
+        max_days: float = 30.0,
+    ) -> Image | None:
+        """Find master calibration frame for given parameters using a cache.
+
+        Args:
+            image_type: image type.
+            instrument: Instrument name.
+            binning: Binning.
+            filter_name: Name of filter.
+            max_days: Maximum number of days from DATE-OBS to find frames.
+
+        Returns:
+            Image or None
+        """
+
+        # is in cache?
+        if (image_type, instrument, binning, filter_name) in self._master_frames:
+            return self._master_frames[image_type, instrument, binning, filter_name]
+
+        # try to download one
+        midnight = Time(night + " 23:59:59")
+        image = await self._pipeline.find_master(
+            self._archive, image_type, midnight, instrument, binning, filter_name, max_days=max_days
+        )
+        if image is not None:
+            # store and return it
+            self._master_frames[image_type, instrument, binning, filter_name] = image
+            return image
+        else:
+            # still nothing
+            return None
+
+    @abstractmethod
+    async def __call__(self, site: str, night: str) -> None:
+        """Reduces all data for the given site and observation period."""
+        ...
+
+
+__all__ = ["ReductionBase"]

{pyobs_core-2.0.0.dev67 → pyobs_core-2.0.0.dev70}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "pyobs-core"
-version = "2.0.0.dev67"
+version = "2.0.0.dev70"
 description = "robotic telescope software"
 authors = [{ name = "Tim-Oliver Husser", email = "thusser@uni-goettingen.de" }]
 requires-python = ">=3.11"

pyobs_core-2.0.0.dev67/pyobs/utils/pipeline/__init__.py

@@ -1,4 +0,0 @@
-from .night import Night
-from .pipeline import Pipeline
-
-__all__ = ["Night", "Pipeline"]