timewise 1.0.0a6__py3-none-any.whl → 1.0.0a8__py3-none-any.whl

ampel/timewise/alert/TimewiseAlertSupplier.py

@@ -16,11 +16,12 @@ import pandas as pd
 from bson import encode
 
 from ampel.alert.AmpelAlert import AmpelAlert
+from ampel.base.AmpelABC import AmpelABC
 from ampel.alert.BaseAlertSupplier import BaseAlertSupplier
 from ampel.view.ReadOnlyDict import ReadOnlyDict
 
 
-class TimewiseAlertSupplier(BaseAlertSupplier):
+class TimewiseAlertSupplier(BaseAlertSupplier, AmpelABC):
     """
     Iterable class that, for each transient name provided by the underlying alert_loader
     returns a PhotoAlert instance.

ampel/timewise/alert/load/TimewiseFileLoader.py

@@ -10,14 +10,16 @@ from typing import Dict, get_args
 
 import numpy as np
 import pandas as pd
-from astropy.table import Table, vstack
+from astropy.table import vstack
 from ampel.abstract.AbsAlertLoader import AbsAlertLoader
+from ampel.base.AmpelABC import AmpelABC
 from timewise.tables import TableType
 from timewise.config import TimewiseConfig
 from timewise.types import TaskID
+from timewise.util.path import expand
 
 
-class TimewiseFileLoader(AbsAlertLoader[Dict]):
+class TimewiseFileLoader(AbsAlertLoader[Dict], AmpelABC):
     """
     Load alerts from one of more files.
     """
@@ -33,8 +35,10 @@ class TimewiseFileLoader(AbsAlertLoader[Dict]):
     def __init__(self, **kwargs) -> None:
         super().__init__(**kwargs)
 
-        self.logger.info(f"loading timewise config file {self.timewise_config_file}")
-        timewise_config = TimewiseConfig.from_yaml(self.timewise_config_file)
+        expanded_config_file = expand(self.timewise_config_file)
+
+        self.logger.info(f"loading timewise config file {expanded_config_file}")
+        timewise_config = TimewiseConfig.from_yaml(expanded_config_file)
         dl = timewise_config.download.build_downloader()
         self._timewise_backend = dl.backend
 

ampel/timewise/ingest/TiCompilerOptions.py

@@ -9,9 +9,10 @@
 from typing import Any
 
 from ampel.model.ingest.CompilerOptions import CompilerOptions
+from ampel.base.AmpelABC import AmpelABC
 
 
-class TiCompilerOptions(CompilerOptions):
+class TiCompilerOptions(CompilerOptions, AmpelABC):
     stock: dict[str, Any] = {"tag": "TIMEWISE"}
     t0: dict[str, Any] = {"tag": "TIMEWISE"}
     t1: dict[str, Any] = {"tag": "TIMEWISE"}

ampel/timewise/ingest/TiMongoMuxer.py

@@ -32,7 +32,7 @@ class TiMongoMuxer(AbsT0Muxer):
     """
 
     # Standard projection used when checking DB for existing PPS/ULS
-    projection = {
+    projection: dict[str, int] = {
         "_id": 0,
         "id": 1,
         "tag": 1,
@@ -51,6 +51,8 @@ class TiMongoMuxer(AbsT0Muxer):
         "body.dec": 1,
     }
 
+    unique_key: list[str] = ["mjd", "ra", "dec"]
+
     def __init__(self, **kwargs) -> None:
         super().__init__(**kwargs)
 
@@ -116,11 +118,7 @@ class TiMongoMuxer(AbsT0Muxer):
             # jd alone is not enough for matching pps because each time is associated with
             # two filters! Also, if there can be multiple sources within the same frame which
             # leads to duplicate MJD and FID. Check position in addition.
-            key = (
-                dp["body"]["mjd"],
-                dp["body"]["ra"],
-                dp["body"]["dec"],
-            )
+            key = tuple(dp["body"][k] for k in self.unique_key)
 
             if target := unique_dps_ids.get(key):
                 # insert id in order

ampel/timewise/t1/T1HDBSCAN.py

@@ -10,8 +10,6 @@ from typing import Iterable, Sequence
 
 import numpy as np
 from numpy import typing as npt
-from ampel.base.AuxUnitRegister import AuxUnitRegister
-from astropy.coordinates.angle_utilities import angular_separation, position_angle
 from sklearn.cluster import HDBSCAN
 from pymongo import MongoClient
 
@@ -20,11 +18,21 @@ from ampel.struct.T1CombineResult import T1CombineResult
 from ampel.types import DataPointId
 from ampel.abstract.AbsT1CombineUnit import AbsT1CombineUnit
 from ampel.model.UnitModel import UnitModel
+from ampel.base.AuxUnitRegister import AuxUnitRegister
 
 from ampel.timewise.util.pdutil import datapoints_to_dataframe
 from ampel.timewise.util.AuxDiagnosticPlotter import AuxDiagnosticPlotter
 from timewise.process import keys
 
+from importlib.util import find_spec
+
+if find_spec("astropy.coordinates.angle_utilities"):
+    # astropy < v6.0.0
+    from astropy.coordinates.angle_utilities import angular_separation, position_angle
+else:
+    # astropy >= v6.0.0
+    from astropy.coordinates.angles import angular_separation, position_angle
+
 
 class T1HDBSCAN(AbsT1CombineUnit):
     input_mongo_db_name: str

ampel/timewise/t1/TimewiseFilter.py

@@ -36,12 +36,12 @@ class TimewiseFilter(AbsAlertFilter):
         visits, counts = np.unique(visit_map, return_counts=True)
         visit_passed = counts >= self.det_per_visit
         if not all(visit_passed):
-            self.logger.info(None, extra={"min_det_per_visit": min(counts).item()})
+            self.logger.debug(None, extra={"min_det_per_visit": min(counts).item()})
             return None
 
         # enough visits
         if not len(visits) >= self.n_visits:
-            self.logger.info(None, extra={"n_visits": len(visits)})
+            self.logger.debug(None, extra={"n_visits": len(visits)})
             return None
 
         return True

ampel/timewise/t2/T2StackVisits.py

@@ -6,6 +6,7 @@
 # Date:                24.09.2025
 # Last Modified Date:  24.09.2025
 # Last Modified By:    Jannis Necker <jannis.necker@gmail.com>
+from typing import Literal
 from scipy import stats
 
 from ampel.abstract.AbsLightCurveT2Unit import AbsLightCurveT2Unit
@@ -32,6 +33,16 @@ class T2StackVisits(AbsLightCurveT2Unit):
     # threshold above which to exclude outliers
     outlier_threshold: float = 5
 
+    # methods to calculate mean and std
+    mean_name: Literal["mean", "median"] = "median"
+    std_name: Literal["std", "sdom", "sdom-1"] = "sdom-1"
+    correction_name: Literal["debias", "tdist", "none"] = "tdist"
+
+    # see timewise.process.stacking
+    calculate_pvalues: bool = False
+    use_single_exposure_errors: bool = True
+    median_zeropoint_per_visit: bool = True
+
     def process(self, light_curve: LightCurve) -> UBson | UnitResult:
         columns = [
             "ra",
@@ -53,4 +64,10 @@ class T2StackVisits(AbsLightCurveT2Unit):
             outlier_threshold=self.outlier_threshold,
             outlier_quantile=self.outlier_quantile,
             clean_outliers=self.clean_outliers,
+            mean_name=self.mean_name,
+            std_name=self.std_name,
+            correction_name=self.correction_name,
+            calculate_pvalues=self.calculate_pvalues,
+            use_single_exposure_errors=self.use_single_exposure_errors,
+            median_zeropoint_per_visit=self.median_zeropoint_per_visit,
         ).to_dict(orient="records")

ampel/timewise/util/AuxDiagnosticPlotter.py

@@ -5,13 +5,14 @@ from numpy import typing as npt
 
 from ampel.plot.create import create_plot_record
 from ampel.base.AmpelBaseModel import AmpelBaseModel
+from ampel.base.AmpelABC import AmpelABC
 from ampel.model.PlotProperties import PlotProperties
 from ampel.content.NewSVGRecord import NewSVGRecord
 
 from timewise.plot.diagnostic import DiagnosticPlotter
 
 
-class AuxDiagnosticPlotter(AmpelBaseModel):
+class AuxDiagnosticPlotter(AmpelBaseModel, AmpelABC):
     plot_properties: PlotProperties
     cutout: Literal["sdss", "panstarrs"] = DiagnosticPlotter.model_fields[
         "cutout"

timewise/__init__.py

@@ -1 +1 @@
-__version__ = "1.0.0a6"
+__version__ = "1.0.0a8"

timewise/backend/filesystem.py

@@ -6,6 +6,7 @@ from astropy.table import Table
 
 from .base import Backend
 from ..types import TaskID
+from ..util.path import expand
 
 
 logger = logging.getLogger(__name__)
@@ -15,17 +16,21 @@ class FileSystemBackend(Backend):
     type: Literal["filesystem"] = "filesystem"
     base_path: Path
 
+    @property
+    def expanded_path(self) -> Path:
+        return expand(self.base_path)
+
     # ----------------------------
     # Helpers for paths
     # ----------------------------
     def _meta_path(self, task: TaskID) -> Path:
-        return self.base_path / f"{task}.meta.json"
+        return self.expanded_path / f"{task}.meta.json"
 
     def _marker_path(self, task: TaskID) -> Path:
-        return self.base_path / f"{task}.ok"
+        return self.expanded_path / f"{task}.ok"
 
     def _data_path(self, task: TaskID) -> Path:
-        return self.base_path / f"{task}.fits"
+        return self.expanded_path / f"{task}.fits"
 
     # ----------------------------
     # Metadata

timewise/chunking.py

@@ -1,20 +1,45 @@
-from typing import Iterator
+import logging
+from functools import cached_property
 from pathlib import Path
+from typing import Iterator, Sequence, cast
+
 import numpy as np
-from numpy import typing as npt
 import pandas as pd
-import logging
+from numpy import typing as npt
 
 logger = logging.getLogger(__name__)
 
 
 class Chunk:
     def __init__(
-        self, chunk_id: int, indices: npt.ArrayLike, row_indices: npt.ArrayLike
+        self, chunk_id: int, input_csv, row_indices: npt.NDArray[np.int_]
     ):
         self.chunk_id = chunk_id
-        self.indices = indices
         self.row_numbers = row_indices
+        self.input_csv = input_csv
+
+    @cached_property
+    def indices(self) -> pd.Index:
+        start = min(self.row_numbers)
+        stop = max(self.row_numbers) + 1
+        return pd.read_csv(self.input_csv, skiprows=start, nrows=stop - start).index
+
+    @property
+    def data(self) -> pd.DataFrame:
+        start = (
+            min(cast(Sequence[int], self.row_numbers)) + 1
+        )  # plus one to always skip header line
+        nrows = (
+            max(cast(Sequence[int], self.row_numbers)) - start + 2
+        )  # plus one: skip header, plus one:
+
+        columns = list(pd.read_csv(self.input_csv, nrows=0).columns)
+        return pd.read_csv(
+            filepath_or_buffer=self.input_csv,
+            skiprows=start,
+            nrows=nrows,
+            names=columns,
+        )
 
 
 class Chunker:
@@ -45,6 +70,5 @@ class Chunker:
             raise IndexError(f"Invalid chunk_id {chunk_id}")
         start = chunk_id * self.chunk_size
         stop = min(start + self.chunk_size, self._n_rows)
-        indices = pd.read_csv(self.input_csv, skiprows=start, nrows=stop - start).index
         logger.debug(f"chunk {chunk_id}: from {start} to {stop}")
-        return Chunk(chunk_id, indices, np.arange(start=start, stop=stop))
+        return Chunk(chunk_id, self.input_csv, np.arange(start=start, stop=stop))

timewise/io/config.py

@@ -7,6 +7,7 @@ from .download import Downloader
 from ..query import QueryType
 from ..backend import BackendType
 from ..types import TYPE_MAP
+from ..util.path import expand
 
 
 class DownloadConfig(BaseModel):
@@ -19,15 +20,19 @@ class DownloadConfig(BaseModel):
 
     service_url: str = "https://irsa.ipac.caltech.edu/TAP"
 
+    @property
+    def expanded_input_csv(self) -> Path:
+        return expand(self.input_csv)
+
     @model_validator(mode="after")
     def validate_input_csv_columns(self) -> "DownloadConfig":
         """Ensure that the input CSV contains all columns required by queries."""
         # only validate if the CSV actually exists
-        if not self.input_csv.exists():
-            raise ValueError(f"CSV file does not exist: {self.input_csv}")
+        if not self.expanded_input_csv.exists():
+            raise ValueError(f"CSV file does not exist: {self.expanded_input_csv}")
 
         # read just the header and first 10 lines
-        input_table = pd.read_csv(self.input_csv, nrows=10)
+        input_table = pd.read_csv(self.expanded_input_csv, nrows=10)
 
         missing_columns = set()
         wrong_dtype = set()
@@ -41,7 +46,7 @@ class DownloadConfig(BaseModel):
                     except Exception:
                         wrong_dtype.add(col)
 
-        msg = f"CSV file {self.input_csv}: "
+        msg = f"CSV file {self.expanded_input_csv}: "
         if missing_columns:
             raise KeyError(msg + f"Missing required columns: {sorted(missing_columns)}")
         if wrong_dtype:
@@ -55,7 +60,7 @@ class DownloadConfig(BaseModel):
     def build_downloader(self) -> Downloader:
         return Downloader(
             service_url=self.service_url,
-            input_csv=self.input_csv,
+            input_csv=self.expanded_input_csv,
             chunk_size=self.chunk_size,
             backend=self.backend,
             queries=self.queries,

timewise/io/download.py

@@ -106,25 +106,10 @@ class Downloader:
     # ----------------------------
     # TAP submission and download
     # ----------------------------
-    def get_chunk_data(self, chunk: Chunk) -> pd.DataFrame:
-        start = (
-            min(cast(Sequence[int], chunk.row_numbers)) + 1
-        )  # plus one to always skip header line
-        nrows = (
-            max(cast(Sequence[int], chunk.row_numbers)) - start + 2
-        )  # plus one: skip header, plus one:
-
-        columns = list(pd.read_csv(self.input_csv, nrows=0).columns)
-        return pd.read_csv(
-            filepath_or_buffer=self.input_csv,
-            skiprows=start,
-            nrows=nrows,
-            names=columns,
-        )
 
     def submit_tap_job(self, query: Query, chunk: Chunk) -> TAPJobMeta:
         adql = query.adql
-        chunk_df = self.get_chunk_data(chunk)
+        chunk_df = chunk.data
 
         assert all(chunk_df.index.isin(chunk.indices)), (
             "Some inputs loaded from wrong chunk!"

timewise/io/stable_tap.py

@@ -3,6 +3,8 @@ import backoff
 import pyvo as vo
 from xml.etree import ElementTree
 
+import requests
+
 from timewise.util.backoff import backoff_hndlr
 
 
@@ -20,6 +22,12 @@ class StableAsyncTAPJob(vo.dal.AsyncTAPJob):
         self.submit_response = None
 
     @classmethod
+    @backoff.on_exception(
+        backoff.expo,
+        requests.exceptions.HTTPError,
+        max_tries=5,
+        on_backoff=backoff_hndlr,
+    )
     def create(
         cls,
         baseurl,
@@ -89,6 +97,15 @@ class StableAsyncTAPJob(vo.dal.AsyncTAPJob):
     def phase(self):
         return super(StableAsyncTAPJob, self).phase
 
+    @backoff.on_exception(
+        backoff.expo,
+        vo.dal.DALServiceError,
+        max_tries=50,
+        on_backoff=backoff_hndlr,
+    )
+    def _update(self, *args, **kwargs):
+        return super(StableAsyncTAPJob, self)._update(*args, **kwargs)
+
 
 class StableTAPService(vo.dal.TAPService):
     """

timewise/plot/lightcurve.py

@@ -49,6 +49,9 @@ def plot_lightcurve(
                     markeredgecolor="k",
                     ecolor="k",
                     capsize=2,
+                    zorder=3,
+                    barsabove=True,
+                    elinewidth=.5
                 )
                 ax.scatter(
                     stacked_lightcurve[keys.MEAN + "_mjd"][ul_mask_stacked],
@@ -75,6 +78,7 @@ def plot_lightcurve(
                         c=colors[b],
                         markersize=4,
                         alpha=0.3,
+                        zorder=2
                     )
 
                 single_ul_m = m & ul_mask_raw
@@ -90,6 +94,7 @@ def plot_lightcurve(
                         alpha=0.3,
                         s=1,
                         label=label,
+                        zorder=1,
                     )
 
         except KeyError as e:

timewise/process/keys.py

@@ -1,10 +1,14 @@
+# keys for stacked lightcurve processing
 MEAN = "mean"
 MEDIAN = "median"
 RMS = "rms"
 UPPER_LIMIT = "ul"
 NPOINTS = "npoints"
 ZEROPOINT_EXT = "zeropoint"
-FLUX_EXT = "flux"
 FLUX_DENSITY_EXT = "fluxdensity"
+KSTEST_NORM_EXT = "kstest_norm_pvalue"
+
+# keys of the single exposure lightcurve
+FLUX_EXT = "flux"
 MAG_EXT = "mpro"
 ERROR_EXT = "sig"

timewise/process/stacking.py

@@ -1,7 +1,8 @@
 import logging
-from typing import cast, Dict, Any
+from typing import cast, Dict, Any, Literal
 
 from scipy import stats
+from scipy.special import gamma
 import numpy as np
 from numpy import typing as npt
 import pandas as pd
@@ -20,6 +21,26 @@ MAGNITUDE_ZEROPOINTS: Dict[str, float] = {"w1": 20.752, "w2": 19.596}
 FLUX_ZEROPOINTS = {"w1": 309.54, "w2": 171.787}
 
 
+def std_debias_factor(n: npt.NDArray[np.int64]) -> npt.NDArray[np.float64]:
+    # Cureton The American Statistician 22, 22–22 (1968). https://www.jstor.org/stable/2681876
+    return gamma((n - 1) / 2) / np.sqrt(2 / (n - 1)) / gamma(n / 2)
+
+
+def t_distribution_correction(n: npt.NDArray[np.int64]) -> npt.NDArray[np.float64]:
+    return stats.t.interval(0.68, df=n - 1)[1]  # type: ignore
+
+
+def no_correction(n: npt.NDArray[np.int64]) -> npt.NDArray[np.float64]:
+    return np.ones_like(n, dtype=float)
+
+
+CORRECTION_FUNCTIONS = {
+    "debias": std_debias_factor,
+    "tdist": t_distribution_correction,
+    "none": no_correction,
+}
+
+
 def calculate_epochs(
     f: pd.Series,
     e: pd.Series,
@@ -29,6 +50,11 @@ def calculate_epochs(
     outlier_threshold: float,
     outlier_quantile: float,
     outlier_mask: npt.NDArray[np.bool_] | None = None,
+    mean_name: Literal["mean", "median"] = "median",
+    std_name: Literal["std", "sdom", "sdom-1"] = "sdom-1",
+    correction_name: Literal["tdist", "debias", "none"] = "tdist",
+    calculate_pvalues: bool = False,
+    use_single_exposure_errors: bool = True,
 ) -> tuple[
     npt.NDArray[np.float64],
     npt.NDArray[np.float64],
@@ -36,6 +62,7 @@ def calculate_epochs(
     npt.NDArray[np.bool_],
     npt.NDArray[np.bool_],
     npt.NDArray[np.int64],
+    npt.NDArray[np.float64],
 ]:
     """
     Calculates the visits within a raw lightcurve.
@@ -50,8 +77,23 @@ def calculate_epochs(
     :type counts: np.array
     :param remove_outliers: whether to remove outliers
     :type remove_outliers: bool
+    :param outlier_threshold: threshold to identify outliers
+    :type outlier_threshold: float
+    :param outlier_quantile: quantile that the outlier_threshold is multiplied with
+    :type outlier_quantile: float
     :param outlier_mask: the outlier mask
-    :type outlier_mask: np.array
+    :type outlier_mask: np.array, optional
+    :param mean_name: name of the numpy function to calculate the mean, defaults to "mean"
+    :type mean_name: str, optional
+    :param std_name: name of the function to calculate the stacked error, defaults to "std"
+    :type std_name: str, optional
+    :param correction_name: name of the correction function to apply to the standard deviation, defaults to "debias"
+    :type correction_name: str, optional
+    :param calculate_pvalues: if true, calculate ks-test p-values to check consistency with normal distribution per visit
+    :type calculate_pvalues: bool
+    :param use_single_exposure_errors:
+        if true, use the maximum of the RMS and the combined single exposure measurements as the final uncertainty
+    :type use_single_exposure_errors: bool
     :return: the epoch
     :rtype: float
     """
@@ -64,6 +106,7 @@ def calculate_epochs(
             np.array([]),
             np.array([]),
             np.array([]),
+            np.array([]),
         )
 
     u_lims = pd.isna(e)
@@ -97,6 +140,14 @@ def calculate_epochs(
     n_loops = 0
 
     # recalculate uncertainty and median as long as no outliers left
+    mean_function = np.mean if mean_name == "mean" else np.median
+
+    # select function to use to correct standard deviation with
+    bias_correction_function = CORRECTION_FUNCTIONS[correction_name]
+
+    one_points_mask = None
+    visits_at_least_two_point = []
+
     while n_remaining_outlier > 0:
         # make a mask of values to use
         use_mask = ~outlier_mask & use_mask_ul & ~nan_mask  # type: ignore[operator]
@@ -109,7 +160,7 @@ def calculate_epochs(
         visits_zero_points = np.unique(visit_mask[zero_points_mask[visit_mask]])
         median[visits_at_least_one_point] = np.array(
             [
-                np.median(f[(visit_mask == i) & use_mask])
+                mean_function(f[(visit_mask == i) & use_mask])
                 for i in visits_at_least_one_point
             ]
         )
@@ -134,10 +185,20 @@ def calculate_epochs(
         one_points_mask = n_points <= 1
         # calculate standard deviation
         std = np.zeros_like(counts, dtype=float)
+        extra_factor = (
+            1
+            if std_name == "std"
+            else 1 / n_points[~one_points_mask]
+            if std_name == "sdom"
+            else 1 / (n_points[~one_points_mask] - 1)
+        )
         std[~one_points_mask] = (
-            np.sqrt(mean_deviation[~one_points_mask])
-            / (n_points[~one_points_mask] - 1)
-            * stats.t.interval(0.68, df=n_points[~one_points_mask] - 1)[1]
+            np.sqrt(
+                mean_deviation[~one_points_mask]
+                / (n_points[~one_points_mask] - 1)
+                * extra_factor
+            )
+            * bias_correction_function(n_points[~one_points_mask])
             # for visits with small number of detections we have to correct according to the t distribution
         )
         std[one_points_mask] = -np.inf
@@ -155,8 +216,14 @@ def calculate_epochs(
             single_exp_measurement_errors[n_points > 0] / n_points[n_points > 0]
         )
         e_meas[zero_points_mask] = np.nan
-        # take the maximum value of the measured single exposure errors and the standard deviation
-        u = np.maximum(std, e_meas)
+
+        if use_single_exposure_errors:
+            # take the maximum value of the measured single exposure errors and the standard deviation
+            u = np.maximum(std, e_meas)
+        else:
+            # take the rms as the measurement uncertainty except if there is only one single exposure
+            u = std
+            u[one_points_mask] = e_meas[one_points_mask]
 
         # Estimate the spread of the flux.
         # To be robust against outliers, do that with quantiles instead of std
@@ -187,7 +254,29 @@ def calculate_epochs(
         if n_loops > 20:
             raise Exception(f"{n_loops}!")
 
-    return median, u, bin_ulim_bool, outlier_mask, use_mask, n_points
+    # --------------------   calculate std for crosscheck   -------------------- #
+    if calculate_pvalues:
+        npstd = np.zeros_like(counts, dtype=float)
+        npstd[~one_points_mask] = np.array(  # type: ignore[operator]
+            [np.std(f[(visit_mask == i) & use_mask]) for i in visits_at_least_two_point]
+        )
+        npstd[one_points_mask] = np.nan
+
+        # ----------------   calculate compatibility with gaussian   ---------------- #
+        pvalues = np.ones_like(counts, dtype=float)
+        pvalues[~one_points_mask] = np.array(  # type: ignore[operator]
+            [
+                stats.kstest(
+                    f[(visit_mask == i) & use_mask],
+                    stats.norm(median[i], npstd[i]).cdf,  # type: ignore
+                ).pvalue
+                for i in visits_at_least_two_point
+            ]
+        )
+    else:
+        pvalues = np.full_like(counts, -999)
+
+    return median, u, bin_ulim_bool, outlier_mask, use_mask, n_points, pvalues
 
 
 def stack_visits(
@@ -195,6 +284,12 @@ def stack_visits(
     outlier_threshold: float,
     outlier_quantile: float,
     clean_outliers: bool = True,
+    mean_name: Literal["mean", "median"] = "median",
+    std_name: Literal["std", "sdom", "sdom-1"] = "sdom-1",
+    correction_name: Literal["tdist", "debias", "none"] = "tdist",
+    calculate_pvalues: bool = False,
+    use_single_exposure_errors: bool = True,
+    median_zeropoint_per_visit: bool = True,
 ):
     """
     Combine the data by visits of the satellite of one region in the sky.
@@ -207,6 +302,27 @@ def stack_visits(
 
     :param lightcurve: the raw lightcurve
     :type lightcurve: pandas.DataFrame
+    :param outlier_threshold: threshold to identify outliers
+    :type outlier_threshold: float
+    :param outlier_quantile: quantile that the outlier_threshold is multiplied with
+    :type outlier_quantile: float
+    :param clean_outliers:
+        if True, remove outliers that are outlier_threshold x outlier_quantile far away from the mean per visit,
+        default is True
+    :type clean_outliers: bool
+    :param mean_name: name of the numpy function to calculate the mean, defaults to "mean"
+    :type mean_name: str, optional
+    :param std_name: name of the function to calculate the stacked error, defaults to "std"
+    :type std_name: str, optional
+    :param correction_name: name of the correction function to apply to the standard deviation, defaults to "debias"
+    :type correction_name: str, optional
+    :param calculate_pvalues: if true, calculate ks-test p-values to check consistency with normal distribution per visit
+    :type calculate_pvalues: bool
+    :param use_single_exposure_errors:
+        if true, use the maximum of the RMS and the combined single exposure measurements as the final uncertainty
+    :type use_single_exposure_errors: bool
+    :param median_zeropoint_per_visit: if true, use the median zeropoint per visit instead of the individual exposure ones
+    :type median_zeropoint_per_visit: bool
     :return: the stacked lightcurve
     :rtype: pandas.DataFrame
     """
@@ -236,15 +352,22 @@ def stack_visits(
             remove_outliers = lum_ext == keys.FLUX_EXT and clean_outliers
             outlier_mask = outlier_masks.get(keys.FLUX_EXT, None)
 
-            mean, u, bin_ulim_bool, outlier_mask, use_mask, n_points = calculate_epochs(
-                f,
-                e,
-                visit_map,
-                counts,
-                remove_outliers=remove_outliers,
-                outlier_mask=outlier_mask,
-                outlier_quantile=outlier_quantile,
-                outlier_threshold=outlier_threshold,
+            mean, u, bin_ulim_bool, outlier_mask, use_mask, n_points, p_values = (
+                calculate_epochs(
+                    f,
+                    e,
+                    visit_map,
+                    counts,
+                    remove_outliers=remove_outliers,
+                    outlier_mask=outlier_mask,
+                    outlier_quantile=outlier_quantile,
+                    outlier_threshold=outlier_threshold,
+                    mean_name=mean_name,
+                    std_name=std_name,
+                    correction_name=correction_name,
+                    calculate_pvalues=calculate_pvalues,
+                    use_single_exposure_errors=use_single_exposure_errors,
+                )
             )
             n_outliers = np.sum(outlier_mask)
 
@@ -257,6 +380,7 @@ def stack_visits(
             stacked_data[f"{b}{lum_ext}{keys.RMS}"] = u
             stacked_data[f"{b}{lum_ext}{keys.UPPER_LIMIT}"] = bin_ulim_bool
             stacked_data[f"{b}{lum_ext}{keys.NPOINTS}"] = n_points
+            stacked_data[f"{b}{lum_ext}{keys.KSTEST_NORM_EXT}"] = p_values
 
             outlier_masks[lum_ext] = outlier_mask
             use_masks[lum_ext] = use_mask
@@ -289,20 +413,27 @@ def stack_visits(
         # if the visit only has upper limits then use the fall-back zeropoint
         zps_median[bin_ulim_bools[keys.FLUX_EXT]] = MAGNITUDE_ZEROPOINTS[b]
 
+        if median_zeropoint_per_visit:
+            use_zp = zps_median[visit_map]
+        else:
+            zp_nan_mask = np.isnan(zps)
+            zps[zp_nan_mask] = zps_median[visit_map[zp_nan_mask]]
+            use_zp = zps
+
         # ---------------   calculate flux density from instrument flux   ---------------- #
         # get the instrument flux [digital numbers], i.e. source count
         inst_fluxes_e = lightcurve[f"{b}{keys.ERROR_EXT}{keys.FLUX_EXT}"]
 
         # calculate the proportionality constant between flux density and source count
         mag_zp = FLUX_ZEROPOINTS[b] * 1e3  # in mJy
-        flux_dens_const = mag_zp * 10 ** (-zps_median / 2.5)
+        flux_dens_const = mag_zp * 10 ** (-use_zp / 2.5)
 
         # calculate flux densities from instrument counts
-        flux_densities = inst_fluxes * flux_dens_const[visit_map]
-        flux_densities_e = inst_fluxes_e * flux_dens_const[visit_map]
+        flux_densities = inst_fluxes * flux_dens_const
+        flux_densities_e = inst_fluxes_e * flux_dens_const
 
         # bin flux densities
-        mean_fd, u_fd, ul_fd, outlier_mask_fd, use_mask_fd, n_points_fd = (
+        mean_fd, u_fd, ul_fd, outlier_mask_fd, use_mask_fd, n_points_fd, p_values_fd = (
             calculate_epochs(
                 flux_densities,
                 flux_densities_e,
@@ -312,11 +443,17 @@ def stack_visits(
                 outlier_mask=outlier_masks[keys.FLUX_EXT],
                 outlier_threshold=outlier_threshold,
                 outlier_quantile=outlier_quantile,
+                mean_name=mean_name,
+                std_name=std_name,
+                correction_name=correction_name,
+                calculate_pvalues=calculate_pvalues,
+                use_single_exposure_errors=use_single_exposure_errors,
             )
         )
         stacked_data[f"{b}{keys.MEAN}{keys.FLUX_DENSITY_EXT}"] = mean_fd
         stacked_data[f"{b}{keys.FLUX_DENSITY_EXT}{keys.RMS}"] = u_fd
         stacked_data[f"{b}{keys.FLUX_DENSITY_EXT}{keys.UPPER_LIMIT}"] = ul_fd
         stacked_data[f"{b}{keys.FLUX_DENSITY_EXT}{keys.NPOINTS}"] = n_points_fd
+        stacked_data[f"{b}{keys.FLUX_DENSITY_EXT}{keys.KSTEST_NORM_EXT}"] = p_values_fd
 
     return pd.DataFrame(stacked_data)

timewise/util/path.py

@@ -0,0 +1,10 @@
+from pathlib import Path
+import os
+
+
+def expand(path: Path | str) -> Path:
+    """
+    Fully expand and resolve the Path with the given environment variables.
+    """
+    path = Path(path)
+    return Path(os.path.expandvars(path)).expanduser().resolve()

{timewise-1.0.0a6.dist-info → timewise-1.0.0a8.dist-info}/METADATA

@@ -1,23 +1,26 @@
 Metadata-Version: 2.4
 Name: timewise
-Version: 1.0.0a6
+Version: 1.0.0a8
 Summary: Download WISE infrared data for many objects and process them with AMPEL
 License: MIT
 License-File: LICENSE
 Author: Jannis Necker
 Author-email: jannis.necker@gmail.com
-Requires-Python: >=3.11,<3.12
+Requires-Python: >=3.11,<3.14
 Classifier: License :: OSI Approved :: MIT License
 Classifier: Programming Language :: Python :: 3
 Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
 Provides-Extra: ampel
 Provides-Extra: dev
 Provides-Extra: docs
-Requires-Dist: ampel-alerts (==0.10.3a5) ; extra == "ampel"
-Requires-Dist: ampel-core (>=0.10.4.post0,<0.11.0) ; extra == "ampel"
-Requires-Dist: ampel-photometry (>=0.10.1,<0.11.0) ; extra == "ampel"
+Requires-Dist: ampel-alerts (==0.10.3a6) ; extra == "ampel"
+Requires-Dist: ampel-core (==0.10.6a17) ; extra == "ampel"
+Requires-Dist: ampel-interface (==0.10.5a8) ; extra == "ampel"
+Requires-Dist: ampel-photometry (==0.10.2a1) ; extra == "ampel"
 Requires-Dist: ampel-plot (>=0.9.1,<0.10.0) ; extra == "ampel"
-Requires-Dist: astropy (>=5.1,<6.0.0)
+Requires-Dist: astropy (>=5.1,<8.0.0)
 Requires-Dist: autodoc_pydantic[erdantic] (>=2.2.0,<3.0.0) ; extra == "docs"
 Requires-Dist: backoff (>=2.1.2,<3.0.0)
 Requires-Dist: coveralls (>=3.3.1,<4.0.0) ; extra == "dev"
@@ -60,7 +63,7 @@ Description-Content-Type: text/markdown
 This package downloads WISE data for positions on the sky and stacks single-exposure photometry per visit
 
 ## Prerequisites
-Python version 3.11.
+Python version 3.11, 3.12 or 3.13.
 
 If you want to not only download individual exposure photometry but also stack detections per visit (see below),
 you must have access to a running [MongoDB](https://www.mongodb.com/)*. 
@@ -74,12 +77,12 @@ to get the MongoDB community edition. </sub>
 ### If you use timewise only for downloading
 The package can be installed via `pip` (but make sure to install the v1 pre-release):
 ```bash
-pip install --pre timewise==1.0.0a6
+pip install --pre timewise==1.0.0a8
 ```
 ### If you use timewise also for stacking individual exposures
 You must install with the `ampel` extra:
 ```bash
-pip install --pre 'timewise[ampel]==1.0.0a6'
+pip install --pre 'timewise[ampel]==1.0.0a8'
 ```
 To tell AMPEL which modules, aka units, to use, build the corresponding configuration file:
 ```bash

{timewise-1.0.0a6.dist-info → timewise-1.0.0a8.dist-info}/RECORD

@@ -1,36 +1,35 @@
-ampel/timewise/alert/TimewiseAlertSupplier.py,sha256=RmAWJVOfObaI1x0gt9agRt45WrVj1nhQchaMQ9QgUIA,4127
-ampel/timewise/alert/load/TimewiseFileLoader.py,sha256=ChjOpQpyGTA-H_fjnvQGwa37IFfvfJanPmnVKORVElo,4271
-ampel/timewise/ingest/TiCompilerOptions.py,sha256=xzGOOWAFoFsazocXLsTqtaHIqt_WUXFaacLUoBVKlbo,758
+ampel/timewise/alert/TimewiseAlertSupplier.py,sha256=HLeQBCLrmVKxqvPc4E2TVj241KWXD9hHK7MJA3EUWJw,4178
+ampel/timewise/alert/load/TimewiseFileLoader.py,sha256=jmLppFEJwjQCSOMUtr-cSP9UbtT0VzqyAf4VDJjUf3w,4409
+ampel/timewise/ingest/TiCompilerOptions.py,sha256=j2eUxSkJEKxAjJCd9plxRRCg6Mv8dNHw72UvKyAxUsA,809
 ampel/timewise/ingest/TiDataPointShaper.py,sha256=KbAg-J0DkMaJA9K67xMY5Pe0SIP0LNKsBNAMnTYV9zA,3180
-ampel/timewise/ingest/TiMongoMuxer.py,sha256=YDiqaYiJ1TvyHUMAlNWHBHCpw0tH4Nvf8eyHUxMbzS8,6475
+ampel/timewise/ingest/TiMongoMuxer.py,sha256=Rg6yGI7AoPZhyXUsYmGD8ZiivyaLT-EJjG-U4J4yh-w,6467
 ampel/timewise/ingest/tags.py,sha256=ggn0Y6QUnjIwMzOlgKmPX1JPhmnvc21nUrORFhaR0uI,564
-ampel/timewise/t1/T1HDBSCAN.py,sha256=vi1PWF0HFAedVJA4rVoR2YOXQ4jcEgEoUICtmg3rZjw,9301
-ampel/timewise/t1/TimewiseFilter.py,sha256=ZSaXiSDKXCusknLJRMhe2Awj_YfkM74JVXeRbiZonmA,1564
-ampel/timewise/t2/T2StackVisits.py,sha256=x7YcVLz_5fOPAfzso4fFWy4Pwn5txPfRT_Hx49PdbTo,1847
-ampel/timewise/util/AuxDiagnosticPlotter.py,sha256=MZP4LWpFgBUPWPxlx8EEFeRc8rhI7FDO_LlbhKtkwrw,1641
+ampel/timewise/t1/T1HDBSCAN.py,sha256=zVhcqQVAjlHgDBd9CJA6rGH3RCrCV84rsIIOWVxODKI,9528
+ampel/timewise/t1/TimewiseFilter.py,sha256=uDjNGsqPOISoFjSH8tX87I3UTW9qGX8rEmkyze8u2ug,1566
+ampel/timewise/t2/T2StackVisits.py,sha256=JBxFZLT-a-8jjenRw45ZVA_FvtcyOBmvMVcpYHyw_Yk,2574
+ampel/timewise/util/AuxDiagnosticPlotter.py,sha256=evvuH5vuEOFkfbkoSQw7ztiEqHos19PP91VQ_SCg_AU,1692
 ampel/timewise/util/pdutil.py,sha256=CJqeYqL2I6A5WnF5S342bwM4omJY-1XwT1XWY2uk3bc,1524
-conf/timewise/ampel.yml,sha256=8NuVSuKqinHT0G66uDPnnw-P7djScKKVHzeFNsH3XZc,360
-timewise/__init__.py,sha256=OhKzin_f9k_6U1rnhXEMzgMu70gbjQImfaWjmZiKhjA,24
+timewise/__init__.py,sha256=uSPlLo1iPfJHnwgxEbL4lAcPqtOK8qgwkOr4GpzJSyo,24
 timewise/backend/__init__.py,sha256=w79nWfCw8n9g98CkHWJELmb4j9xblWC4DGZOV3_XhH4,134
 timewise/backend/base.py,sha256=dHxRzu2q3uQ0wdGmDxnn-p68Tp19qChue7HMEu56wNA,1080
-timewise/backend/filesystem.py,sha256=GQ4Hrb6_7Q7fKOn6QUl8bqAihAmyeZoTRxElNIwPQ1Y,2465
-timewise/chunking.py,sha256=q7njvTSD84gdcvIk54SC-Ob863MsR79RPec8HS-bm4U,1668
+timewise/backend/filesystem.py,sha256=qFBmiOtfv8b3uhhiZSbH8bRIUrpS4f0P8Zl03VxpgAI,2598
+timewise/chunking.py,sha256=eRd3xof-5uHaxo8Pmwo4kr_cdZSfFpgTiPu-xuL-SGA,2404
 timewise/cli.py,sha256=W8SXlUQo7a5aS4_kW40KTs2UkTLQS12JuhgwkRb23DY,4172
 timewise/config.py,sha256=ZTSokRZMZDqBqVFV9DxvO-47yE9E9xWF48Rcjb0QG10,1044
 timewise/io/__init__.py,sha256=S7vb0glKJnw6ztOlrD-0Wma2bQZ2RwpmXDLFJLKBMVo,35
-timewise/io/config.py,sha256=aizLxt9l4aeWQNsvcemtQdr_fW1vLmpRSofcgA3Bgvk,2239
-timewise/io/download.py,sha256=IMNWur1FWPKJ7SNMTjevZgWqPP_eYLzXr4Pjp3XOeXI,10589
-timewise/io/stable_tap.py,sha256=jukCkBi2d7WACOo_kXTMCppzWUsN-pLVg9EDqHi3qd0,3478
+timewise/io/config.py,sha256=u_nlfstvuz2_bfepRZo3y43XAqQ0ilReYg9e081CTQg,2410
+timewise/io/download.py,sha256=4uKWIPD61wLk6rPLiX72yjByDFgrRR7ww4LzsDfVugE,10019
+timewise/io/stable_tap.py,sha256=obwzic_xvB_hh-x6S5NT4bW847emabKxp8NH1WgAh-g,3899
 timewise/plot/__init__.py,sha256=cc00UenWC_8zAkBH-Ylhs3yCF49tAqZ2Al9MfOoXYDI,120
 timewise/plot/diagnostic.py,sha256=GRp-OUwz2yzzDu9qdViFg_e3Mxl5t1IvUJXZHuMKB2U,8276
-timewise/plot/lightcurve.py,sha256=oK0y6RFzv7QSrO1Qqyc1wNghsTILC9QAfIjuoA8i92I,3757
+timewise/plot/lightcurve.py,sha256=K0UvjmvbCSsRKP3Mu9X5L5AVUwqURR76ESJCIMEgE6Q,3924
 timewise/plot/panstarrs.py,sha256=X2ZULm7QT91cp4qociG0fVeI0saGLJwyKzL0141Vqis,8014
 timewise/plot/sdss.py,sha256=cc1zU-4XFkqc8xH5yqCyMsDJf9w_54B_6NeRMjr9Pt8,2622
 timewise/process/__init__.py,sha256=Yk-j1B1MnBuuaM6eFi43TxdWmFKbwFHvDsuQZt4yB_c,70
 timewise/process/config.py,sha256=d4hCvNb8JyJf2-0BTQgJaeOmHGZK_aHEDMb_xyqJgbU,1060
 timewise/process/interface.py,sha256=JOIh1wYuE1s9XdSfBRS7qUXexlr-p5UxdwaSjKlRZxE,5207
-timewise/process/keys.py,sha256=0TEVn-BwfzHGlScU-N8AtxgkhA2mUO0wBu4_ol_ylH4,197
-timewise/process/stacking.py,sha256=5qk9SYTfDFNFsnw9KAGYW-RndGtdmENKEiIqD6mrJ6I,12707
+timewise/process/keys.py,sha256=MGnTHM4UGUSyX7BaCzoT_dg81NEAiLCxMLBRz3Kq8yc,319
+timewise/process/stacking.py,sha256=vVyrhrBRpqkptNryxNxWwEvXeQLH2GYNUtUxWggYfTk,18997
 timewise/process/template.yml,sha256=U_xKmygDl3E-viTgZEI8pQIJwWduB52SdI2X9vy61Yo,1037
 timewise/query/__init__.py,sha256=1OA_FlLI7O0aIDOXHpBKMOyMvYLCd0kQVkzoqbxouyE,242
 timewise/query/base.py,sha256=GjAoQZpaxVHcpCgPco1y5bpR0Jtd_sp_bYMI26WpAK0,1262
@@ -43,9 +42,10 @@ timewise/types.py,sha256=MB-aqpyos1aCS58QWBas34WcwFHiHOkrDj_d9_ZxuVc,667
 timewise/util/backoff.py,sha256=bU5yhsBO4U53XEPJ_32tgql9rq_Rzrv7w32nVQYHr64,272
 timewise/util/csv_utils.py,sha256=5i3Jd4c58-doPs1N_hyYZ8Uc2nvuk9nGwgPNZoNrlu0,298
 timewise/util/error_threading.py,sha256=uyV1Ri-wf87lpa17Xlp520B1V8DWHh3v9Mk97QrPmv0,2264
+timewise/util/path.py,sha256=-CcpYdjidg5FcZnRyQqrK_EKjc-7USYbJ8gtKatGl6k,254
 timewise/util/visits.py,sha256=4ZXwH7OXmp98tUPZplvuOtykB048kZoYXyuRTwQxT7w,998
-timewise-1.0.0a6.dist-info/METADATA,sha256=2m1fvHTkYBVmko9pENN3GjqWTHpmU9Dgq1Ut54sqmK8,10146
-timewise-1.0.0a6.dist-info/WHEEL,sha256=zp0Cn7JsFoX2ATtOhtaFYIiE2rmFAD4OcMhtUki8W3U,88
-timewise-1.0.0a6.dist-info/entry_points.txt,sha256=mYh1HsUFbV7KT8kxiGqVtR3Pk0oEk6Bd-2c5FsYVhG4,45
-timewise-1.0.0a6.dist-info/licenses/LICENSE,sha256=sVoNJWiTlH-NarJx0wdsob468Pg3JE6vIIgll4lCa3E,1070
-timewise-1.0.0a6.dist-info/RECORD,,
+timewise-1.0.0a8.dist-info/METADATA,sha256=QnfpFxRvJDiL_a5Z0ElFj6_F1Mylk4bscAqE0SCNjxE,10308
+timewise-1.0.0a8.dist-info/WHEEL,sha256=zp0Cn7JsFoX2ATtOhtaFYIiE2rmFAD4OcMhtUki8W3U,88
+timewise-1.0.0a8.dist-info/entry_points.txt,sha256=mYh1HsUFbV7KT8kxiGqVtR3Pk0oEk6Bd-2c5FsYVhG4,45
+timewise-1.0.0a8.dist-info/licenses/LICENSE,sha256=sVoNJWiTlH-NarJx0wdsob468Pg3JE6vIIgll4lCa3E,1070
+timewise-1.0.0a8.dist-info/RECORD,,

conf/timewise/ampel.yml

@@ -1,10 +0,0 @@
-unit:
-- ampel.timewise.alert.load.TimewiseFileLoader
-- ampel.timewise.alert.TimewiseAlertSupplier
-- ampel.timewise.ingest.TiDataPointShaper
-- ampel.timewise.ingest.TiMongoMuxer
-- ampel.timewise.ingest.TiCompilerOptions
-- ampel.timewise.t2.T2StackVisits
-- ampel.timewise.t1.T1HDBSCAN
-- ampel.timewise.util.AuxDiagnosticPlotter
-- ampel.timewise.t1.TimewiseFilter