timewise 1.0.0a1__py3-none-any.whl → 1.0.0a5__py3-none-any.whl

timewise/process/interface.py

@@ -1,6 +1,7 @@
 from pathlib import Path
 import logging
 from typing import Iterable, List, cast
+from importlib.util import find_spec
 
 import numpy as np
 from numpy import typing as npt
@@ -8,12 +9,19 @@ import pandas as pd
 from pymongo import MongoClient, ASCENDING
 from pymongo.collection import Collection
 from pymongo.database import Database
-from ampel.cli.JobCommand import JobCommand
-from ampel.types import DataPointId, StockId
+
+if find_spec("ampel.core"):
+    AMPEL_EXISTS = True
+    from ampel.cli.JobCommand import JobCommand
+else:
+    AMPEL_EXISTS = False
 
 
 logger = logging.getLogger(__name__)
 
+# copy from ampel.types
+StockId = int | bytes | str
+
 
 class AmpelInterface:
     def __init__(
@@ -71,6 +79,10 @@ class AmpelInterface:
         return self.make_ampel_job_file(cfg_path)
 
     def run(self, timewise_cfg_path: Path, ampel_config_path: Path):
+        if not AMPEL_EXISTS:
+            raise ModuleNotFoundError(
+                "You are trying to run ampel but it is not installed!"
+            )
         ampel_job_path = self.prepare(timewise_cfg_path)
         cmd = JobCommand()
         parser = cmd.get_parser()
@@ -123,7 +135,7 @@ class AmpelInterface:
             index.append(ic["id"])
         return pd.DataFrame(records, index=index)
 
-    def extract_selected_datapoint_ids(self, stock_id: StockId) -> List[DataPointId]:
+    def extract_selected_datapoint_ids(self, stock_id: StockId) -> List[int]:
         d = self.t1.find_one({"stock": stock_id})
         if d is None:
             return []

timewise/process/stacking.py

@@ -26,6 +26,8 @@ def calculate_epochs(
     visit_mask: npt.NDArray[np.int64],
     counts: npt.NDArray[np.int64],
     remove_outliers: bool,
+    outlier_threshold: float,
+    outlier_quantile: float,
     outlier_mask: npt.NDArray[np.bool_] | None = None,
 ) -> tuple[
     npt.NDArray[np.float64],
@@ -70,7 +72,7 @@ def calculate_epochs(
     # ---------------------   remove outliers in the bins   ---------------------- #
 
     # if we do not want to clean outliers just set the threshold to infinity
-    outlier_thresh = np.inf if not remove_outliers else 20
+    _outlier_threshold = np.inf if not remove_outliers else outlier_threshold
 
     # set up empty masks
     outlier_mask = cast(
@@ -156,15 +158,16 @@ def calculate_epochs(
         # take the maximum value of the measured single exposure errors and the standard deviation
         u = np.maximum(std, e_meas)
 
-        # calculate 90% confidence interval
-        u70 = np.zeros_like(counts, dtype=float)
-        u70[one_points_mask] = 1e-10
+        # Estimate the spread of the flux.
+        # To be robust against outliers, do that with quantiles instead of std
+        qs = np.zeros_like(counts, dtype=float)
+        qs[one_points_mask] = 1e-10
         visits_at_least_two_point = np.unique(visit_mask[~one_points_mask[visit_mask]])
-        u70[visits_at_least_two_point] = np.array(
+        qs[visits_at_least_two_point] = np.array(
             [
                 np.quantile(
                     abs(f[(visit_mask == i) & use_mask] - median[i]),
-                    0.7,
+                    outlier_quantile,
                     method="interpolated_inverted_cdf",
                 )
                 for i in visits_at_least_two_point
@@ -173,7 +176,7 @@ def calculate_epochs(
 
         # ---------------------   remove outliers in the bins   ---------------------- #
         remaining_outliers = (
-            abs(median[visit_mask] - f) > outlier_thresh * u70[visit_mask]
+            abs(median[visit_mask] - f) > _outlier_threshold * qs[visit_mask]
         ) & ~outlier_mask
         outlier_mask |= remaining_outliers
         n_remaining_outlier = sum(remaining_outliers) if remove_outliers else 0
@@ -187,7 +190,12 @@ def calculate_epochs(
     return median, u, bin_ulim_bool, outlier_mask, use_mask, n_points
 
 
-def stack_visits(lightcurve: pd.DataFrame, clean_outliers: bool = True):
+def stack_visits(
+    lightcurve: pd.DataFrame,
+    outlier_threshold: float,
+    outlier_quantile: float,
+    clean_outliers: bool = True,
+):
     """
     Combine the data by visits of the satellite of one region in the sky.
     The visits typically consist of some tens of observations. The individual visits are separated by about
@@ -204,7 +212,7 @@ def stack_visits(lightcurve: pd.DataFrame, clean_outliers: bool = True):
     """
 
     # -------------------------   create visit mask   -------------------------- #
-    visit_map = get_visit_map(lightcurve)
+    visit_map = get_visit_map(lightcurve.mjd)
     counts = np.bincount(visit_map)
 
     stacked_data: Dict[str, Any] = dict()
@@ -235,6 +243,8 @@ def stack_visits(lightcurve: pd.DataFrame, clean_outliers: bool = True):
                 counts,
                 remove_outliers=remove_outliers,
                 outlier_mask=outlier_mask,
+                outlier_quantile=outlier_quantile,
+                outlier_threshold=outlier_threshold,
             )
             n_outliers = np.sum(outlier_mask)
 
@@ -300,6 +310,8 @@ def stack_visits(lightcurve: pd.DataFrame, clean_outliers: bool = True):
                 counts,
                 remove_outliers=False,
                 outlier_mask=outlier_masks[keys.FLUX_EXT],
+                outlier_threshold=outlier_threshold,
+                outlier_quantile=outlier_quantile,
             )
         )
         stacked_data[f"{b}{keys.MEAN}{keys.FLUX_DENSITY_EXT}"] = mean_fd

timewise/query/base.py

@@ -1,6 +1,6 @@
 import abc
-from typing import ClassVar, List
-from pydantic import BaseModel
+from typing import ClassVar, List, Self
+from pydantic import BaseModel, model_validator
 from hashlib import sha256
 
 from ..tables import TableType
@@ -22,6 +22,13 @@ class Query(abc.ABC, BaseModel):
     columns: List[str]
     table: TableType
 
+    @model_validator(mode="after")
+    def check_columns(self) -> Self:
+        for column in self.columns:
+            if column not in self.table.columns_dtypes:
+                raise KeyError(f"{column} not found in table {self.table.name}")
+        return self
+
     @property
     @abc.abstractmethod
     def input_columns(self) -> dict[str, str]: ...

timewise/util/visits.py

@@ -3,31 +3,31 @@ import pandas as pd
 import numpy.typing as npt
 
 
-def get_visit_map(lightcurve: pd.DataFrame) -> npt.NDArray[np.int64]:
+def get_visit_map(
+    mjd: npt.NDArray[np.float64] | pd.Series,
+) -> npt.NDArray[np.int64]:
     """
     Create a map datapoint to visit
 
-    :param lightcurve: the raw lightcurve
-    :type lightcurve: pd.DataFrame
+    :param mjd: the MJDs of the observations
+    :type mjd: npt.NDArray[float]
     :returns: visit map
     :rtype: npt.ArrayLike
     """
     # -------------------------   find epoch intervals   -------------------------- #
-    sorted_mjds = np.sort(lightcurve.mjd)
+    sorted_mjds = np.sort(mjd)
     epoch_bounds_mask = (sorted_mjds[1:] - sorted_mjds[:-1]) > 100
     epoch_bins = np.array(
-        [
-            lightcurve.mjd.min() * 0.99
-        ]  # this makes sure that the first datapoint gets selected
+        [mjd.min() * 0.99]  # this makes sure that the first datapoint gets selected
         + list(
             ((sorted_mjds[1:] + sorted_mjds[:-1]) / 2)[epoch_bounds_mask]
         )  # finding the middle between
         +
         # two visits
         [
-            lightcurve.mjd.max() * 1.01
+            mjd.max() * 1.01
         ]  # this just makes sure that the last datapoint gets selected as well
     )
 
-    visit_mask = np.digitize(lightcurve.mjd, epoch_bins) - 1
+    visit_mask = np.digitize(mjd, epoch_bins) - 1
     return visit_mask

{timewise-1.0.0a1.dist-info → timewise-1.0.0a5.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: timewise
-Version: 1.0.0a1
+Version: 1.0.0a5
 Summary: Download WISE infrared data for many objects and process them with AMPEL
 License: MIT
 License-File: LICENSE
@@ -10,12 +10,13 @@ Requires-Python: >=3.11,<3.12
 Classifier: License :: OSI Approved :: MIT License
 Classifier: Programming Language :: Python :: 3
 Classifier: Programming Language :: Python :: 3.11
+Provides-Extra: ampel
 Provides-Extra: dev
 Provides-Extra: docs
-Requires-Dist: ampel-alerts (==0.10.3a5)
-Requires-Dist: ampel-core (>=0.10.4.post0,<0.11.0)
-Requires-Dist: ampel-photometry (>=0.10.1,<0.11.0)
-Requires-Dist: ampel-plot (>=0.9.1,<0.10.0)
+Requires-Dist: ampel-alerts (==0.10.3a5) ; extra == "ampel"
+Requires-Dist: ampel-core (>=0.10.4.post0,<0.11.0) ; extra == "ampel"
+Requires-Dist: ampel-photometry (>=0.10.1,<0.11.0) ; extra == "ampel"
+Requires-Dist: ampel-plot (>=0.9.1,<0.10.0) ; extra == "ampel"
 Requires-Dist: astropy (>=5.1,<6.0.0)
 Requires-Dist: autodoc_pydantic[erdantic] (>=2.2.0,<3.0.0) ; extra == "docs"
 Requires-Dist: backoff (>=2.1.2,<3.0.0)
@@ -36,7 +37,6 @@ Requires-Dist: ruff (>=0.13.0,<0.14.0) ; extra == "dev"
 Requires-Dist: scikit-image (>=0.19.3,<0.22.0)
 Requires-Dist: scikit-learn (>=1.3.0,<2.0.0)
 Requires-Dist: scipy-stubs (>=1.16.2.0,<2.0.0.0) ; extra == "dev"
-Requires-Dist: seaborn (>=0.11.2,<0.14.0)
 Requires-Dist: sphinx-rtd-theme (>=1.3.0,<2.0.0) ; extra == "docs"
 Requires-Dist: tqdm (>=4.64.0,<5.0.0)
 Requires-Dist: typer (>=0.19.2,<0.20.0)
@@ -60,23 +60,33 @@ Description-Content-Type: text/markdown
 This package downloads WISE data for positions on the sky and stacks single-exposure photometry per visit
 
 ## Prerequisites
+Python version 3.11.
 
-`timewise` makes use of [AMPEL](https://ampelproject.github.io/ampelastro/) and needs a running [MongoDB](https://www.mongodb.com/).
+If you want to not only download individual exposure photometry but also stack detections per visit (see below),
+you must have access to a running [MongoDB](https://www.mongodb.com/)*. 
+
+<sub>* On MacOS have alook at the custom `brew` tap 
+[here](https://stackoverflow.com/questions/57856809/installing-mongodb-with-homebrew)
+to get the MongoDB community edition. </sub>
 
 ## Installation
-The package can be installed via `pip`:
+
+### If you use timewise only for downloading
+The package can be installed via `pip` (but make sure to install the v1 pre-release):
 ```bash
-pip install timewise
+pip install --pre timewise==1.0.0a5
+```
+### If you use timewise also for stacking individual exposures
+You must install with the `ampel` extra:
+```bash
+pip install --pre timewise[ampel]==1.0.0a5
 ```
-
 To tell AMPEL which modules, aka units, to use, build the corresponding configuration file:
 ```bash
 ampel config build -distributions ampel timewise -stop-on-errors 0 -out <path-to-ampel-config-file>
 ```
 
-## Usage
-
-### Command line interface
+## Command line interface
 
 ```
  Usage: timewise [OPTIONS] COMMAND [ARGS]...                                                                
@@ -161,40 +171,50 @@ ampel:
 This configuration file will be the input to all subcommands. Downloading and stacking can be run together or separate.
 
 
-#### All-in-one:
-Run download, stacking, and export:
+### To only download the data:
 ```bash
-timewise run-chain <path-to-config-file> <path-to-ampel-config-file> <output-directory>
+timewise download <path-to-config-file>
 ```
+The photometry can be found in FITS files in the working directory specified in the configuration file\
+along with metadata JSON files. These tell `timewise` which quries have already completed (per chunk) so the
+download process can be interrupted and re-started at a later time.
+
+### Stack individual exposure by visits
+As mentioned above, this needs installation with the ampel extra.
+
 
-#### Separate download and processing:
-To only download the data:
+To **execute the stacking** after the download:
 ```bash
-timewise download <path-to-config-file>
+timewise process <path-to-config-file> <path-to-ampel-config-file>
 ```
 
-To execute the stacking:
+Make some **diagnostic plots** to check the datapoint selection and binning:
 ```bash
-timewise process <path-to-config-file> <path-to-ampel-config-file>
+timewise plot <path-to-config-file> <indices-to-plot> <output-directory>
 ```
 
-#### Run AMPEL manually
-Prepare an AMPEL job file for stacking the single-exposure data:
+As a shortcut, you can also run **download, stacking, and export in one command**:
 ```bash
-timewise prepare-ampel <path-to-config-file>
+timewise run-chain <path-to-config-file> <path-to-ampel-config-file> <output-directory>
 ```
-The result will contain the path to the prepared AMPEL job file that can be run with
+
+For more configuration options of the stacking, you can **run AMPEL manually**.
+
+1. Prepare an AMPEL job file for stacking the single-exposure data:
 ```bash
-ampel job -config <path-to-ampel-config-file> -schema <path-to-ampel-job-file>
+timewise prepare-ampel <path-to-config-file>
 ```
+The result will contain the path to the prepared AMPEL job file.
 
-#### Make some diagnostic plots
-To check the datapoint selection and binning, take a quick look at the data:
+2. Run the AMPEL job
 ```bash
-timewise plot <path-to-config-file> <indices-to-plot> <output-directory>
+ampel job -config <path-to-ampel-config-file> -schema <path-to-ampel-job-file>
 ```
 
 
+
+
+
 ## Citation
 If you use `timewise` please make sure to cite [Necker et al. A&A 695, A228 (2025)](https://www.aanda.org/articles/aa/abs/2025/03/aa51340-24/aa51340-24.html).
 Additionally, you might want to include a reference to the specific version you are using: [![DOI](https://zenodo.org/badge/449677569.svg)](https://zenodo.org/badge/latestdoi/449677569)

{timewise-1.0.0a1.dist-info → timewise-1.0.0a5.dist-info}/RECORD

@@ -1,27 +1,38 @@
-timewise/__init__.py,sha256=pHLUWHD0-i3VhQGA7h59p8IgmMU_fyxItpb1wjckTmY,24
+ampel/timewise/alert/TimewiseAlertSupplier.py,sha256=RmAWJVOfObaI1x0gt9agRt45WrVj1nhQchaMQ9QgUIA,4127
+ampel/timewise/alert/load/TimewiseFileLoader.py,sha256=ChjOpQpyGTA-H_fjnvQGwa37IFfvfJanPmnVKORVElo,4271
+ampel/timewise/ingest/TiCompilerOptions.py,sha256=xzGOOWAFoFsazocXLsTqtaHIqt_WUXFaacLUoBVKlbo,758
+ampel/timewise/ingest/TiDataPointShaper.py,sha256=KbAg-J0DkMaJA9K67xMY5Pe0SIP0LNKsBNAMnTYV9zA,3180
+ampel/timewise/ingest/TiMongoMuxer.py,sha256=YDiqaYiJ1TvyHUMAlNWHBHCpw0tH4Nvf8eyHUxMbzS8,6475
+ampel/timewise/ingest/tags.py,sha256=ggn0Y6QUnjIwMzOlgKmPX1JPhmnvc21nUrORFhaR0uI,564
+ampel/timewise/t1/T1HDBSCAN.py,sha256=vi1PWF0HFAedVJA4rVoR2YOXQ4jcEgEoUICtmg3rZjw,9301
+ampel/timewise/t1/TimewiseFilter.py,sha256=ZSaXiSDKXCusknLJRMhe2Awj_YfkM74JVXeRbiZonmA,1564
+ampel/timewise/t2/T2StackVisits.py,sha256=x7YcVLz_5fOPAfzso4fFWy4Pwn5txPfRT_Hx49PdbTo,1847
+ampel/timewise/util/AuxDiagnosticPlotter.py,sha256=MZP4LWpFgBUPWPxlx8EEFeRc8rhI7FDO_LlbhKtkwrw,1641
+ampel/timewise/util/pdutil.py,sha256=CJqeYqL2I6A5WnF5S342bwM4omJY-1XwT1XWY2uk3bc,1524
+timewise/__init__.py,sha256=8aDB3NLZbXhuSHcb6tn7CHSOTJ5XB8CCwEQpahSJtxw,24
 timewise/backend/__init__.py,sha256=w79nWfCw8n9g98CkHWJELmb4j9xblWC4DGZOV3_XhH4,134
 timewise/backend/base.py,sha256=dHxRzu2q3uQ0wdGmDxnn-p68Tp19qChue7HMEu56wNA,1080
 timewise/backend/filesystem.py,sha256=GQ4Hrb6_7Q7fKOn6QUl8bqAihAmyeZoTRxElNIwPQ1Y,2465
 timewise/chunking.py,sha256=q7njvTSD84gdcvIk54SC-Ob863MsR79RPec8HS-bm4U,1668
-timewise/cli.py,sha256=l6j-M2-x1LeDEAEnuDv9tadTNmVoxxMNgfyOaS-0oAw,3641
+timewise/cli.py,sha256=W8SXlUQo7a5aS4_kW40KTs2UkTLQS12JuhgwkRb23DY,4172
 timewise/config.py,sha256=ZTSokRZMZDqBqVFV9DxvO-47yE9E9xWF48Rcjb0QG10,1044
 timewise/io/__init__.py,sha256=S7vb0glKJnw6ztOlrD-0Wma2bQZ2RwpmXDLFJLKBMVo,35
 timewise/io/config.py,sha256=aizLxt9l4aeWQNsvcemtQdr_fW1vLmpRSofcgA3Bgvk,2239
-timewise/io/download.py,sha256=rKjeW7nh7xUtHh8llvsg_qmkx71Wzn2LggQnyHwqJOI,10719
+timewise/io/download.py,sha256=IMNWur1FWPKJ7SNMTjevZgWqPP_eYLzXr4Pjp3XOeXI,10589
 timewise/io/stable_tap.py,sha256=jukCkBi2d7WACOo_kXTMCppzWUsN-pLVg9EDqHi3qd0,3478
 timewise/plot/__init__.py,sha256=cc00UenWC_8zAkBH-Ylhs3yCF49tAqZ2Al9MfOoXYDI,120
-timewise/plot/diagnostic.py,sha256=CKxOYd030CRz-JBkxW5lo0GoQ3nNpEWKh3WlxVdio20,8272
+timewise/plot/diagnostic.py,sha256=GRp-OUwz2yzzDu9qdViFg_e3Mxl5t1IvUJXZHuMKB2U,8276
 timewise/plot/lightcurve.py,sha256=oK0y6RFzv7QSrO1Qqyc1wNghsTILC9QAfIjuoA8i92I,3757
 timewise/plot/panstarrs.py,sha256=X2ZULm7QT91cp4qociG0fVeI0saGLJwyKzL0141Vqis,8014
 timewise/plot/sdss.py,sha256=cc1zU-4XFkqc8xH5yqCyMsDJf9w_54B_6NeRMjr9Pt8,2622
 timewise/process/__init__.py,sha256=Yk-j1B1MnBuuaM6eFi43TxdWmFKbwFHvDsuQZt4yB_c,70
-timewise/process/config.py,sha256=0TtlxbZs5FNEUt_tv8DKpOxjxp7OoTVunLRUOSw0zPM,831
-timewise/process/interface.py,sha256=ZTUreyu_WkFTs8pOBNqFmCqgVs5OXEtPHjwpGVV1X_s,4929
+timewise/process/config.py,sha256=d4hCvNb8JyJf2-0BTQgJaeOmHGZK_aHEDMb_xyqJgbU,1060
+timewise/process/interface.py,sha256=JOIh1wYuE1s9XdSfBRS7qUXexlr-p5UxdwaSjKlRZxE,5207
 timewise/process/keys.py,sha256=0TEVn-BwfzHGlScU-N8AtxgkhA2mUO0wBu4_ol_ylH4,197
-timewise/process/stacking.py,sha256=X5GTkCV5TMn4JmAu8WVM0tQR6mocxNbxrMdR4OVx40Y,12256
+timewise/process/stacking.py,sha256=5qk9SYTfDFNFsnw9KAGYW-RndGtdmENKEiIqD6mrJ6I,12707
 timewise/process/template.yml,sha256=U_xKmygDl3E-viTgZEI8pQIJwWduB52SdI2X9vy61Yo,1037
 timewise/query/__init__.py,sha256=1OA_FlLI7O0aIDOXHpBKMOyMvYLCd0kQVkzoqbxouyE,242
-timewise/query/base.py,sha256=LzG207uIQOEE_RucEKBI4-sHR_EwyILGSlSk10HlEeU,973
+timewise/query/base.py,sha256=GjAoQZpaxVHcpCgPco1y5bpR0Jtd_sp_bYMI26WpAK0,1262
 timewise/query/positional.py,sha256=6fLfRdzat1jMqJAY7k7dEdnAZRiWyRKpWkbqpwzDKLo,1255
 timewise/tables/__init__.py,sha256=5efKk4YEhHLA_QTMYc8CQp0nQWXG-F3EAPE6935ManU,263
 timewise/tables/allwise_p3as_mep.py,sha256=NfCnTOdj_6DSFUWGmrwDyrrxIWvJFHktHWF40K_w5wQ,632
@@ -31,9 +42,9 @@ timewise/types.py,sha256=MB-aqpyos1aCS58QWBas34WcwFHiHOkrDj_d9_ZxuVc,667
 timewise/util/backoff.py,sha256=bU5yhsBO4U53XEPJ_32tgql9rq_Rzrv7w32nVQYHr64,272
 timewise/util/csv_utils.py,sha256=5i3Jd4c58-doPs1N_hyYZ8Uc2nvuk9nGwgPNZoNrlu0,298
 timewise/util/error_threading.py,sha256=uyV1Ri-wf87lpa17Xlp520B1V8DWHh3v9Mk97QrPmv0,2264
-timewise/util/visits.py,sha256=3intUo1iiSovu7PB7uUrT-IAxyuOxl58aL0mKj5dAMI,1039
-timewise-1.0.0a1.dist-info/METADATA,sha256=2KeU_-cjdIZePzr7sKNiu-nY97FeuBq087UrzXj_aI8,9172
-timewise-1.0.0a1.dist-info/WHEEL,sha256=zp0Cn7JsFoX2ATtOhtaFYIiE2rmFAD4OcMhtUki8W3U,88
-timewise-1.0.0a1.dist-info/entry_points.txt,sha256=mYh1HsUFbV7KT8kxiGqVtR3Pk0oEk6Bd-2c5FsYVhG4,45
-timewise-1.0.0a1.dist-info/licenses/LICENSE,sha256=sVoNJWiTlH-NarJx0wdsob468Pg3JE6vIIgll4lCa3E,1070
-timewise-1.0.0a1.dist-info/RECORD,,
+timewise/util/visits.py,sha256=4ZXwH7OXmp98tUPZplvuOtykB048kZoYXyuRTwQxT7w,998
+timewise-1.0.0a5.dist-info/METADATA,sha256=PWFFKX75JxlhRVYm6S9ypWaif_QT22vVe1b2VCQF18w,10181
+timewise-1.0.0a5.dist-info/WHEEL,sha256=zp0Cn7JsFoX2ATtOhtaFYIiE2rmFAD4OcMhtUki8W3U,88
+timewise-1.0.0a5.dist-info/entry_points.txt,sha256=mYh1HsUFbV7KT8kxiGqVtR3Pk0oEk6Bd-2c5FsYVhG4,45
+timewise-1.0.0a5.dist-info/licenses/LICENSE,sha256=sVoNJWiTlH-NarJx0wdsob468Pg3JE6vIIgll4lCa3E,1070
+timewise-1.0.0a5.dist-info/RECORD,,