timewise 1.0.0a8__tar.gz → 1.0.0a10__tar.gz

{timewise-1.0.0a8 → timewise-1.0.0a10}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: timewise
-Version: 1.0.0a8
+Version: 1.0.0a10
 Summary: Download WISE infrared data for many objects and process them with AMPEL
 License: MIT
 License-File: LICENSE
@@ -15,21 +15,22 @@ Classifier: Programming Language :: Python :: 3.13
 Provides-Extra: ampel
 Provides-Extra: dev
 Provides-Extra: docs
-Requires-Dist: ampel-alerts (==0.10.3a6) ; extra == "ampel"
-Requires-Dist: ampel-core (==0.10.6a17) ; extra == "ampel"
+Requires-Dist: ampel-alerts (==0.10.4a0) ; extra == "ampel"
+Requires-Dist: ampel-core (==0.10.6a21) ; extra == "ampel"
 Requires-Dist: ampel-interface (==0.10.5a8) ; extra == "ampel"
 Requires-Dist: ampel-photometry (==0.10.2a1) ; extra == "ampel"
 Requires-Dist: ampel-plot (>=0.9.1,<0.10.0) ; extra == "ampel"
 Requires-Dist: astropy (>=5.1,<8.0.0)
 Requires-Dist: autodoc_pydantic[erdantic] (>=2.2.0,<3.0.0) ; extra == "docs"
 Requires-Dist: backoff (>=2.1.2,<3.0.0)
-Requires-Dist: coveralls (>=3.3.1,<4.0.0) ; extra == "dev"
+Requires-Dist: coveralls (>=4.0.0,<5.0.0) ; extra == "dev"
 Requires-Dist: jupyter[jupyter] (>=1.0.0,<2.0.0)
 Requires-Dist: jupyterlab[jupyter] (>=4.0.6,<5.0.0)
 Requires-Dist: matplotlib (>=3.5.3,<4.0.0)
+Requires-Dist: mongomock (>=4.3.0,<5.0.0) ; extra == "dev"
 Requires-Dist: mypy (>=1.18.2,<2.0.0) ; extra == "dev"
 Requires-Dist: myst-parser (>=1,<3) ; extra == "docs"
-Requires-Dist: numpy (>=1.23.2,<2.0.0)
+Requires-Dist: numpy (>=1.23.2,<3.0.0)
 Requires-Dist: pandas (>=1.4.3,<3.0.0)
 Requires-Dist: pandas-stubs (>=2.3.2.250926,<3.0.0.0) ; extra == "dev"
 Requires-Dist: pydantic (>=2.0.0,<3.0.0)
@@ -37,12 +38,12 @@ Requires-Dist: pytest (>=7.2.2,<8.0.0) ; extra == "dev"
 Requires-Dist: pyvo (>=1.7.0,<2.0.0)
 Requires-Dist: requests (>=2.28.1,<3.0.0)
 Requires-Dist: ruff (>=0.13.0,<0.14.0) ; extra == "dev"
-Requires-Dist: scikit-image (>=0.19.3,<0.22.0)
+Requires-Dist: scikit-image (>=0.26.0,<0.27.0)
 Requires-Dist: scikit-learn (>=1.3.0,<2.0.0)
 Requires-Dist: scipy-stubs (>=1.16.2.0,<2.0.0.0) ; extra == "dev"
 Requires-Dist: sphinx-rtd-theme (>=1.3.0,<2.0.0) ; extra == "docs"
 Requires-Dist: tqdm (>=4.64.0,<5.0.0)
-Requires-Dist: typer (>=0.19.2,<0.20.0)
+Requires-Dist: typer (>=0.19.2,<0.30.0)
 Requires-Dist: types-pyyaml (>=6.0.12.20250915,<7.0.0.0) ; extra == "dev"
 Requires-Dist: types-requests (>=2.32.4.20250913,<3.0.0.0) ; extra == "dev"
 Requires-Dist: urllib3 (>=2.5.0,<3.0.0)
@@ -60,29 +61,31 @@ Description-Content-Type: text/markdown
 ![](timewise.png)
 # Infrared light curves from WISE data
 
-This package downloads WISE data for positions on the sky and stacks single-exposure photometry per visit
+This package downloads WISE data for positions on the sky and stacks single-exposure photometry per visit. It is designed to do so for efficiently for large samples of millions of objects.
 
 ## Prerequisites
 Python version 3.11, 3.12 or 3.13.
 
 If you want to not only download individual exposure photometry but also stack detections per visit (see below),
-you must have access to a running [MongoDB](https://www.mongodb.com/)*. 
+you must have access to a running [MongoDB](https://www.mongodb.com/)* **. 
 
 <sub>* On MacOS have alook at the custom `brew` tap 
 [here](https://github.com/mongodb/homebrew-brew)
 to get the MongoDB community edition. </sub>
 
+<sub>** On some systems this is not straight forward to set up. `timewise` requires it nevertheless as an integral part of the AMPEL system which is used to efficiently schedule and store the stacking of lightcurves. If you do not foresee a big overhead in calculating lightcurves for a sample of O(1000) objects, a more lightweight package might be more applicable. </sub>
+
 ## Installation
 
 ### If you use timewise only for downloading
 The package can be installed via `pip` (but make sure to install the v1 pre-release):
 ```bash
-pip install --pre timewise==1.0.0a8
+pip install --pre timewise==1.0.0a10
 ```
 ### If you use timewise also for stacking individual exposures
 You must install with the `ampel` extra:
 ```bash
-pip install --pre 'timewise[ampel]==1.0.0a8'
+pip install --pre 'timewise[ampel]==1.0.0a10'
 ```
 To tell AMPEL which modules, aka units, to use, build the corresponding configuration file:
 ```bash

{timewise-1.0.0a8 → timewise-1.0.0a10}/README.md

@@ -7,29 +7,31 @@
 ![](timewise.png)
 # Infrared light curves from WISE data
 
-This package downloads WISE data for positions on the sky and stacks single-exposure photometry per visit
+This package downloads WISE data for positions on the sky and stacks single-exposure photometry per visit. It is designed to do so for efficiently for large samples of millions of objects.
 
 ## Prerequisites
 Python version 3.11, 3.12 or 3.13.
 
 If you want to not only download individual exposure photometry but also stack detections per visit (see below),
-you must have access to a running [MongoDB](https://www.mongodb.com/)*. 
+you must have access to a running [MongoDB](https://www.mongodb.com/)* **. 
 
 <sub>* On MacOS have alook at the custom `brew` tap 
 [here](https://github.com/mongodb/homebrew-brew)
 to get the MongoDB community edition. </sub>
 
+<sub>** On some systems this is not straight forward to set up. `timewise` requires it nevertheless as an integral part of the AMPEL system which is used to efficiently schedule and store the stacking of lightcurves. If you do not foresee a big overhead in calculating lightcurves for a sample of O(1000) objects, a more lightweight package might be more applicable. </sub>
+
 ## Installation
 
 ### If you use timewise only for downloading
 The package can be installed via `pip` (but make sure to install the v1 pre-release):
 ```bash
-pip install --pre timewise==1.0.0a8
+pip install --pre timewise==1.0.0a10
 ```
 ### If you use timewise also for stacking individual exposures
 You must install with the `ampel` extra:
 ```bash
-pip install --pre 'timewise[ampel]==1.0.0a8'
+pip install --pre 'timewise[ampel]==1.0.0a10'
 ```
 To tell AMPEL which modules, aka units, to use, build the corresponding configuration file:
 ```bash

{timewise-1.0.0a8 → timewise-1.0.0a10}/ampel/timewise/alert/TimewiseAlertSupplier.py

@@ -9,7 +9,7 @@
 
 import sys
 from hashlib import blake2b
-from typing import Literal, List
+from typing import Literal, List, Dict, Any
 
 import pandas as pd
 
@@ -71,8 +71,8 @@ class TimewiseAlertSupplier(BaseAlertSupplier, AmpelABC):
 
             move = {
                 c: c.replace("_ep", "")
-                for c in columns_to_rename
-                if c.replace("_ep", "") in table.columns
+                for c in table.columns
+                if (c.replace("_ep", "") in table.columns) and (c.endswith("_ep"))
             }
             if move:
                 # In this case, the columns already exists because the neowise data is present
@@ -88,7 +88,7 @@ class TimewiseAlertSupplier(BaseAlertSupplier, AmpelABC):
         for i, row in table.iterrows():
             # convert table row to dict, convert data types from numpy to native python
             # Respect masked fields and convert to None
-            pp = {k: None if pd.isna(v) else v for k, v in row.to_dict().items()}
+            pp = {str(k): None if pd.isna(v) else v for k, v in row.to_dict().items()}
             pp_hash = blake2b(encode(pp), digest_size=7).digest()
             if self.counter:
                 pp["candid"] = self.counter

{timewise-1.0.0a8 → timewise-1.0.0a10}/ampel/timewise/alert/load/TimewiseFileLoader.py

@@ -32,6 +32,9 @@ class TimewiseFileLoader(AbsAlertLoader[Dict], AmpelABC):
 
     chunks: list[int] | None = None
 
+    # optionally skip files that are missing
+    skip_missing_files: bool = False
+
     def __init__(self, **kwargs) -> None:
         super().__init__(**kwargs)
 
@@ -81,7 +84,14 @@ class TimewiseFileLoader(AbsAlertLoader[Dict], AmpelABC):
             data = []
             for task in tasks:
                 self.logger.debug(f"reading {task}")
-                idata = backend.load_data(task)
+                try:
+                    idata = backend.load_data(task)
+                except FileNotFoundError as e:
+                    if self.skip_missing_files:
+                        self.logger.warn(f"file for task {task} not found, skipping...")
+                        continue
+                    else:
+                        raise e
 
                 # add table name
                 idata["table_name"] = (

{timewise-1.0.0a8 → timewise-1.0.0a10}/ampel/timewise/ingest/TiMongoMuxer.py

@@ -8,13 +8,20 @@
 
 from bisect import bisect_right
 from contextlib import suppress
-from typing import Any
-
+from typing import Any, Sequence
 
 from ampel.abstract.AbsT0Muxer import AbsT0Muxer
 from ampel.content.DataPoint import DataPoint
-from ampel.types import DataPointId, StockId
+from ampel.model.operator.AllOf import AllOf
+from ampel.model.operator.AnyOf import AnyOf
+from ampel.types import ChannelId, DataPointId, StockId
 from ampel.util.mappings import unflatten_dict
+from astropy.table import Table
+from pydantic import TypeAdapter
+from timewise.io.stable_tap import StableTAPService
+from timewise.query import QueryType
+from timewise.tables.allwise_p3as_mep import allwise_p3as_mep
+from timewise.types import TYPE_MAP
 
 
 class ConcurrentUpdateError(Exception):
@@ -51,8 +58,13 @@ class TiMongoMuxer(AbsT0Muxer):
         "body.dec": 1,
     }
 
+    channel: None | ChannelId | AnyOf[ChannelId] | AllOf[ChannelId] = None
+
     unique_key: list[str] = ["mjd", "ra", "dec"]
 
+    # URL of tap service for query of AllWISE Source Table
+    tap_service_url: str = "https://irsa.ipac.caltech.edu/TAP"
+
     def __init__(self, **kwargs) -> None:
         super().__init__(**kwargs)
 
@@ -60,6 +72,11 @@ class TiMongoMuxer(AbsT0Muxer):
         self._photo_col = self.context.db.get_collection("t0")
         self._projection_spec = unflatten_dict(self.projection)
 
+        self._tap_service = StableTAPService(self.tap_service_url)
+
+        self._allwise_source_cntr: list[str] = []
+        self._not_allwise_source_cntr: list[str] = []
+
     def process(
         self, dps: list[DataPoint], stock_id: None | StockId = None
     ) -> tuple[None | list[DataPoint], None | list[DataPoint]]:
@@ -81,7 +98,76 @@ class TiMongoMuxer(AbsT0Muxer):
 
     # NB: this 1-liner is a separate method to provide a patch point for race condition testing
     def _get_dps(self, stock_id: None | StockId) -> list[DataPoint]:
-        return list(self._photo_col.find({"stock": stock_id}, self.projection))
+        if self.channel is not None:
+            if isinstance(self.channel, ChannelId):
+                channel_query: (
+                    ChannelId | dict[str, Sequence[ChannelId | AllOf[ChannelId]]]
+                ) = self.channel
+            elif isinstance(self.channel, AnyOf):
+                channel_query = {"$in": self.channel.any_of}
+            elif isinstance(self.channel, AllOf):
+                channel_query = {"$all": self.channel.all_of}
+            else:
+                # should not happen
+                raise TypeError()
+            _channel = {"channel": channel_query}
+        else:
+            _channel = {}
+        query = {"stock": stock_id, **_channel}
+        return list(self._photo_col.find(query, self.projection))
+
+    def _check_cntrs(self, dps: Sequence[DataPoint]) -> None:
+        # assemble query
+        query_config = {
+            "type": "by_allwise_cntr_and_position",
+            "radius_arcsec": 10,
+            "columns": ["cntr"],
+            "constraints": [],
+            "table": {"name": "allwise_p3as_psd"},
+        }
+        query: QueryType = TypeAdapter(QueryType).validate_python(query_config)
+
+        # load datapoints into astropy table
+        upload = Table([dp["body"] for dp in dps])
+        upload["allwise_cntr"] = upload[allwise_p3as_mep.allwise_cntr_column]
+        upload[query.original_id_key] = [dp["id"] for dp in dps]
+        for key, dtype in query.input_columns.items():
+            upload[key] = upload[key].astype(TYPE_MAP[dtype])
+        for key in upload.colnames:
+            if key not in query.input_columns:
+                upload.remove_column(key)
+
+        # run query
+        self.logger.info("Querying AllWISE Source Table for MEP CNTRs ...")
+        res = self._tap_service.run_sync(
+            query.adql, uploads={query.upload_name: upload}
+        )
+
+        # update internal state
+        res_cntr = res.to_table()["cntr"].astype(str)
+        self._allwise_source_cntr.extend(list(res_cntr))
+        self._not_allwise_source_cntr.extend(
+            list(set(upload["allwise_cntr"].astype(str)) - set(res_cntr))
+        )
+
+    def _check_mep_allwise_sources(self, dps: Sequence[DataPoint]) -> list[DataPointId]:
+        dps_with_unchecked_cntr = [
+            dp
+            for dp in dps
+            if str(dp["body"][allwise_p3as_mep.allwise_cntr_column])
+            not in self._allwise_source_cntr + self._not_allwise_source_cntr
+        ]
+        if len(dps_with_unchecked_cntr) > 0:
+            self._check_cntrs(dps_with_unchecked_cntr)
+
+        # compile list of invalid datapoint ids
+        invalid_dp_ids = []
+        for dp in dps:
+            cntr = str(dp["body"][allwise_p3as_mep.allwise_cntr_column])
+            if cntr in self._not_allwise_source_cntr:
+                invalid_dp_ids.append(dp["id"])
+
+        return invalid_dp_ids
 
     def _process(
         self, dps: list[DataPoint], stock_id: None | StockId = None
@@ -128,7 +214,10 @@ class TiMongoMuxer(AbsT0Muxer):
             else:
                 unique_dps_ids[key] = [dp["id"]]
 
-        # make sure no duplicate datapoints exist
+        # Part 2: Check that there are no duplicates and handle redundant AllWISE MEP data
+        ##################################################################################
+
+        invalid_dp_ids = []
         for key, simultaneous_dps in unique_dps_ids.items():
             dps_db_wrong = [dp for dp in dps_db if dp["id"] in simultaneous_dps]
             dps_wrong = [dp for dp in dps if dp["id"] in simultaneous_dps]
@@ -136,20 +225,47 @@ class TiMongoMuxer(AbsT0Muxer):
                 f"stockID {str(stock_id)}: Duplicate photopoints at {key}!\nDPS from DB:"
                 f"\n{dps_db_wrong}\nNew DPS:\n{dps_wrong}"
             )
-            assert len(simultaneous_dps) == 1, msg
 
-        # Part 2: Update new data points that are already superseded
-        ############################################################
+            all_wrong_dps = dps_db_wrong + dps_wrong
+            if len(simultaneous_dps) > 1:
+                # if these datapoints come from the AllWISE MEP database, downloaded by timewise
+                # there can be duplicates. Only the AllWISE CNTR can tell us which datapoints
+                # should be used: the CNTR that appears in the AllWISE source catalog.
+                if all(
+                    [
+                        ("TIMEWISE" in dp["tag"]) and ("allwise_p3as_mep" in dp["tag"])
+                        for dp in all_wrong_dps
+                    ]
+                ):
+                    self.logger.info(
+                        f"{len(all_wrong_dps)} duplicate MEP datapoints found. Checking ..."
+                    )
+                    i_invalid_dp_ids = self._check_mep_allwise_sources(
+                        dps_db_wrong + dps_wrong
+                    )
+                    self.logger.info(
+                        f"Found {len(i_invalid_dp_ids)} invalid MEP datapoints."
+                    )
+                    invalid_dp_ids.extend(i_invalid_dp_ids)
+
+                else:
+                    raise RuntimeError(msg)
+
+        # Part 3: Compile final lists of datapoints to insert and combine
+        #################################################################
 
         # Difference between candids from the alert and candids present in DB
-        ids_dps_to_insert = ids_dps_alert - ids_dps_db
+        ids_dps_to_insert = ids_dps_alert - ids_dps_db - set(invalid_dp_ids)
         dps_to_insert = [dp for dp in dps if dp["id"] in ids_dps_to_insert]
         dps_to_combine = [
-            dp for dp in dps + dps_db if dp["id"] in ids_dps_alert | ids_dps_db
+            dp
+            for dp in dps + dps_db
+            if dp["id"] in ((ids_dps_alert | ids_dps_db) - set(invalid_dp_ids))
         ]
         self.logger.debug(
             f"Got {len(ids_dps_alert)} datapoints from alerts, "
             f"found {len(dps_db)} in DB, "
+            f"{len(invalid_dp_ids)} invalid datapoints, "
             f"inserting {len(dps_to_insert)} datapoints, "
             f"combining {len(dps_to_combine)} datapoints"
         )

{timewise-1.0.0a8 → timewise-1.0.0a10}/ampel/timewise/t1/T1HDBSCAN.py

@@ -37,6 +37,7 @@ else:
 class T1HDBSCAN(AbsT1CombineUnit):
     input_mongo_db_name: str
     original_id_key: str
+    mongo: str = "mongodb://localhost:27017/"
     whitelist_region_arcsec: float = 1
     cluster_distance_arcsec: float = 0.5
 
@@ -57,7 +58,7 @@ class T1HDBSCAN(AbsT1CombineUnit):
 
     def __init__(self, **kwargs):
         super().__init__(**kwargs)
-        self._col = MongoClient()[self.input_mongo_db_name]["input"]
+        self._col = MongoClient(self.mongo)[self.input_mongo_db_name]["input"]
         self._plotter = AuxUnitRegister.new_unit(
             model=self.plotter, sub_type=AuxDiagnosticPlotter
         )

{timewise-1.0.0a8 → timewise-1.0.0a10}/ampel/timewise/t1/TimewiseFilter.py

@@ -17,7 +17,7 @@ from timewise.process import keys
 
 class TimewiseFilter(AbsAlertFilter):
     det_per_visit: int = 8
-    n_visits = 10
+    n_visits: int = 10
 
     def process(self, alert: AmpelAlertProtocol) -> None | bool | int:
         columns = [

{timewise-1.0.0a8 → timewise-1.0.0a10}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "poetry.core.masonry.api"
 
 [project]
 name = "timewise"
-version = "1.0.0a8"
+version = "1.0.0a10"
 description = "Download WISE infrared data for many objects and process them with AMPEL"
 authors = [
     { name = "Jannis Necker", email = "jannis.necker@gmail.com" },
@@ -16,18 +16,18 @@ dependencies = [
     "tqdm>=4.64.0,<5.0.0",
     "requests>=2.28.1,<3.0.0",
     "pandas>=1.4.3,<3.0.0",
-    "numpy>=1.23.2,<2.0.0",
+    "numpy>=1.23.2,<3.0.0",
     "pyvo>=1.7.0,<2.0.0",
     "astropy>=5.1,<8.0.0",
     "matplotlib>=3.5.3,<4.0.0",
-    "scikit-image>=0.19.3,<0.22.0",
+    "scikit-image>=0.26.0,<0.27.0",
     "backoff>=2.1.2,<3.0.0",
     "virtualenv>=20.16.3,<21.0.0",
     "pydantic>=2.0.0,<3.0.0",
     "scikit-learn>=1.3.0,<2.0.0",
     "jupyterlab[jupyter]>=4.0.6,<5.0.0",
     "jupyter[jupyter]>=1.0.0,<2.0.0",
-    "typer (>=0.19.2,<0.20.0)",
+    "typer (>=0.19.2,<0.30.0)",
     "urllib3 (>=2.5.0,<3.0.0)",
 ]
 
@@ -46,7 +46,7 @@ Homepage = "https://github.com/JannisNe/timewise"
 
 [project.optional-dependencies]
 dev = [
-    "coveralls>=3.3.1,<4.0.0",
+    "coveralls>=4.0.0,<5.0.0",
     "pytest>=7.2.2,<8.0.0",
     "ruff>=0.13.0,<0.14.0",
     "mypy (>=1.18.2,<2.0.0)",
@@ -54,6 +54,7 @@ dev = [
     "scipy-stubs (>=1.16.2.0,<2.0.0.0)",
     "types-pyyaml (>=6.0.12.20250915,<7.0.0.0)",
     "types-requests (>=2.32.4.20250913,<3.0.0.0)",
+    "mongomock (>=4.3.0,<5.0.0)",
 ]
 docs = [
     "myst-parser>=1,<3",
@@ -63,8 +64,8 @@ docs = [
 ampel= [
     "ampel-photometry (==0.10.2a1)",
     "ampel-plot (>=0.9.1,<0.10.0)",
-    "ampel-core (==0.10.6a17)",
-    "ampel-alerts (==0.10.3a6)",
+    "ampel-core (==0.10.6a21)",
+    "ampel-alerts (==0.10.4a0)",
     "ampel-interface (==0.10.5a8)"
 ]
 

timewise-1.0.0a10/timewise/__init__.py

@@ -0,0 +1 @@
+__version__ = "1.0.0a10"

{timewise-1.0.0a8 → timewise-1.0.0a10}/timewise/backend/base.py

@@ -20,6 +20,8 @@ class Backend(abc.ABC, BaseModel):
     def save_meta(self, task: TaskID, meta: dict[str, Any]) -> None: ...
     @abc.abstractmethod
     def load_meta(self, task: TaskID) -> dict[str, Any] | None: ...
+    @abc.abstractmethod
+    def drop_meta(self, task: TaskID) -> None: ...
 
     # --- Markers ---
     @abc.abstractmethod

{timewise-1.0.0a8 → timewise-1.0.0a10}/timewise/backend/filesystem.py

@@ -52,6 +52,9 @@ class FileSystemBackend(Backend):
     def meta_exists(self, task: TaskID) -> bool:
         return self._meta_path(task).exists()
 
+    def drop_meta(self, task: TaskID) -> None:
+        self._meta_path(task).unlink()
+
     # ----------------------------
     # Markers
     # ----------------------------

{timewise-1.0.0a8 → timewise-1.0.0a10}/timewise/chunking.py

@@ -11,9 +11,7 @@ logger = logging.getLogger(__name__)
 
 
 class Chunk:
-    def __init__(
-        self, chunk_id: int, input_csv, row_indices: npt.NDArray[np.int_]
-    ):
+    def __init__(self, chunk_id: int, input_csv, row_indices: npt.NDArray[np.int_]):
         self.chunk_id = chunk_id
         self.row_numbers = row_indices
         self.input_csv = input_csv
@@ -71,4 +69,4 @@ class Chunker:
         start = chunk_id * self.chunk_size
         stop = min(start + self.chunk_size, self._n_rows)
         logger.debug(f"chunk {chunk_id}: from {start} to {stop}")
-        return Chunk(chunk_id, self.input_csv, np.arange(start=start, stop=stop))
+        return Chunk(chunk_id, self.input_csv, np.arange(start, stop))

{timewise-1.0.0a8 → timewise-1.0.0a10}/timewise/cli.py

@@ -53,8 +53,16 @@ def main(
 @app.command(help="Download WISE photometry from IRSA")
 def download(
     config_path: config_path_type,
+    resubmit_failed: Annotated[
+        bool,
+        typer.Option(
+            help="Re-submit jobs when failed due to connection issues",
+        ),
+    ] = False,
 ):
-    TimewiseConfig.from_yaml(config_path).download.build_downloader().run()
+    TimewiseConfig.from_yaml(config_path).download.build_downloader(
+        resubmit_failed=resubmit_failed
+    ).run()
 
 
 # the following commands will only be added if ampel is installed

{timewise-1.0.0a8 → timewise-1.0.0a10}/timewise/io/config.py

@@ -17,6 +17,7 @@ class DownloadConfig(BaseModel):
     poll_interval: float = 10.0
     queries: List[QueryType] = Field(..., description="One or more queries per chunk")
     backend: BackendType = Field(..., discriminator="type")
+    resubmit_failed: bool = False
 
     service_url: str = "https://irsa.ipac.caltech.edu/TAP"
 
@@ -57,13 +58,16 @@ class DownloadConfig(BaseModel):
 
         return self
 
-    def build_downloader(self) -> Downloader:
-        return Downloader(
-            service_url=self.service_url,
-            input_csv=self.expanded_input_csv,
-            chunk_size=self.chunk_size,
-            backend=self.backend,
-            queries=self.queries,
-            max_concurrent_jobs=self.max_concurrent_jobs,
-            poll_interval=self.poll_interval,
-        )
+    def build_downloader(self, **overwrite) -> Downloader:
+        default = {
+            "service_url": self.service_url,
+            "input_csv": self.expanded_input_csv,
+            "chunk_size": self.chunk_size,
+            "backend": self.backend,
+            "queries": self.queries,
+            "max_concurrent_jobs": self.max_concurrent_jobs,
+            "poll_interval": self.poll_interval,
+            "resubmit_failed": self.resubmit_failed,
+        }
+        default.update(overwrite)
+        return Downloader(**default)  # type: ignore

{timewise-1.0.0a8 → timewise-1.0.0a10}/timewise/io/download.py

@@ -1,25 +1,22 @@
-import time
-import threading
 import logging
-from queue import Empty
-from typing import Dict, Iterator, cast, Sequence
+import threading
+import time
+from datetime import datetime, timedelta
 from itertools import product
 from pathlib import Path
-from datetime import datetime, timedelta
+from queue import Empty
+from typing import Dict, Iterator
 
-import pandas as pd
 import numpy as np
 from astropy.table import Table
 from pyvo.utils.http import create_session
 
-from .stable_tap import StableTAPService
 from ..backend import BackendType
-from ..types import TAPJobMeta, TaskID, TYPE_MAP
+from ..chunking import Chunk, Chunker
 from ..query import QueryType
-from ..query.base import Query
+from ..types import TYPE_MAP, TAPJobMeta, TaskID
 from ..util.error_threading import ErrorQueue, ExceptionSafeThread
-from ..chunking import Chunker, Chunk
-
+from .stable_tap import StableTAPService
 
 logger = logging.getLogger(__name__)
 
@@ -34,6 +31,7 @@ class Downloader:
         queries: list[QueryType],
         max_concurrent_jobs: int,
         poll_interval: float,
+        resubmit_failed: bool,
     ):
         self.backend = backend
         self.queries = queries
@@ -67,6 +65,7 @@ class Downloader:
         self.service: StableTAPService = StableTAPService(
             service_url, session=self.session
         )
+        self.resubmit_failed = resubmit_failed
 
         self.chunker = Chunker(input_csv=input_csv, chunk_size=chunk_size)
 
@@ -74,7 +73,7 @@ class Downloader:
     # helpers
     # ----------------------------
     @staticmethod
-    def get_task_id(chunk: Chunk, query: Query) -> TaskID:
+    def get_task_id(chunk: Chunk, query: QueryType) -> TaskID:
         return TaskID(
             namespace="download", key=f"chunk{chunk.chunk_id:04d}_{query.hash}"
         )
@@ -107,7 +106,7 @@ class Downloader:
     # TAP submission and download
     # ----------------------------
 
-    def submit_tap_job(self, query: Query, chunk: Chunk) -> TAPJobMeta:
+    def submit_tap_job(self, query: QueryType, chunk: Chunk) -> TAPJobMeta:
         adql = query.adql
         chunk_df = chunk.data
 
@@ -133,7 +132,6 @@ class Downloader:
         logger.debug(f"uploading {len(upload)} objects.")
         job = self.service.submit_job(adql, uploads={query.upload_name: upload})
         job.run()
-        logger.debug(job.url)
 
         return TAPJobMeta(
             url=job.url,
@@ -163,7 +161,7 @@ class Downloader:
     def _submission_worker(self):
         while not self.stop_event.is_set():
             try:
-                chunk, query = self.submit_queue.get(timeout=1.0)  # type: Chunk, Query
+                chunk, query = self.submit_queue.get(timeout=1.0)  # type: Chunk, QueryType
             except Empty:
                 if self.all_chunks_queued:
                     self.all_chunks_submitted = True
@@ -194,6 +192,26 @@ class Downloader:
     # ----------------------------
     # Polling thread
     # ----------------------------
+
+    def resubmit(self, resubmit_task: TaskID):
+        logger.info(f"resubmitting {resubmit_task}")
+        submit = None
+        for chunk, q in product(self.chunker, self.queries):
+            task = self.get_task_id(chunk, q)
+            if task == resubmit_task:
+                submit = chunk, q
+                break
+        if submit is None:
+            raise RuntimeError(f"resubmit task {resubmit_task} not found!")
+
+        # remove current info, so the job won't be re-submitted over and over again
+        self.backend.drop_meta(resubmit_task)
+        with self.job_lock:
+            self.jobs.pop(resubmit_task)
+
+        # put task back in resubmit queue
+        self.submit_queue.put(submit)
+
     def _polling_worker(self):
         logger.debug("starting polling worker")
         backend = self.backend
@@ -225,6 +243,9 @@ class Downloader:
                         f"No job found under {meta['url']} for {task}! "
                         f"Probably took too long before downloading results."
                     )
+                    if self.resubmit_failed:
+                        self.resubmit(task)
+                        continue
 
                 meta["status"] = status
                 with self.job_lock:

{timewise-1.0.0a8 → timewise-1.0.0a10}/timewise/io/stable_tap.py

@@ -5,9 +5,6 @@ from xml.etree import ElementTree
 
 import requests
 
-from timewise.util.backoff import backoff_hndlr
-
-
 logger = logging.getLogger(__name__)
 
 
@@ -26,7 +23,6 @@ class StableAsyncTAPJob(vo.dal.AsyncTAPJob):
         backoff.expo,
         requests.exceptions.HTTPError,
         max_tries=5,
-        on_backoff=backoff_hndlr,
     )
     def create(
         cls,
@@ -92,7 +88,6 @@ class StableAsyncTAPJob(vo.dal.AsyncTAPJob):
         backoff.expo,
         (vo.dal.DALServiceError, AttributeError),
         max_tries=50,
-        on_backoff=backoff_hndlr,
     )
     def phase(self):
         return super(StableAsyncTAPJob, self).phase
@@ -100,11 +95,32 @@ class StableAsyncTAPJob(vo.dal.AsyncTAPJob):
     @backoff.on_exception(
         backoff.expo,
         vo.dal.DALServiceError,
-        max_tries=50,
-        on_backoff=backoff_hndlr,
+        max_tries=5,
     )
-    def _update(self, *args, **kwargs):
-        return super(StableAsyncTAPJob, self)._update(*args, **kwargs)
+    def _update(self, wait_for_statechange=False, timeout=60.0):
+        n_tries = 0
+        max_tries = 10
+        while n_tries < max_tries:
+            try:
+                res = super(StableAsyncTAPJob, self)._update(
+                    wait_for_statechange=wait_for_statechange,
+                    timeout=timeout * (1 + n_tries),
+                )
+            except vo.dal.DALServiceError as e:
+                if "Read timed out" in str(e):
+                    logger.debug(
+                        f"{self.url} timed out after {timeout * (1 + n_tries):.0f}s"
+                    )
+                    n_tries += 1
+                    continue
+                else:
+                    raise e
+
+            return res
+
+        raise vo.dal.DALServiceError(
+            f"No success after {max_tries} tries for {self.url}!"
+        )
 
 
 class StableTAPService(vo.dal.TAPService):
@@ -116,7 +132,6 @@ class StableTAPService(vo.dal.TAPService):
         backoff.expo,
         (vo.dal.DALServiceError, AttributeError, AssertionError),
         max_tries=5,
-        on_backoff=backoff_hndlr,
     )
     def submit_job(
         self, query, *, language="ADQL", maxrec=None, uploads=None, **keywords
@@ -136,3 +151,15 @@ class StableTAPService(vo.dal.TAPService):
 
     def get_job_from_url(self, url):
         return StableAsyncTAPJob(url, session=self._session)
+
+    @backoff.on_exception(
+        backoff.expo,
+        (vo.dal.DALServiceError, vo.dal.DALFormatError),
+        max_tries=5,
+    )
+    def run_sync(
+        self, query, *, language="ADQL", maxrec=None, uploads=None, **keywords
+    ):
+        return super().run_sync(
+            query, language=language, maxrec=maxrec, uploads=uploads, **keywords
+        )

{timewise-1.0.0a8 → timewise-1.0.0a10}/timewise/plot/sdss.py

@@ -5,8 +5,6 @@ import logging
 import matplotlib.pyplot as plt
 import backoff
 
-from ..util.backoff import backoff_hndlr
-
 
 logger = logging.getLogger(__name__)
 
@@ -34,7 +32,7 @@ def login_to_sciserver():
 
 
 @backoff.on_exception(
-    backoff.expo, requests.RequestException, max_tries=50, on_backoff=backoff_hndlr
+    backoff.expo, requests.RequestException, max_tries=50
 )
 def get_cutout(*args, **kwargs):
     login_to_sciserver()

{timewise-1.0.0a8 → timewise-1.0.0a10}/timewise/process/interface.py

@@ -10,6 +10,8 @@ from pymongo import MongoClient, ASCENDING
 from pymongo.collection import Collection
 from pymongo.database import Database
 
+from ..util.path import expand
+
 if find_spec("ampel.core"):
     AMPEL_EXISTS = True
     from ampel.cli.JobCommand import JobCommand
@@ -40,6 +42,10 @@ class AmpelInterface:
         self.template_path = Path(template_path)
         self.uri = uri
 
+    @property
+    def expanded_input_csv(self) -> Path:
+        return expand(self.input_csv)
+
     def import_input(self):
         # if collection already exists, assume import was already done
         if "input" in self.client[self.input_mongo_db_name].list_collection_names():
@@ -48,12 +54,12 @@ class AmpelInterface:
             )
             return
 
-        logger.debug(f"importing {self.input_csv} into {self.input_mongo_db_name}")
+        logger.debug(f"importing {self.expanded_input_csv} into {self.input_mongo_db_name}")
         col = self.client[self.input_mongo_db_name]["input"]
 
         # create an index from stock id
         col.create_index([(self.orig_id_key, ASCENDING)], unique=True)
-        col.insert_many(pd.read_csv(self.input_csv).to_dict(orient="records"))
+        col.insert_many(pd.read_csv(self.expanded_input_csv).to_dict(orient="records"))
 
     def make_ampel_job_file(self, cfg_path: Path) -> Path:
         logger.debug(f"loading ampel job template from {self.template_path}")

{timewise-1.0.0a8 → timewise-1.0.0a10}/timewise/process/stacking.py

@@ -146,7 +146,7 @@ def calculate_epochs(
     bias_correction_function = CORRECTION_FUNCTIONS[correction_name]
 
     one_points_mask = None
-    visits_at_least_two_point = []
+    visits_at_least_two_point: npt.NDArray[np.generic] = np.array([])
 
     while n_remaining_outlier > 0:
         # make a mask of values to use

timewise-1.0.0a10/timewise/query/__init__.py

@@ -0,0 +1,11 @@
+from typing import Annotated, TypeAlias, Union
+
+from pydantic import Field
+
+from .by_allwise_cntr_and_position import AllWISECntrQuery
+from .positional import PositionalQuery
+
+# Discriminated union of all query types
+QueryType: TypeAlias = Annotated[
+    Union[PositionalQuery, AllWISECntrQuery], Field(discriminator="type")
+]

timewise-1.0.0a10/timewise/query/by_allwise_cntr_and_position.py

@@ -0,0 +1,49 @@
+import logging
+from typing import Dict, Literal
+
+from .base import Query
+
+logger = logging.getLogger(__name__)
+
+
+class AllWISECntrQuery(Query):
+    type: Literal["by_allwise_cntr_and_position"] = "by_allwise_cntr_and_position"
+    radius_arcsec: float
+
+    @property
+    def input_columns(self) -> Dict[str, str]:
+        return {
+            "allwise_cntr": "int",
+            "ra": "float",
+            "dec": "float",
+            self.original_id_key: "int",
+        }
+
+    def build(self) -> str:
+        logger.debug(f"constructing query by AllWISE cntr for {self.table.name}")
+
+        q = "SELECT \n\t"
+        for k in self.columns:
+            q += f"{self.table.name}.{k}, "
+        q += f"\n\tmine.{self.original_id_key} \n"
+        q += f"FROM\n\tTAP_UPLOAD.{self.upload_name} AS mine \n"
+        q += f"RIGHT JOIN\n\t{self.table.name} \n"
+        q += "WHERE \n"
+        q += (
+            f"\tCONTAINS(POINT('J2000',{self.table.name}.{self.table.ra_column},{self.table.name}.{self.table.dec_column}),"
+            f"CIRCLE('J2000',mine.ra,mine.dec,{self.radius_arcsec / 3600:.18f}))=1 "
+        )
+
+        constraints = self.constraints + [
+            f"{self.table.allwise_cntr_column} = {self.upload_name}.allwise_cntr"
+        ]
+
+        if len(constraints) > 0:
+            q += " AND (\n"
+            for c in constraints:
+                q += f"\t{self.table.name}.{c} AND \n"
+            q = q.strip(" AND \n")
+            q += "\t)"
+
+        logger.debug(f"\n{q}")
+        return q

{timewise-1.0.0a8 → timewise-1.0.0a10}/timewise/query/positional.py

@@ -36,5 +36,4 @@ class PositionalQuery(Query):
             q = q.strip(" AND \n")
             q += "\t)"
 
-        logger.debug(f"\n{q}")
         return q

timewise-1.0.0a10/timewise/tables/__init__.py

@@ -0,0 +1,11 @@
+from typing import Annotated, Union
+
+from pydantic import Field
+
+from .allwise_p3as_mep import allwise_p3as_mep
+from .allwise_p3as_psd import allwise_p3as_psd
+from .neowiser_p1bs_psd import neowiser_p1bs_psd
+
+TableType = Annotated[
+    Union[allwise_p3as_mep, neowiser_p1bs_psd, allwise_p3as_psd], Field(discriminator="name")
+]

{timewise-1.0.0a8 → timewise-1.0.0a10}/timewise/tables/allwise_p3as_mep.py

@@ -1,4 +1,5 @@
-from typing import Literal, ClassVar, Type, Dict
+from typing import ClassVar, Dict, Literal, Type
+
 from .base import TableConfig
 
 
@@ -20,3 +21,4 @@ class allwise_p3as_mep(TableConfig):
     }
     ra_column: ClassVar[str] = "ra"
     dec_column: ClassVar[str] = "dec"
+    allwise_cntr_column: ClassVar[str] = "cntr_mf"

timewise-1.0.0a10/timewise/tables/allwise_p3as_psd.py

@@ -0,0 +1,24 @@
+from typing import ClassVar, Dict, Literal, Type
+
+from .base import TableConfig
+
+
+class allwise_p3as_psd(TableConfig):
+    name: Literal["allwise_p3as_psd"] = "allwise_p3as_psd"
+    columns_dtypes: ClassVar[Dict[str, Type]] = {
+        "ra": float,
+        "dec": float,
+        "mjd": float,
+        "cntr": str,
+        "w1mpro": float,
+        "w1sigmpro": float,
+        "w2mpro": float,
+        "w2sigmpro": float,
+        "w1flux": float,
+        "w1sigflux": float,
+        "w2flux": float,
+        "w2sigflux": float,
+    }
+    ra_column: ClassVar[str] = "ra"
+    dec_column: ClassVar[str] = "dec"
+    allwise_cntr_column: ClassVar[str] = "cntr"

{timewise-1.0.0a8 → timewise-1.0.0a10}/timewise/tables/neowiser_p1bs_psd.py

@@ -1,4 +1,5 @@
-from typing import Literal, ClassVar, Dict, Type
+from typing import ClassVar, Dict, Literal, Type
+
 from .base import TableConfig
 
 
@@ -20,3 +21,4 @@ class neowiser_p1bs_psd(TableConfig):
     }
     ra_column: ClassVar[str] = "ra"
     dec_column: ClassVar[str] = "dec"
+    allwise_cntr_column: ClassVar[str] = "allwise_cntr"

timewise-1.0.0a8/timewise/__init__.py

@@ -1 +0,0 @@
-__version__ = "1.0.0a8"

timewise-1.0.0a8/timewise/query/__init__.py

@@ -1,6 +0,0 @@
-from pydantic import Field
-from typing import Union, Annotated, TypeAlias
-from .positional import PositionalQuery
-
-# Discriminated union of all query types
-QueryType: TypeAlias = Annotated[Union[PositionalQuery], Field(discriminator="type")]

timewise-1.0.0a8/timewise/tables/__init__.py

@@ -1,10 +0,0 @@
-from pydantic import Field
-from typing import Union, Annotated
-
-from .allwise_p3as_mep import allwise_p3as_mep
-from .neowiser_p1bs_psd import neowiser_p1bs_psd
-
-
-TableType = Annotated[
-    Union[allwise_p3as_mep, neowiser_p1bs_psd], Field(discriminator="name")
-]

timewise-1.0.0a8/timewise/util/backoff.py

@@ -1,12 +0,0 @@
-import logging
-
-
-logger = logging.getLogger(__name__)
-
-
-def backoff_hndlr(details):
-    logger.info(
-        "Backing off {wait:0.1f} seconds after {tries} tries "
-        "calling function {target} with args {args} and kwargs "
-        "{kwargs}".format(**details)
-    )