timewise 1.0.0a7__tar.gz → 1.0.0a9__tar.gz

{timewise-1.0.0a7 → timewise-1.0.0a9}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: timewise
-Version: 1.0.0a7
+Version: 1.0.0a9
 Summary: Download WISE infrared data for many objects and process them with AMPEL
 License: MIT
 License-File: LICENSE
@@ -27,6 +27,7 @@ Requires-Dist: coveralls (>=3.3.1,<4.0.0) ; extra == "dev"
 Requires-Dist: jupyter[jupyter] (>=1.0.0,<2.0.0)
 Requires-Dist: jupyterlab[jupyter] (>=4.0.6,<5.0.0)
 Requires-Dist: matplotlib (>=3.5.3,<4.0.0)
+Requires-Dist: mongomock (>=4.3.0,<5.0.0) ; extra == "dev"
 Requires-Dist: mypy (>=1.18.2,<2.0.0) ; extra == "dev"
 Requires-Dist: myst-parser (>=1,<3) ; extra == "docs"
 Requires-Dist: numpy (>=1.23.2,<2.0.0)
@@ -63,7 +64,7 @@ Description-Content-Type: text/markdown
 This package downloads WISE data for positions on the sky and stacks single-exposure photometry per visit
 
 ## Prerequisites
-Python version 3.11.
+Python version 3.11, 3.12 or 3.13.
 
 If you want to not only download individual exposure photometry but also stack detections per visit (see below),
 you must have access to a running [MongoDB](https://www.mongodb.com/)*. 
@@ -77,12 +78,12 @@ to get the MongoDB community edition. </sub>
 ### If you use timewise only for downloading
 The package can be installed via `pip` (but make sure to install the v1 pre-release):
 ```bash
-pip install --pre timewise==1.0.0a6
+pip install --pre timewise==1.0.0a9
 ```
 ### If you use timewise also for stacking individual exposures
 You must install with the `ampel` extra:
 ```bash
-pip install --pre 'timewise[ampel]==1.0.0a6'
+pip install --pre 'timewise[ampel]==1.0.0a9'
 ```
 To tell AMPEL which modules, aka units, to use, build the corresponding configuration file:
 ```bash

{timewise-1.0.0a7 → timewise-1.0.0a9}/README.md

@@ -10,7 +10,7 @@
 This package downloads WISE data for positions on the sky and stacks single-exposure photometry per visit
 
 ## Prerequisites
-Python version 3.11.
+Python version 3.11, 3.12 or 3.13.
 
 If you want to not only download individual exposure photometry but also stack detections per visit (see below),
 you must have access to a running [MongoDB](https://www.mongodb.com/)*. 
@@ -24,12 +24,12 @@ to get the MongoDB community edition. </sub>
 ### If you use timewise only for downloading
 The package can be installed via `pip` (but make sure to install the v1 pre-release):
 ```bash
-pip install --pre timewise==1.0.0a6
+pip install --pre timewise==1.0.0a9
 ```
 ### If you use timewise also for stacking individual exposures
 You must install with the `ampel` extra:
 ```bash
-pip install --pre 'timewise[ampel]==1.0.0a6'
+pip install --pre 'timewise[ampel]==1.0.0a9'
 ```
 To tell AMPEL which modules, aka units, to use, build the corresponding configuration file:
 ```bash

{timewise-1.0.0a7 → timewise-1.0.0a9}/ampel/timewise/alert/TimewiseAlertSupplier.py

@@ -71,8 +71,8 @@ class TimewiseAlertSupplier(BaseAlertSupplier, AmpelABC):
 
             move = {
                 c: c.replace("_ep", "")
-                for c in columns_to_rename
-                if c.replace("_ep", "") in table.columns
+                for c in table.columns
+                if (c.replace("_ep", "") in table.columns) and (c.endswith("_ep"))
             }
             if move:
                 # In this case, the columns already exists because the neowise data is present

{timewise-1.0.0a7 → timewise-1.0.0a9}/ampel/timewise/alert/load/TimewiseFileLoader.py

@@ -32,6 +32,9 @@ class TimewiseFileLoader(AbsAlertLoader[Dict], AmpelABC):
 
     chunks: list[int] | None = None
 
+    # optionally skip files that are missing
+    skip_missing_files: bool = False
+
     def __init__(self, **kwargs) -> None:
         super().__init__(**kwargs)
 
@@ -81,7 +84,14 @@ class TimewiseFileLoader(AbsAlertLoader[Dict], AmpelABC):
             data = []
             for task in tasks:
                 self.logger.debug(f"reading {task}")
-                idata = backend.load_data(task)
+                try:
+                    idata = backend.load_data(task)
+                except FileNotFoundError as e:
+                    if self.skip_missing_files:
+                        self.logger.warn(f"file for task {task} not found, skipping...")
+                        continue
+                    else:
+                        raise e
 
                 # add table name
                 idata["table_name"] = (

{timewise-1.0.0a7 → timewise-1.0.0a9}/ampel/timewise/ingest/TiMongoMuxer.py

@@ -8,13 +8,20 @@
 
 from bisect import bisect_right
 from contextlib import suppress
-from typing import Any
-
+from typing import Any, Sequence
 
 from ampel.abstract.AbsT0Muxer import AbsT0Muxer
 from ampel.content.DataPoint import DataPoint
-from ampel.types import DataPointId, StockId
+from ampel.model.operator.AllOf import AllOf
+from ampel.model.operator.AnyOf import AnyOf
+from ampel.types import ChannelId, DataPointId, StockId
 from ampel.util.mappings import unflatten_dict
+from astropy.table import Table
+from pydantic import TypeAdapter
+from timewise.io.stable_tap import StableTAPService
+from timewise.query import QueryType
+from timewise.tables.allwise_p3as_mep import allwise_p3as_mep
+from timewise.types import TYPE_MAP
 
 
 class ConcurrentUpdateError(Exception):
@@ -32,7 +39,7 @@ class TiMongoMuxer(AbsT0Muxer):
     """
 
     # Standard projection used when checking DB for existing PPS/ULS
-    projection = {
+    projection: dict[str, int] = {
         "_id": 0,
         "id": 1,
         "tag": 1,
@@ -51,6 +58,13 @@ class TiMongoMuxer(AbsT0Muxer):
         "body.dec": 1,
     }
 
+    channel: None | ChannelId | AnyOf[ChannelId] | AllOf[ChannelId] = None
+
+    unique_key: list[str] = ["mjd", "ra", "dec"]
+
+    # URL of tap service for query of AllWISE Source Table
+    tap_service_url: str = "https://irsa.ipac.caltech.edu/TAP"
+
     def __init__(self, **kwargs) -> None:
         super().__init__(**kwargs)
 
@@ -58,6 +72,11 @@ class TiMongoMuxer(AbsT0Muxer):
         self._photo_col = self.context.db.get_collection("t0")
         self._projection_spec = unflatten_dict(self.projection)
 
+        self._tap_service = StableTAPService(self.tap_service_url)
+
+        self._allwise_source_cntr: list[str] = []
+        self._not_allwise_source_cntr: list[str] = []
+
     def process(
         self, dps: list[DataPoint], stock_id: None | StockId = None
     ) -> tuple[None | list[DataPoint], None | list[DataPoint]]:
@@ -79,7 +98,76 @@ class TiMongoMuxer(AbsT0Muxer):
 
     # NB: this 1-liner is a separate method to provide a patch point for race condition testing
     def _get_dps(self, stock_id: None | StockId) -> list[DataPoint]:
-        return list(self._photo_col.find({"stock": stock_id}, self.projection))
+        if self.channel is not None:
+            if isinstance(self.channel, ChannelId):
+                channel_query: (
+                    ChannelId | dict[str, Sequence[ChannelId | AllOf[ChannelId]]]
+                ) = self.channel
+            elif isinstance(self.channel, AnyOf):
+                channel_query = {"$in": self.channel.any_of}
+            elif isinstance(self.channel, AllOf):
+                channel_query = {"$all": self.channel.all_of}
+            else:
+                # should not happen
+                raise TypeError()
+            _channel = {"channel": channel_query}
+        else:
+            _channel = {}
+        query = {"stock": stock_id, **_channel}
+        return list(self._photo_col.find(query, self.projection))
+
+    def _check_cntrs(self, dps: Sequence[DataPoint]) -> None:
+        # assemble query
+        query_config = {
+            "type": "by_allwise_cntr_and_position",
+            "radius_arcsec": 10,
+            "columns": ["cntr"],
+            "constraints": [],
+            "table": {"name": "allwise_p3as_psd"},
+        }
+        query: QueryType = TypeAdapter(QueryType).validate_python(query_config)
+
+        # load datapoints into astropy table
+        upload = Table([dp["body"] for dp in dps])
+        upload["allwise_cntr"] = upload[allwise_p3as_mep.allwise_cntr_column]
+        upload[query.original_id_key] = [dp["id"] for dp in dps]
+        for key, dtype in query.input_columns.items():
+            upload[key] = upload[key].astype(TYPE_MAP[dtype])
+        for key in upload.colnames:
+            if key not in query.input_columns:
+                upload.remove_column(key)
+
+        # run query
+        self.logger.info("Querying AllWISE Source Table for MEP CNTRs ...")
+        res = self._tap_service.run_sync(
+            query.adql, uploads={query.upload_name: upload}
+        )
+
+        # update internal state
+        res_cntr = res.to_table()["cntr"].astype(str)
+        self._allwise_source_cntr.extend(list(res_cntr))
+        self._not_allwise_source_cntr.extend(
+            list(set(upload["allwise_cntr"].astype(str)) - set(res_cntr))
+        )
+
+    def _check_mep_allwise_sources(self, dps: Sequence[DataPoint]) -> list[DataPointId]:
+        dps_with_unchecked_cntr = [
+            dp
+            for dp in dps
+            if str(dp["body"][allwise_p3as_mep.allwise_cntr_column])
+            not in self._allwise_source_cntr + self._not_allwise_source_cntr
+        ]
+        if len(dps_with_unchecked_cntr) > 0:
+            self._check_cntrs(dps_with_unchecked_cntr)
+
+        # compile list of invalid datapoint ids
+        invalid_dp_ids = []
+        for dp in dps:
+            cntr = str(dp["body"][allwise_p3as_mep.allwise_cntr_column])
+            if cntr in self._not_allwise_source_cntr:
+                invalid_dp_ids.append(dp["id"])
+
+        return invalid_dp_ids
 
     def _process(
         self, dps: list[DataPoint], stock_id: None | StockId = None
@@ -116,11 +204,7 @@ class TiMongoMuxer(AbsT0Muxer):
             # jd alone is not enough for matching pps because each time is associated with
             # two filters! Also, if there can be multiple sources within the same frame which
             # leads to duplicate MJD and FID. Check position in addition.
-            key = (
-                dp["body"]["mjd"],
-                dp["body"]["ra"],
-                dp["body"]["dec"],
-            )
+            key = tuple(dp["body"][k] for k in self.unique_key)
 
             if target := unique_dps_ids.get(key):
                 # insert id in order
@@ -130,7 +214,10 @@ class TiMongoMuxer(AbsT0Muxer):
             else:
                 unique_dps_ids[key] = [dp["id"]]
 
-        # make sure no duplicate datapoints exist
+        # Part 2: Check that there are no duplicates and handle redundant AllWISE MEP data
+        ##################################################################################
+
+        invalid_dp_ids = []
         for key, simultaneous_dps in unique_dps_ids.items():
             dps_db_wrong = [dp for dp in dps_db if dp["id"] in simultaneous_dps]
             dps_wrong = [dp for dp in dps if dp["id"] in simultaneous_dps]
@@ -138,20 +225,47 @@ class TiMongoMuxer(AbsT0Muxer):
                 f"stockID {str(stock_id)}: Duplicate photopoints at {key}!\nDPS from DB:"
                 f"\n{dps_db_wrong}\nNew DPS:\n{dps_wrong}"
             )
-            assert len(simultaneous_dps) == 1, msg
 
-        # Part 2: Update new data points that are already superseded
-        ############################################################
+            all_wrong_dps = dps_db_wrong + dps_wrong
+            if len(simultaneous_dps) > 1:
+                # if these datapoints come from the AllWISE MEP database, downloaded by timewise
+                # there can be duplicates. Only the AllWISE CNTR can tell us which datapoints
+                # should be used: the CNTR that appears in the AllWISE source catalog.
+                if all(
+                    [
+                        ("TIMEWISE" in dp["tag"]) and ("allwise_p3as_mep" in dp["tag"])
+                        for dp in all_wrong_dps
+                    ]
+                ):
+                    self.logger.info(
+                        f"{len(all_wrong_dps)} duplicate MEP datapoints found. Checking ..."
+                    )
+                    i_invalid_dp_ids = self._check_mep_allwise_sources(
+                        dps_db_wrong + dps_wrong
+                    )
+                    self.logger.info(
+                        f"Found {len(i_invalid_dp_ids)} invalid MEP datapoints."
+                    )
+                    invalid_dp_ids.extend(i_invalid_dp_ids)
+
+                else:
+                    raise RuntimeError(msg)
+
+        # Part 3: Compile final lists of datapoints to insert and combine
+        #################################################################
 
         # Difference between candids from the alert and candids present in DB
-        ids_dps_to_insert = ids_dps_alert - ids_dps_db
+        ids_dps_to_insert = ids_dps_alert - ids_dps_db - set(invalid_dp_ids)
         dps_to_insert = [dp for dp in dps if dp["id"] in ids_dps_to_insert]
         dps_to_combine = [
-            dp for dp in dps + dps_db if dp["id"] in ids_dps_alert | ids_dps_db
+            dp
+            for dp in dps + dps_db
+            if dp["id"] in ((ids_dps_alert | ids_dps_db) - set(invalid_dp_ids))
         ]
         self.logger.debug(
             f"Got {len(ids_dps_alert)} datapoints from alerts, "
             f"found {len(dps_db)} in DB, "
+            f"{len(invalid_dp_ids)} invalid datapoints, "
             f"inserting {len(dps_to_insert)} datapoints, "
             f"combining {len(dps_to_combine)} datapoints"
         )

{timewise-1.0.0a7 → timewise-1.0.0a9}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "poetry.core.masonry.api"
 
 [project]
 name = "timewise"
-version = "1.0.0a7"
+version = "1.0.0a9"
 description = "Download WISE infrared data for many objects and process them with AMPEL"
 authors = [
     { name = "Jannis Necker", email = "jannis.necker@gmail.com" },
@@ -54,6 +54,7 @@ dev = [
     "scipy-stubs (>=1.16.2.0,<2.0.0.0)",
     "types-pyyaml (>=6.0.12.20250915,<7.0.0.0)",
     "types-requests (>=2.32.4.20250913,<3.0.0.0)",
+    "mongomock (>=4.3.0,<5.0.0)",
 ]
 docs = [
     "myst-parser>=1,<3",

timewise-1.0.0a9/timewise/__init__.py

@@ -0,0 +1 @@
+__version__ = "1.0.0a9"

{timewise-1.0.0a7 → timewise-1.0.0a9}/timewise/backend/base.py

@@ -20,6 +20,8 @@ class Backend(abc.ABC, BaseModel):
     def save_meta(self, task: TaskID, meta: dict[str, Any]) -> None: ...
     @abc.abstractmethod
     def load_meta(self, task: TaskID) -> dict[str, Any] | None: ...
+    @abc.abstractmethod
+    def drop_meta(self, task: TaskID) -> None: ...
 
     # --- Markers ---
     @abc.abstractmethod

{timewise-1.0.0a7 → timewise-1.0.0a9}/timewise/backend/filesystem.py

@@ -52,6 +52,9 @@ class FileSystemBackend(Backend):
     def meta_exists(self, task: TaskID) -> bool:
         return self._meta_path(task).exists()
 
+    def drop_meta(self, task: TaskID) -> None:
+        self._meta_path(task).unlink()
+
     # ----------------------------
     # Markers
     # ----------------------------

{timewise-1.0.0a7 → timewise-1.0.0a9}/timewise/chunking.py

@@ -1,7 +1,7 @@
 import logging
 from functools import cached_property
 from pathlib import Path
-from typing import Iterator
+from typing import Iterator, Sequence, cast
 
 import numpy as np
 import pandas as pd
@@ -24,6 +24,23 @@ class Chunk:
         stop = max(self.row_numbers) + 1
         return pd.read_csv(self.input_csv, skiprows=start, nrows=stop - start).index
 
+    @property
+    def data(self) -> pd.DataFrame:
+        start = (
+            min(cast(Sequence[int], self.row_numbers)) + 1
+        )  # plus one to always skip header line
+        nrows = (
+            max(cast(Sequence[int], self.row_numbers)) - start + 2
+        )  # plus one: skip header, plus one:
+
+        columns = list(pd.read_csv(self.input_csv, nrows=0).columns)
+        return pd.read_csv(
+            filepath_or_buffer=self.input_csv,
+            skiprows=start,
+            nrows=nrows,
+            names=columns,
+        )
+
 
 class Chunker:
     def __init__(self, input_csv: Path, chunk_size: int):

{timewise-1.0.0a7 → timewise-1.0.0a9}/timewise/cli.py

@@ -53,8 +53,16 @@ def main(
 @app.command(help="Download WISE photometry from IRSA")
 def download(
     config_path: config_path_type,
+    resubmit_failed: Annotated[
+        bool,
+        typer.Option(
+            help="Re-submit jobs when failed due to connection issues",
+        ),
+    ] = False,
 ):
-    TimewiseConfig.from_yaml(config_path).download.build_downloader().run()
+    TimewiseConfig.from_yaml(config_path).download.build_downloader(
+        resubmit_failed=resubmit_failed
+    ).run()
 
 
 # the following commands will only be added if ampel is installed

{timewise-1.0.0a7 → timewise-1.0.0a9}/timewise/io/config.py

@@ -17,6 +17,7 @@ class DownloadConfig(BaseModel):
     poll_interval: float = 10.0
     queries: List[QueryType] = Field(..., description="One or more queries per chunk")
     backend: BackendType = Field(..., discriminator="type")
+    resubmit_failed: bool = False
 
     service_url: str = "https://irsa.ipac.caltech.edu/TAP"
 
@@ -57,13 +58,16 @@ class DownloadConfig(BaseModel):
 
         return self
 
-    def build_downloader(self) -> Downloader:
-        return Downloader(
-            service_url=self.service_url,
-            input_csv=self.expanded_input_csv,
-            chunk_size=self.chunk_size,
-            backend=self.backend,
-            queries=self.queries,
-            max_concurrent_jobs=self.max_concurrent_jobs,
-            poll_interval=self.poll_interval,
-        )
+    def build_downloader(self, **overwrite) -> Downloader:
+        default = {
+            "service_url": self.service_url,
+            "input_csv": self.expanded_input_csv,
+            "chunk_size": self.chunk_size,
+            "backend": self.backend,
+            "queries": self.queries,
+            "max_concurrent_jobs": self.max_concurrent_jobs,
+            "poll_interval": self.poll_interval,
+            "resubmit_failed": self.resubmit_failed,
+        }
+        default.update(overwrite)
+        return Downloader(**default)  # type: ignore

{timewise-1.0.0a7 → timewise-1.0.0a9}/timewise/io/download.py

@@ -1,25 +1,22 @@
-import time
-import threading
 import logging
-from queue import Empty
-from typing import Dict, Iterator, cast, Sequence
+import threading
+import time
+from datetime import datetime, timedelta
 from itertools import product
 from pathlib import Path
-from datetime import datetime, timedelta
+from queue import Empty
+from typing import Dict, Iterator
 
-import pandas as pd
 import numpy as np
 from astropy.table import Table
 from pyvo.utils.http import create_session
 
-from .stable_tap import StableTAPService
 from ..backend import BackendType
-from ..types import TAPJobMeta, TaskID, TYPE_MAP
+from ..chunking import Chunk, Chunker
 from ..query import QueryType
-from ..query.base import Query
+from ..types import TYPE_MAP, TAPJobMeta, TaskID
 from ..util.error_threading import ErrorQueue, ExceptionSafeThread
-from ..chunking import Chunker, Chunk
-
+from .stable_tap import StableTAPService
 
 logger = logging.getLogger(__name__)
 
@@ -34,6 +31,7 @@ class Downloader:
         queries: list[QueryType],
         max_concurrent_jobs: int,
         poll_interval: float,
+        resubmit_failed: bool,
     ):
         self.backend = backend
         self.queries = queries
@@ -67,6 +65,7 @@ class Downloader:
         self.service: StableTAPService = StableTAPService(
             service_url, session=self.session
         )
+        self.resubmit_failed = resubmit_failed
 
         self.chunker = Chunker(input_csv=input_csv, chunk_size=chunk_size)
 
@@ -74,7 +73,7 @@ class Downloader:
     # helpers
     # ----------------------------
     @staticmethod
-    def get_task_id(chunk: Chunk, query: Query) -> TaskID:
+    def get_task_id(chunk: Chunk, query: QueryType) -> TaskID:
         return TaskID(
             namespace="download", key=f"chunk{chunk.chunk_id:04d}_{query.hash}"
         )
@@ -106,25 +105,10 @@ class Downloader:
     # ----------------------------
     # TAP submission and download
     # ----------------------------
-    def get_chunk_data(self, chunk: Chunk) -> pd.DataFrame:
-        start = (
-            min(cast(Sequence[int], chunk.row_numbers)) + 1
-        )  # plus one to always skip header line
-        nrows = (
-            max(cast(Sequence[int], chunk.row_numbers)) - start + 2
-        )  # plus one: skip header, plus one:
-
-        columns = list(pd.read_csv(self.input_csv, nrows=0).columns)
-        return pd.read_csv(
-            filepath_or_buffer=self.input_csv,
-            skiprows=start,
-            nrows=nrows,
-            names=columns,
-        )
 
-    def submit_tap_job(self, query: Query, chunk: Chunk) -> TAPJobMeta:
+    def submit_tap_job(self, query: QueryType, chunk: Chunk) -> TAPJobMeta:
         adql = query.adql
-        chunk_df = self.get_chunk_data(chunk)
+        chunk_df = chunk.data
 
         assert all(chunk_df.index.isin(chunk.indices)), (
             "Some inputs loaded from wrong chunk!"
@@ -148,7 +132,6 @@ class Downloader:
         logger.debug(f"uploading {len(upload)} objects.")
         job = self.service.submit_job(adql, uploads={query.upload_name: upload})
         job.run()
-        logger.debug(job.url)
 
         return TAPJobMeta(
             url=job.url,
@@ -178,7 +161,7 @@ class Downloader:
     def _submission_worker(self):
         while not self.stop_event.is_set():
             try:
-                chunk, query = self.submit_queue.get(timeout=1.0)  # type: Chunk, Query
+                chunk, query = self.submit_queue.get(timeout=1.0)  # type: Chunk, QueryType
             except Empty:
                 if self.all_chunks_queued:
                     self.all_chunks_submitted = True
@@ -209,6 +192,26 @@ class Downloader:
     # ----------------------------
     # Polling thread
     # ----------------------------
+
+    def resubmit(self, resubmit_task: TaskID):
+        logger.info(f"resubmitting {resubmit_task}")
+        submit = None
+        for chunk, q in product(self.chunker, self.queries):
+            task = self.get_task_id(chunk, q)
+            if task == resubmit_task:
+                submit = chunk, q
+                break
+        if submit is None:
+            raise RuntimeError(f"resubmit task {resubmit_task} not found!")
+
+        # remove current info, so the job won't be re-submitted over and over again
+        self.backend.drop_meta(resubmit_task)
+        with self.job_lock:
+            self.jobs.pop(resubmit_task)
+
+        # put task back in resubmit queue
+        self.submit_queue.put(submit)
+
     def _polling_worker(self):
         logger.debug("starting polling worker")
         backend = self.backend
@@ -240,6 +243,9 @@ class Downloader:
                         f"No job found under {meta['url']} for {task}! "
                         f"Probably took too long before downloading results."
                     )
+                    if self.resubmit_failed:
+                        self.resubmit(task)
+                        continue
 
                 meta["status"] = status
                 with self.job_lock:

{timewise-1.0.0a7 → timewise-1.0.0a9}/timewise/io/stable_tap.py

@@ -3,8 +3,7 @@ import backoff
 import pyvo as vo
 from xml.etree import ElementTree
 
-from timewise.util.backoff import backoff_hndlr
-
+import requests
 
 logger = logging.getLogger(__name__)
 
@@ -20,6 +19,11 @@ class StableAsyncTAPJob(vo.dal.AsyncTAPJob):
         self.submit_response = None
 
     @classmethod
+    @backoff.on_exception(
+        backoff.expo,
+        requests.exceptions.HTTPError,
+        max_tries=5,
+    )
     def create(
         cls,
         baseurl,
@@ -84,11 +88,18 @@ class StableAsyncTAPJob(vo.dal.AsyncTAPJob):
         backoff.expo,
         (vo.dal.DALServiceError, AttributeError),
         max_tries=50,
-        on_backoff=backoff_hndlr,
     )
     def phase(self):
         return super(StableAsyncTAPJob, self).phase
 
+    @backoff.on_exception(
+        backoff.expo,
+        vo.dal.DALServiceError,
+        max_tries=50,
+    )
+    def _update(self, *args, **kwargs):
+        return super(StableAsyncTAPJob, self)._update(*args, **kwargs)
+
 
 class StableTAPService(vo.dal.TAPService):
     """
@@ -99,7 +110,6 @@ class StableTAPService(vo.dal.TAPService):
         backoff.expo,
         (vo.dal.DALServiceError, AttributeError, AssertionError),
         max_tries=5,
-        on_backoff=backoff_hndlr,
     )
     def submit_job(
         self, query, *, language="ADQL", maxrec=None, uploads=None, **keywords
@@ -119,3 +129,13 @@ class StableTAPService(vo.dal.TAPService):
 
     def get_job_from_url(self, url):
         return StableAsyncTAPJob(url, session=self._session)
+
+    @backoff.on_exception(
+        backoff.expo,
+        (vo.dal.DALServiceError, vo.dal.DALFormatError),
+        max_tries=5,
+    )
+    def run_sync(
+            self, query, *, language="ADQL", maxrec=None, uploads=None,
+            **keywords):
+        return super().run_sync(query, language=language, maxrec=maxrec, uploads=uploads, **keywords)

{timewise-1.0.0a7 → timewise-1.0.0a9}/timewise/plot/sdss.py

@@ -5,8 +5,6 @@ import logging
 import matplotlib.pyplot as plt
 import backoff
 
-from ..util.backoff import backoff_hndlr
-
 
 logger = logging.getLogger(__name__)
 
@@ -34,7 +32,7 @@ def login_to_sciserver():
 
 
 @backoff.on_exception(
-    backoff.expo, requests.RequestException, max_tries=50, on_backoff=backoff_hndlr
+    backoff.expo, requests.RequestException, max_tries=50
 )
 def get_cutout(*args, **kwargs):
     login_to_sciserver()

timewise-1.0.0a9/timewise/query/__init__.py

@@ -0,0 +1,11 @@
+from typing import Annotated, TypeAlias, Union
+
+from pydantic import Field
+
+from .by_allwise_cntr_and_position import AllWISECntrQuery
+from .positional import PositionalQuery
+
+# Discriminated union of all query types
+QueryType: TypeAlias = Annotated[
+    Union[PositionalQuery, AllWISECntrQuery], Field(discriminator="type")
+]

timewise-1.0.0a9/timewise/query/by_allwise_cntr_and_position.py

@@ -0,0 +1,49 @@
+import logging
+from typing import Dict, Literal
+
+from .base import Query
+
+logger = logging.getLogger(__name__)
+
+
+class AllWISECntrQuery(Query):
+    type: Literal["by_allwise_cntr_and_position"] = "by_allwise_cntr_and_position"
+    radius_arcsec: float
+
+    @property
+    def input_columns(self) -> Dict[str, str]:
+        return {
+            "allwise_cntr": "int",
+            "ra": "float",
+            "dec": "float",
+            self.original_id_key: "int",
+        }
+
+    def build(self) -> str:
+        logger.debug(f"constructing query by AllWISE cntr for {self.table.name}")
+
+        q = "SELECT \n\t"
+        for k in self.columns:
+            q += f"{self.table.name}.{k}, "
+        q += f"\n\tmine.{self.original_id_key} \n"
+        q += f"FROM\n\tTAP_UPLOAD.{self.upload_name} AS mine \n"
+        q += f"RIGHT JOIN\n\t{self.table.name} \n"
+        q += "WHERE \n"
+        q += (
+            f"\tCONTAINS(POINT('J2000',{self.table.name}.{self.table.ra_column},{self.table.name}.{self.table.dec_column}),"
+            f"CIRCLE('J2000',mine.ra,mine.dec,{self.radius_arcsec / 3600:.18f}))=1 "
+        )
+
+        constraints = self.constraints + [
+            f"{self.table.allwise_cntr_column} = {self.upload_name}.allwise_cntr"
+        ]
+
+        if len(constraints) > 0:
+            q += " AND (\n"
+            for c in constraints:
+                q += f"\t{self.table.name}.{c} AND \n"
+            q = q.strip(" AND \n")
+            q += "\t)"
+
+        logger.debug(f"\n{q}")
+        return q

{timewise-1.0.0a7 → timewise-1.0.0a9}/timewise/query/positional.py

@@ -36,5 +36,4 @@ class PositionalQuery(Query):
             q = q.strip(" AND \n")
             q += "\t)"
 
-        logger.debug(f"\n{q}")
         return q

timewise-1.0.0a9/timewise/tables/__init__.py

@@ -0,0 +1,11 @@
+from typing import Annotated, Union
+
+from pydantic import Field
+
+from .allwise_p3as_mep import allwise_p3as_mep
+from .allwise_p3as_psd import allwise_p3as_psd
+from .neowiser_p1bs_psd import neowiser_p1bs_psd
+
+TableType = Annotated[
+    Union[allwise_p3as_mep, neowiser_p1bs_psd, allwise_p3as_psd], Field(discriminator="name")
+]

{timewise-1.0.0a7 → timewise-1.0.0a9}/timewise/tables/allwise_p3as_mep.py

@@ -1,4 +1,5 @@
-from typing import Literal, ClassVar, Type, Dict
+from typing import ClassVar, Dict, Literal, Type
+
 from .base import TableConfig
 
 
@@ -20,3 +21,4 @@ class allwise_p3as_mep(TableConfig):
     }
     ra_column: ClassVar[str] = "ra"
     dec_column: ClassVar[str] = "dec"
+    allwise_cntr_column: ClassVar[str] = "cntr_mf"

timewise-1.0.0a9/timewise/tables/allwise_p3as_psd.py

@@ -0,0 +1,24 @@
+from typing import ClassVar, Dict, Literal, Type
+
+from .base import TableConfig
+
+
+class allwise_p3as_psd(TableConfig):
+    name: Literal["allwise_p3as_psd"] = "allwise_p3as_psd"
+    columns_dtypes: ClassVar[Dict[str, Type]] = {
+        "ra": float,
+        "dec": float,
+        "mjd": float,
+        "cntr": str,
+        "w1mpro": float,
+        "w1sigmpro": float,
+        "w2mpro": float,
+        "w2sigmpro": float,
+        "w1flux": float,
+        "w1sigflux": float,
+        "w2flux": float,
+        "w2sigflux": float,
+    }
+    ra_column: ClassVar[str] = "ra"
+    dec_column: ClassVar[str] = "dec"
+    allwise_cntr_column: ClassVar[str] = "cntr"

{timewise-1.0.0a7 → timewise-1.0.0a9}/timewise/tables/neowiser_p1bs_psd.py

@@ -1,4 +1,5 @@
-from typing import Literal, ClassVar, Dict, Type
+from typing import ClassVar, Dict, Literal, Type
+
 from .base import TableConfig
 
 
@@ -20,3 +21,4 @@ class neowiser_p1bs_psd(TableConfig):
     }
     ra_column: ClassVar[str] = "ra"
     dec_column: ClassVar[str] = "dec"
+    allwise_cntr_column: ClassVar[str] = "allwise_cntr"

timewise-1.0.0a7/timewise/__init__.py

@@ -1 +0,0 @@
-__version__ = "1.0.0a7"

timewise-1.0.0a7/timewise/query/__init__.py

@@ -1,6 +0,0 @@
-from pydantic import Field
-from typing import Union, Annotated, TypeAlias
-from .positional import PositionalQuery
-
-# Discriminated union of all query types
-QueryType: TypeAlias = Annotated[Union[PositionalQuery], Field(discriminator="type")]

timewise-1.0.0a7/timewise/tables/__init__.py

@@ -1,10 +0,0 @@
-from pydantic import Field
-from typing import Union, Annotated
-
-from .allwise_p3as_mep import allwise_p3as_mep
-from .neowiser_p1bs_psd import neowiser_p1bs_psd
-
-
-TableType = Annotated[
-    Union[allwise_p3as_mep, neowiser_p1bs_psd], Field(discriminator="name")
-]

timewise-1.0.0a7/timewise/util/backoff.py

@@ -1,12 +0,0 @@
-import logging
-
-
-logger = logging.getLogger(__name__)
-
-
-def backoff_hndlr(details):
-    logger.info(
-        "Backing off {wait:0.1f} seconds after {tries} tries "
-        "calling function {target} with args {args} and kwargs "
-        "{kwargs}".format(**details)
-    )