timewise 0.5.4__tar.gz → 1.0.0a1__tar.gz

timewise-1.0.0a1/PKG-INFO

@@ -0,0 +1,205 @@
+Metadata-Version: 2.4
+Name: timewise
+Version: 1.0.0a1
+Summary: Download WISE infrared data for many objects and process them with AMPEL
+License: MIT
+License-File: LICENSE
+Author: Jannis Necker
+Author-email: jannis.necker@gmail.com
+Requires-Python: >=3.11,<3.12
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.11
+Provides-Extra: dev
+Provides-Extra: docs
+Requires-Dist: ampel-alerts (==0.10.3a5)
+Requires-Dist: ampel-core (>=0.10.4.post0,<0.11.0)
+Requires-Dist: ampel-photometry (>=0.10.1,<0.11.0)
+Requires-Dist: ampel-plot (>=0.9.1,<0.10.0)
+Requires-Dist: astropy (>=5.1,<6.0.0)
+Requires-Dist: autodoc_pydantic[erdantic] (>=2.2.0,<3.0.0) ; extra == "docs"
+Requires-Dist: backoff (>=2.1.2,<3.0.0)
+Requires-Dist: coveralls (>=3.3.1,<4.0.0) ; extra == "dev"
+Requires-Dist: jupyter[jupyter] (>=1.0.0,<2.0.0)
+Requires-Dist: jupyterlab[jupyter] (>=4.0.6,<5.0.0)
+Requires-Dist: matplotlib (>=3.5.3,<4.0.0)
+Requires-Dist: mypy (>=1.18.2,<2.0.0) ; extra == "dev"
+Requires-Dist: myst-parser (>=1,<3) ; extra == "docs"
+Requires-Dist: numpy (>=1.23.2,<2.0.0)
+Requires-Dist: pandas (>=1.4.3,<3.0.0)
+Requires-Dist: pandas-stubs (>=2.3.2.250926,<3.0.0.0) ; extra == "dev"
+Requires-Dist: pydantic (>=2.0.0,<3.0.0)
+Requires-Dist: pytest (>=7.2.2,<8.0.0) ; extra == "dev"
+Requires-Dist: pyvo (>=1.7.0,<2.0.0)
+Requires-Dist: requests (>=2.28.1,<3.0.0)
+Requires-Dist: ruff (>=0.13.0,<0.14.0) ; extra == "dev"
+Requires-Dist: scikit-image (>=0.19.3,<0.22.0)
+Requires-Dist: scikit-learn (>=1.3.0,<2.0.0)
+Requires-Dist: scipy-stubs (>=1.16.2.0,<2.0.0.0) ; extra == "dev"
+Requires-Dist: seaborn (>=0.11.2,<0.14.0)
+Requires-Dist: sphinx-rtd-theme (>=1.3.0,<2.0.0) ; extra == "docs"
+Requires-Dist: tqdm (>=4.64.0,<5.0.0)
+Requires-Dist: typer (>=0.19.2,<0.20.0)
+Requires-Dist: types-pyyaml (>=6.0.12.20250915,<7.0.0.0) ; extra == "dev"
+Requires-Dist: types-requests (>=2.32.4.20250913,<3.0.0.0) ; extra == "dev"
+Requires-Dist: urllib3 (>=2.5.0,<3.0.0)
+Requires-Dist: virtualenv (>=20.16.3,<21.0.0)
+Project-URL: Bug Tracker, https://github.com/JannisNe/timewise/issues
+Project-URL: Homepage, https://github.com/JannisNe/timewise
+Description-Content-Type: text/markdown
+
+[![CI](https://github.com/JannisNe/timewise/actions/workflows/continous_integration.yml/badge.svg)](https://github.com/JannisNe/timewise/actions/workflows/continous_integration.yml)
+[![Coverage Status](https://coveralls.io/repos/github/JannisNe/timewise/badge.svg?branch=main)](https://coveralls.io/github/JannisNe/timewise?branch=main)
+[![PyPI version](https://badge.fury.io/py/timewise.svg)](https://badge.fury.io/py/timewise)
+[![DOI](https://zenodo.org/badge/449677569.svg)](https://zenodo.org/badge/latestdoi/449677569)
+
+
+![](timewise.png)
+# Infrared light curves from WISE data
+
+This package downloads WISE data for positions on the sky and stacks single-exposure photometry per visit
+
+## Prerequisites
+
+`timewise` makes use of [AMPEL](https://ampelproject.github.io/ampelastro/) and needs a running [MongoDB](https://www.mongodb.com/).
+
+## Installation
+The package can be installed via `pip`:
+```bash
+pip install timewise
+```
+
+To tell AMPEL which modules, aka units, to use, build the corresponding configuration file:
+```bash
+ampel config build -distributions ampel timewise -stop-on-errors 0 -out <path-to-ampel-config-file>
+```
+
+## Usage
+
+### Command line interface
+
+```
+ Usage: timewise [OPTIONS] COMMAND [ARGS]...                                                                
+                                                                                                            
+ Timewsie CLI                                                                                               
+                                                                                                            
+╭─ Options ────────────────────────────────────────────────────────────────────────────────────────────────╮
+│ --log-level           -l      TEXT  Logging level (DEBUG, INFO, WARNING, ERROR, CRITICAL)                │
+│                                     [default: INFO]                                                      │
+│ --install-completion                Install completion for the current shell.                            │
+│ --show-completion                   Show completion for the current shell, to copy it or customize the   │
+│                                     installation.                                                        │
+│ --help                              Show this message and exit.                                          │
+╰──────────────────────────────────────────────────────────────────────────────────────────────────────────╯
+╭─ Commands ───────────────────────────────────────────────────────────────────────────────────────────────╮
+│ download        Download WISE photometry from IRSA                                                       │
+│ prepare-ampel   Prepares the AMPEL job file so AMPEL can be run manually                                 │
+│ process         Processes the lightcurves using AMPEL                                                    │
+│ export          Write stacked lightcurves to disk                                                        │
+│ run-chain       Run download, process and export                                                         │
+│ plot            Make diagnostic plots                                                                    │
+╰──────────────────────────────────────────────────────────────────────────────────────────────────────────╯
+
+```
+
+The input is a CSV file with at least three columns:  
+- `orig_id`: an original identifier that **must** be an integer (for now)
+- `ra`, `dec`: Right Ascension and Declination
+
+
+
+`timewise` is configured with a YAML file. This is a sensible default which will use all single exposure photometry from AllWISE and NEOWISE:
+```yaml
+download:
+  input_csv: <path-to-input>
+
+  backend:
+    type: filesystem
+    base_path: <path-to-working-directory>
+
+  queries:
+    - type: positional
+      radius_arcsec: 6
+      table:
+        name: allwise_p3as_mep
+      columns:
+        - ra
+        - dec
+        - mjd
+        - cntr_mf
+        - w1mpro_ep
+        - w1sigmpro_ep
+        - w2mpro_ep
+        - w2sigmpro_ep
+        - w1flux_ep
+        - w1sigflux_ep
+        - w2flux_ep
+        - w2sigflux_ep
+
+    - type: positional
+      radius_arcsec: 6
+      table:
+        name: neowiser_p1bs_psd
+      columns:
+        - ra
+        - dec
+        - mjd
+        - allwise_cntr
+        - w1mpro
+        - w1sigmpro
+        - w2mpro
+        - w2sigmpro
+        - w1flux
+        - w1sigflux
+        - w2flux
+        - w2sigflux
+
+ampel:
+  mongo_db_name: <mongodb-name>
+```
+
+This configuration file will be the input to all subcommands. Downloading and stacking can be run together or separate.
+
+
+#### All-in-one:
+Run download, stacking, and export:
+```bash
+timewise run-chain <path-to-config-file> <path-to-ampel-config-file> <output-directory>
+```
+
+#### Separate download and processing:
+To only download the data:
+```bash
+timewise download <path-to-config-file>
+```
+
+To execute the stacking:
+```bash
+timewise process <path-to-config-file> <path-to-ampel-config-file>
+```
+
+#### Run AMPEL manually
+Prepare an AMPEL job file for stacking the single-exposure data:
+```bash
+timewise prepare-ampel <path-to-config-file>
+```
+The result will contain the path to the prepared AMPEL job file that can be run with
+```bash
+ampel job -config <path-to-ampel-config-file> -schema <path-to-ampel-job-file>
+```
+
+#### Make some diagnostic plots
+To check the datapoint selection and binning, take a quick look at the data:
+```bash
+timewise plot <path-to-config-file> <indices-to-plot> <output-directory>
+```
+
+
+## Citation
+If you use `timewise` please make sure to cite [Necker et al. A&A 695, A228 (2025)](https://www.aanda.org/articles/aa/abs/2025/03/aa51340-24/aa51340-24.html).
+Additionally, you might want to include a reference to the specific version you are using: [![DOI](https://zenodo.org/badge/449677569.svg)](https://zenodo.org/badge/latestdoi/449677569)
+
+## Difference lightcurves
+Make sure to check out `timewise-sup`, the Timewise Subtraction Pipeline: 
+[link](https://gitlab.desy.de/jannisnecker/timewise_sup).
+

timewise-1.0.0a1/README.md

@@ -0,0 +1,154 @@
+[![CI](https://github.com/JannisNe/timewise/actions/workflows/continous_integration.yml/badge.svg)](https://github.com/JannisNe/timewise/actions/workflows/continous_integration.yml)
+[![Coverage Status](https://coveralls.io/repos/github/JannisNe/timewise/badge.svg?branch=main)](https://coveralls.io/github/JannisNe/timewise?branch=main)
+[![PyPI version](https://badge.fury.io/py/timewise.svg)](https://badge.fury.io/py/timewise)
+[![DOI](https://zenodo.org/badge/449677569.svg)](https://zenodo.org/badge/latestdoi/449677569)
+
+
+![](timewise.png)
+# Infrared light curves from WISE data
+
+This package downloads WISE data for positions on the sky and stacks single-exposure photometry per visit
+
+## Prerequisites
+
+`timewise` makes use of [AMPEL](https://ampelproject.github.io/ampelastro/) and needs a running [MongoDB](https://www.mongodb.com/).
+
+## Installation
+The package can be installed via `pip`:
+```bash
+pip install timewise
+```
+
+To tell AMPEL which modules, aka units, to use, build the corresponding configuration file:
+```bash
+ampel config build -distributions ampel timewise -stop-on-errors 0 -out <path-to-ampel-config-file>
+```
+
+## Usage
+
+### Command line interface
+
+```
+ Usage: timewise [OPTIONS] COMMAND [ARGS]...                                                                
+                                                                                                            
+ Timewsie CLI                                                                                               
+                                                                                                            
+╭─ Options ────────────────────────────────────────────────────────────────────────────────────────────────╮
+│ --log-level           -l      TEXT  Logging level (DEBUG, INFO, WARNING, ERROR, CRITICAL)                │
+│                                     [default: INFO]                                                      │
+│ --install-completion                Install completion for the current shell.                            │
+│ --show-completion                   Show completion for the current shell, to copy it or customize the   │
+│                                     installation.                                                        │
+│ --help                              Show this message and exit.                                          │
+╰──────────────────────────────────────────────────────────────────────────────────────────────────────────╯
+╭─ Commands ───────────────────────────────────────────────────────────────────────────────────────────────╮
+│ download        Download WISE photometry from IRSA                                                       │
+│ prepare-ampel   Prepares the AMPEL job file so AMPEL can be run manually                                 │
+│ process         Processes the lightcurves using AMPEL                                                    │
+│ export          Write stacked lightcurves to disk                                                        │
+│ run-chain       Run download, process and export                                                         │
+│ plot            Make diagnostic plots                                                                    │
+╰──────────────────────────────────────────────────────────────────────────────────────────────────────────╯
+
+```
+
+The input is a CSV file with at least three columns:  
+- `orig_id`: an original identifier that **must** be an integer (for now)
+- `ra`, `dec`: Right Ascension and Declination
+
+
+
+`timewise` is configured with a YAML file. This is a sensible default which will use all single exposure photometry from AllWISE and NEOWISE:
+```yaml
+download:
+  input_csv: <path-to-input>
+
+  backend:
+    type: filesystem
+    base_path: <path-to-working-directory>
+
+  queries:
+    - type: positional
+      radius_arcsec: 6
+      table:
+        name: allwise_p3as_mep
+      columns:
+        - ra
+        - dec
+        - mjd
+        - cntr_mf
+        - w1mpro_ep
+        - w1sigmpro_ep
+        - w2mpro_ep
+        - w2sigmpro_ep
+        - w1flux_ep
+        - w1sigflux_ep
+        - w2flux_ep
+        - w2sigflux_ep
+
+    - type: positional
+      radius_arcsec: 6
+      table:
+        name: neowiser_p1bs_psd
+      columns:
+        - ra
+        - dec
+        - mjd
+        - allwise_cntr
+        - w1mpro
+        - w1sigmpro
+        - w2mpro
+        - w2sigmpro
+        - w1flux
+        - w1sigflux
+        - w2flux
+        - w2sigflux
+
+ampel:
+  mongo_db_name: <mongodb-name>
+```
+
+This configuration file will be the input to all subcommands. Downloading and stacking can be run together or separate.
+
+
+#### All-in-one:
+Run download, stacking, and export:
+```bash
+timewise run-chain <path-to-config-file> <path-to-ampel-config-file> <output-directory>
+```
+
+#### Separate download and processing:
+To only download the data:
+```bash
+timewise download <path-to-config-file>
+```
+
+To execute the stacking:
+```bash
+timewise process <path-to-config-file> <path-to-ampel-config-file>
+```
+
+#### Run AMPEL manually
+Prepare an AMPEL job file for stacking the single-exposure data:
+```bash
+timewise prepare-ampel <path-to-config-file>
+```
+The result will contain the path to the prepared AMPEL job file that can be run with
+```bash
+ampel job -config <path-to-ampel-config-file> -schema <path-to-ampel-job-file>
+```
+
+#### Make some diagnostic plots
+To check the datapoint selection and binning, take a quick look at the data:
+```bash
+timewise plot <path-to-config-file> <indices-to-plot> <output-directory>
+```
+
+
+## Citation
+If you use `timewise` please make sure to cite [Necker et al. A&A 695, A228 (2025)](https://www.aanda.org/articles/aa/abs/2025/03/aa51340-24/aa51340-24.html).
+Additionally, you might want to include a reference to the specific version you are using: [![DOI](https://zenodo.org/badge/449677569.svg)](https://zenodo.org/badge/latestdoi/449677569)
+
+## Difference lightcurves
+Make sure to check out `timewise-sup`, the Timewise Subtraction Pipeline: 
+[link](https://gitlab.desy.de/jannisnecker/timewise_sup).

timewise-1.0.0a1/pyproject.toml

@@ -0,0 +1,96 @@
+[build-system]
+requires = ["poetry-core>=2.0.0"]
+build-backend = "poetry.core.masonry.api"
+
+[project]
+name = "timewise"
+version = "1.0.0a1"
+description = "Download WISE infrared data for many objects and process them with AMPEL"
+authors = [
+    { name = "Jannis Necker", email = "jannis.necker@gmail.com" },
+]
+license = { text = "MIT" }
+readme = "README.md"
+requires-python = ">=3.11,<3.12"
+dependencies = [
+    "tqdm>=4.64.0,<5.0.0",
+    "requests>=2.28.1,<3.0.0",
+    "pandas>=1.4.3,<3.0.0",
+    "numpy>=1.23.2,<2.0.0",
+    "pyvo>=1.7.0,<2.0.0",
+    "astropy>=5.1,<6.0.0",
+    "matplotlib>=3.5.3,<4.0.0",
+    "scikit-image>=0.19.3,<0.22.0",
+    "backoff>=2.1.2,<3.0.0",
+    "virtualenv>=20.16.3,<21.0.0",
+    "seaborn>=0.11.2,<0.14.0",
+    "pydantic>=2.0.0,<3.0.0",
+    "scikit-learn>=1.3.0,<2.0.0",
+    "jupyterlab[jupyter]>=4.0.6,<5.0.0",
+    "jupyter[jupyter]>=1.0.0,<2.0.0",
+    "ampel-alerts (==0.10.3a5)",
+    "typer (>=0.19.2,<0.20.0)",
+    "ampel-photometry (>=0.10.1,<0.11.0)",
+    "ampel-plot (>=0.9.1,<0.10.0)",
+    "ampel-core (>=0.10.4.post0,<0.11.0)",
+    "urllib3 (>=2.5.0,<3.0.0)",
+]
+
+[project.scripts]
+timewise = "timewise.cli:app"
+
+[project.urls]
+Homepage = "https://github.com/JannisNe/timewise"
+"Bug Tracker" = "https://github.com/JannisNe/timewise/issues"
+
+[project.optional-dependencies]
+dev = [
+    "coveralls>=3.3.1,<4.0.0",
+    "pytest>=7.2.2,<8.0.0",
+    "ruff>=0.13.0,<0.14.0",
+    "mypy (>=1.18.2,<2.0.0)",
+    "pandas-stubs (>=2.3.2.250926,<3.0.0.0)",
+    "scipy-stubs (>=1.16.2.0,<2.0.0.0)",
+    "types-pyyaml (>=6.0.12.20250915,<7.0.0.0)",
+    "types-requests (>=2.32.4.20250913,<3.0.0.0)",
+]
+docs = [
+    "myst-parser>=1,<3",
+    "sphinx-rtd-theme>=1.3.0,<2.0.0",
+    "autodoc_pydantic[erdantic]>=2.2.0,<3.0.0"
+]
+
+[tool.coverage.run]
+source = ["timewise", "ampel"]
+omit = ["timewise/v0"]
+parallel = true
+concurrency = ["thread", "multiprocessing"]
+
+[tool.pytest]
+testpaths = "tests"
+pythonpath = "."
+
+# Per-module ignores
+[tool.mypy]
+python_version = "3.11"
+explicit_package_bases = true
+install_types = false
+non_interactive = false
+mypy_path = ["."]
+packages = ["timewise", "ampel"]
+
+[[tool.mypy.overrides]]
+module = [
+    "astropy.*",
+    "pyvo.*",
+    "SciServer.*",
+    "sklearn.*"
+]
+ignore_missing_imports = true
+
+[tool.ruff]
+src = ["timewise", "ampel"]
+exclude = ["tests", "SciScript-Python"]
+
+[tool.ruff.lint]
+ignore = ["F401"]

timewise-1.0.0a1/timewise/__init__.py

@@ -0,0 +1 @@
+__version__ = "1.0.0a1"

timewise-1.0.0a1/timewise/backend/__init__.py

@@ -0,0 +1,6 @@
+from typing import Union
+
+from .base import Backend
+from .filesystem import FileSystemBackend
+
+BackendType = Union[FileSystemBackend]

timewise-1.0.0a1/timewise/backend/base.py

@@ -0,0 +1,36 @@
+import abc
+from typing import Any
+from pydantic import BaseModel
+from astropy.table import Table
+from ..types import TaskID
+
+
+class Backend(abc.ABC, BaseModel):
+    type: str
+    base_path: Any
+    """
+    Abstract persistence backend for jobs, results, and markers.
+    Works with generic TaskIDs so it can be reused across Downloader/Processor.
+    """
+
+    # --- metadata ---
+    @abc.abstractmethod
+    def meta_exists(self, task: TaskID) -> bool: ...
+    @abc.abstractmethod
+    def save_meta(self, task: TaskID, meta: dict[str, Any]) -> None: ...
+    @abc.abstractmethod
+    def load_meta(self, task: TaskID) -> dict[str, Any] | None: ...
+
+    # --- Markers ---
+    @abc.abstractmethod
+    def mark_done(self, task: TaskID) -> None: ...
+    @abc.abstractmethod
+    def is_done(self, task: TaskID) -> bool: ...
+
+    # --- Data ---
+    @abc.abstractmethod
+    def save_data(self, task: TaskID, content: Table) -> None: ...
+    @abc.abstractmethod
+    def load_data(self, task: TaskID) -> Table: ...
+    @abc.abstractmethod
+    def data_exists(self, task: TaskID) -> bool: ...

timewise-1.0.0a1/timewise/backend/filesystem.py

@@ -0,0 +1,80 @@
+import json
+import logging
+from pathlib import Path
+from typing import Any, Literal
+from astropy.table import Table
+
+from .base import Backend
+from ..types import TaskID
+
+
+logger = logging.getLogger(__name__)
+
+
+class FileSystemBackend(Backend):
+    type: Literal["filesystem"] = "filesystem"
+    base_path: Path
+
+    # ----------------------------
+    # Helpers for paths
+    # ----------------------------
+    def _meta_path(self, task: TaskID) -> Path:
+        return self.base_path / f"{task}.meta.json"
+
+    def _marker_path(self, task: TaskID) -> Path:
+        return self.base_path / f"{task}.ok"
+
+    def _data_path(self, task: TaskID) -> Path:
+        return self.base_path / f"{task}.fits"
+
+    # ----------------------------
+    # Metadata
+    # ----------------------------
+    def save_meta(self, task: TaskID, meta: dict[str, Any]) -> None:
+        path = self._meta_path(task)
+        tmp = path.with_suffix(".tmp")
+        tmp.parent.mkdir(parents=True, exist_ok=True)
+        logger.debug(f"writing {path}")
+        tmp.write_text(json.dumps(meta, indent=2))
+        tmp.replace(path)
+
+    def load_meta(self, task: TaskID) -> dict[str, Any] | None:
+        path = self._meta_path(task)
+        if not path.exists():
+            return None
+        return json.loads(path.read_text())
+
+    def meta_exists(self, task: TaskID) -> bool:
+        return self._meta_path(task).exists()
+
+    # ----------------------------
+    # Markers
+    # ----------------------------
+    def mark_done(self, task: TaskID) -> None:
+        mp = self._marker_path(task)
+        mp.parent.mkdir(parents=True, exist_ok=True)
+        logger.debug(f"writing {mp}")
+        mp.write_text("done")
+
+    def is_done(self, task: TaskID) -> bool:
+        return self._marker_path(task).exists()
+
+    # ----------------------------
+    # Data
+    # ----------------------------
+    def save_data(self, task: TaskID, content: Table) -> None:
+        path = self._data_path(task)
+        tmp = path.with_suffix(".tmp")
+        tmp.parent.mkdir(parents=True, exist_ok=True)
+        logger.debug(f"writing {path}")
+        content.write(tmp, format="fits")
+        tmp.replace(path)
+
+    def load_data(self, task: TaskID) -> Table:
+        path = self._data_path(task)
+        if not path.exists():
+            raise FileNotFoundError(path)
+        return Table.read(path, format="fits")
+
+    def data_exists(self, task: TaskID) -> bool:
+        return self._data_path(task).exists()

timewise-1.0.0a1/timewise/chunking.py

@@ -0,0 +1,50 @@
+from typing import Iterator
+from pathlib import Path
+import numpy as np
+from numpy import typing as npt
+import pandas as pd
+import logging
+
+logger = logging.getLogger(__name__)
+
+
+class Chunk:
+    def __init__(
+        self, chunk_id: int, indices: npt.ArrayLike, row_indices: npt.ArrayLike
+    ):
+        self.chunk_id = chunk_id
+        self.indices = indices
+        self.row_numbers = row_indices
+
+
+class Chunker:
+    def __init__(self, input_csv: Path, chunk_size: int):
+        self.input_csv = input_csv
+        self.chunk_size = chunk_size
+        self._n_rows = self._count_rows()
+        logger.debug(f"found {self._n_rows} rows in {self.input_csv}")
+
+    def _count_rows(self) -> int:
+        chunk = 1024 * 1024  # Process 1 MB at a time.
+        f = np.memmap(self.input_csv)
+        num_newlines = sum(
+            np.sum(f[i : i + chunk] == ord("\n")) for i in range(0, len(f), chunk)
+        )
+        del f
+        return num_newlines - 1  # one header row
+
+    def __len__(self) -> int:
+        return int(np.ceil(self._n_rows / self.chunk_size))
+
+    def __iter__(self) -> Iterator[Chunk]:
+        for chunk_id in range(len(self)):
+            yield self.get_chunk(chunk_id)
+
+    def get_chunk(self, chunk_id: int) -> Chunk:
+        if chunk_id >= len(self):
+            raise IndexError(f"Invalid chunk_id {chunk_id}")
+        start = chunk_id * self.chunk_size
+        stop = min(start + self.chunk_size, self._n_rows)
+        indices = pd.read_csv(self.input_csv, skiprows=start, nrows=stop - start).index
+        logger.debug(f"chunk {chunk_id}: from {start} to {stop}")
+        return Chunk(chunk_id, indices, np.arange(start=start, stop=stop))