timewise 0.4.10__tar.gz → 0.5.2__tar.gz

{timewise-0.4.10 → timewise-0.5.2}/PKG-INFO

@@ -1,8 +1,7 @@
-Metadata-Version: 2.1
+Metadata-Version: 2.3
 Name: timewise
-Version: 0.4.10
+Version: 0.5.2
 Summary: A small package to download infrared data from the WISE satellite
-Home-page: https://github.com/JannisNe/timewise
 License: MIT
 Author: Jannis Necker
 Author-email: jannis.necker@gmail.com
@@ -13,25 +12,31 @@ Classifier: Programming Language :: Python :: 3.8
 Classifier: Programming Language :: Python :: 3.9
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
-Requires-Dist: astropy (>=5.1,<6.0)
+Provides-Extra: dev
+Provides-Extra: docs
+Requires-Dist: astropy (>=5.1,<6.0.0)
 Requires-Dist: backoff (>=2.1.2,<3.0.0)
+Requires-Dist: coveralls (>=3.3.1,<4.0.0) ; extra == "dev"
 Requires-Dist: jupyter[jupyter] (>=1.0.0,<2.0.0)
 Requires-Dist: jupyterlab[jupyter] (>=4.0.6,<5.0.0)
 Requires-Dist: matplotlib (>=3.5.3,<4.0.0)
+Requires-Dist: myst-parser (>=1,<3) ; extra == "docs"
 Requires-Dist: numpy (>=1.23.2,<2.0.0)
 Requires-Dist: pandas (>=1.4.3,<3.0.0)
 Requires-Dist: pydantic (>=1.9.0,<2.0.0)
+Requires-Dist: pytest (>=7.2.2,<8.0.0) ; extra == "dev"
 Requires-Dist: pyvo (>=1.4.1,<2.0.0)
 Requires-Dist: requests (>=2.28.1,<3.0.0)
-Requires-Dist: scikit-image (>=0.19.3,<0.21.0)
+Requires-Dist: scikit-image (>=0.19.3,<0.22.0)
 Requires-Dist: scikit-learn (>=1.3.0,<2.0.0)
-Requires-Dist: seaborn (>=0.11.2,<0.13.0)
+Requires-Dist: seaborn (>=0.11.2,<0.14.0)
+Requires-Dist: sphinx-rtd-theme (>=1.3.0,<2.0.0) ; extra == "docs"
 Requires-Dist: tqdm (>=4.64.0,<5.0.0)
 Requires-Dist: urllib3 (==1.26.15)
 Requires-Dist: virtualenv (>=20.16.3,<21.0.0)
 Project-URL: Bug Tracker, https://github.com/JannisNe/timewise/issues
 Project-URL: Documentation, https://timewise.readthedocs.io/en/latest
-Project-URL: Repository, https://github.com/JannisNe/timewise
+Project-URL: Homepage, https://github.com/JannisNe/timewise
 Description-Content-Type: text/markdown
 
 [![CI](https://github.com/JannisNe/timewise/actions/workflows/continous_integration.yml/badge.svg)](https://github.com/JannisNe/timewise/actions/workflows/continous_integration.yml)

timewise-0.5.2/pyproject.toml

@@ -0,0 +1,50 @@
+[build-system]
+requires = ["poetry-core>=2.0.0"]
+build-backend = "poetry.core.masonry.api"
+
+[project]
+name = "timewise"
+version = "0.5.2"
+description = "A small package to download infrared data from the WISE satellite"
+authors = [
+    { name = "Jannis Necker", email = "jannis.necker@gmail.com" },
+]
+license = { text = "MIT" }
+readme = "README.md"
+requires-python = ">=3.8,<3.12"
+dependencies = [
+    "tqdm>=4.64.0,<5.0.0",
+    "requests>=2.28.1,<3.0.0",
+    "pandas>=1.4.3,<3.0.0",
+    "numpy>=1.23.2,<2.0.0",
+    "pyvo>=1.4.1,<2.0.0",
+    "astropy>=5.1,<6.0.0",
+    "matplotlib>=3.5.3,<4.0.0",
+    "scikit-image>=0.19.3,<0.22.0",
+    "backoff>=2.1.2,<3.0.0",
+    "virtualenv>=20.16.3,<21.0.0",
+    "seaborn>=0.11.2,<0.14.0",
+    "urllib3==1.26.15",
+    "pydantic>=1.9.0,<2.0.0",
+    "scikit-learn>=1.3.0,<2.0.0",
+    "jupyterlab[jupyter]>=4.0.6,<5.0.0",
+    "jupyter[jupyter]>=1.0.0,<2.0.0",
+]
+
+[project.scripts]
+timewise = "timewise.cli:timewise_cli"
+
+[project.urls]
+Homepage = "https://github.com/JannisNe/timewise"
+Documentation = "https://timewise.readthedocs.io/en/latest"
+"Bug Tracker" = "https://github.com/JannisNe/timewise/issues"
+
+[project.optional-dependencies]
+dev = [
+    "coveralls>=3.3.1,<4.0.0",
+    "pytest>=7.2.2,<8.0.0",
+]
+docs = [
+    "myst-parser>=1,<3",
+    "sphinx-rtd-theme>=1.3.0,<2.0.0",
+]

{timewise-0.4.10 → timewise-0.5.2}/timewise/__init__.py

@@ -1,5 +1,3 @@
 from timewise.wise_data_by_visit import WiseDataByVisit
 from timewise.wise_bigdata_desy_cluster import WISEDataDESYCluster
 from timewise.parent_sample_base import ParentSampleBase
-
-__version__ = "0.4.10"

{timewise-0.4.10 → timewise-0.5.2}/timewise/big_parent_sample.py

@@ -1,5 +1,4 @@
 import gc
-import os
 import pickle
 import threading
 import time
@@ -30,7 +29,7 @@ class BigParentSampleBase(ParentSampleBase):
         self._keep_df_in_memory = keep_file_in_memory
         self._time_when_df_was_used_last = time.time()
         self._df = None
-        self._cache_file = os.path.join(self.cache_dir, "cache.pkl")
+        self._cache_file = self.cache_dir / "cache.pkl"
         self._lock_cache_file = False
 
         self._clean_thread = threading.Thread(target=self._periodically_drop_df_to_disk, daemon=True, name='ParentSampleCleanThread').start()
@@ -50,7 +49,7 @@ class BigParentSampleBase(ParentSampleBase):
 
         if isinstance(self._df, type(None)):
 
-            if os.path.isfile(self._cache_file):
+            if self._cache_file.is_file():
                 logger.debug(f'loading from {self._cache_file}')
                 self._wait_for_unlock_cache_file()
                 self._lock_cache_file = True
@@ -97,9 +96,9 @@ class BigParentSampleBase(ParentSampleBase):
         logger.debug('stopped clean thread')
 
     def __del__(self):
-        if hasattr(self, "_cache_file") and os.path.isfile(self._cache_file):
+        if hasattr(self, "_cache_file") and self._cache_file.is_file():
             logger.debug(f'removing {self._cache_file}')
-            os.remove(self._cache_file)
+            self._cache_file.unlink()
 
         if hasattr(self, "clean_thread"):
             logger.debug(f'stopping clean thread')

{timewise-0.4.10 → timewise-0.5.2}/timewise/config_loader.py

@@ -1,11 +1,11 @@
 import logging
 import yaml
 import json
-import os
 import inspect
 from pydantic import BaseModel, validator
 import pandas as pd
 import importlib
+from pathlib import Path
 
 from timewise.parent_sample_base import ParentSampleBase
 from timewise.wise_data_base import WISEDataBase
@@ -80,7 +80,7 @@ class TimewiseConfigLoader(BaseModel):
     @validator("filename")
     def validate_file(cls, v: str):
         if v is not None:
-            if not os.path.isfile(v):
+            if not Path(v).is_file():
                 raise ValueError(f"No file {v}!")
         return v
 

timewise-0.5.2/timewise/general.py

@@ -0,0 +1,52 @@
+import logging
+import os
+from pathlib import Path
+
+
+# Setting up the Logger
+main_logger = logging.getLogger('timewise')
+logger_format = logging.Formatter(
+    '%(levelname)s:%(name)s:%(funcName)s - [%(threadName)s] - %(asctime)s: \n\t%(message)s', "%H:%M:%S"
+)
+stream_handler = logging.StreamHandler()
+stream_handler.setFormatter(logger_format)
+main_logger.addHandler(stream_handler)
+main_logger.propagate = False  # do not propagate to root logger
+
+logger = logging.getLogger(__name__)
+
+
+def get_directories() -> dict[str, Path | None]:
+    # Setting up data directory
+    DATA_DIR_KEY = 'TIMEWISE_DATA'
+    if DATA_DIR_KEY in os.environ:
+        data_dir = Path(os.environ[DATA_DIR_KEY]).expanduser()
+    else:
+        logger.warning(f'{DATA_DIR_KEY} not set! Using home directory.')
+        data_dir = Path('~/').expanduser()
+
+    BIGDATA_DIR_KEY = 'TIMEWISE_BIGDATA'
+    if BIGDATA_DIR_KEY in os.environ:
+        bigdata_dir = Path(os.environ[BIGDATA_DIR_KEY]).expanduser()
+        logger.info(f"Using bigdata directory {bigdata_dir}")
+    else:
+        bigdata_dir = None
+        logger.info(f"No bigdata directory set.")
+
+    output_dir = data_dir / 'output'
+    plots_dir = output_dir / 'plots'
+    cache_dir = data_dir / 'cache'
+
+    return {
+        'data_dir': data_dir,
+        'bigdata_dir': bigdata_dir,
+        'output_dir': output_dir,
+        'plots_dir': plots_dir,
+        'cache_dir': cache_dir
+    }
+
+
+def backoff_hndlr(details):
+    logger.info("Backing off {wait:0.1f} seconds after {tries} tries "
+                "calling function {target} with args {args} and kwargs "
+                "{kwargs}".format(**details))

{timewise-0.4.10 → timewise-0.5.2}/timewise/parent_sample_base.py

@@ -3,7 +3,7 @@ import pandas as pd
 import numpy as np
 import logging
 
-from timewise.general import cache_dir, plots_dir
+from timewise.general import get_directories
 from timewise.utils import plot_sdss_cutout, plot_panstarrs_cutout
 
 
@@ -26,14 +26,14 @@ class ParentSampleBase(abc.ABC):
 
     def __init__(self, base_name):
         # set up directories
-        self.cache_dir = os.path.join(cache_dir, base_name)
-        self.plots_dir = os.path.join(plots_dir, base_name)
+        d = get_directories()
+        self.cache_dir = d["cache_dir"] / base_name
+        self.plots_dir = d["plots_dir"] / base_name
 
         for d in [self.cache_dir, self.plots_dir]:
-            if not os.path.isdir(d):
-                os.makedirs(d)
+            d.parent.mkdir(parents=True, exist_ok=True)
 
-        self.local_sample_copy = os.path.join(self.cache_dir, 'sample.csv')
+        self.local_sample_copy = self.cache_dir / 'sample.csv'
 
     def plot_cutout(self, ind, arcsec=20, interactive=False, **kwargs):
         """
@@ -54,9 +54,10 @@ class ParentSampleBase(abc.ABC):
 
         fn = kwargs.pop(
             "fn",
-            [os.path.join(self.plots_dir, f"{i}_{r[self.default_keymap['id']]}.pdf")
+            [self.plots_dir / f"{i}_{r[self.default_keymap['id']]}.pdf"
              for i, r in sel.iterrows()]
         )
+        self.plots_dir.mkdir(parents=True, exist_ok=True)
 
         logger.debug(f"\nRA: {ra}\nDEC: {dec}\nTITLE: {title}\nFN: {fn}")
         ou = list()
@@ -84,4 +85,5 @@ class ParentSampleBase(abc.ABC):
 
     def save_local(self):
         logger.debug(f"saving under {self.local_sample_copy}")
+        self.local_sample_copy.parent.mkdir(parents=True, exist_ok=True)
         self.df.to_csv(self.local_sample_copy)

{timewise-0.4.10 → timewise-0.5.2}/timewise/utils.py

@@ -12,14 +12,20 @@ from functools import cache
 from astropy.table import Table
 from PIL import Image
 from io import BytesIO
+import hashlib
+from threading import Thread
+from queue import Queue
+import sys
 
-
-from timewise.general import cache_dir, backoff_hndlr
+from timewise.general import backoff_hndlr, get_directories
 
 
 logger = logging.getLogger(__name__)
 mirong_url = 'http://staff.ustc.edu.cn/~jnac/data_public/wisevar.txt'
-local_copy = os.path.join(cache_dir, 'mirong_sample.csv')
+
+
+def get_mirong_path():
+    return get_directories()['cache_dir'] / 'mirong_sample.csv'
 
 
 @cache
@@ -29,7 +35,8 @@ def get_2d_gaussian_correction(cl):
 
 def get_mirong_sample():
 
-    if not os.path.isfile(local_copy):
+    mirong_path = get_mirong_path()
+    if not mirong_path.is_file():
 
         logger.info(f'getting MIRONG sample from {mirong_url}')
         r = requests.get(mirong_url)
@@ -44,16 +51,16 @@ def get_mirong_sample():
         mirong_sample = pd.DataFrame(lll[1:-1], columns=lll[0])
         mirong_sample['ra'] = mirong_sample['RA']
         mirong_sample['dec'] = mirong_sample['DEC']
-        logger.debug(f'saving to {local_copy}')
-
-        mirong_sample.to_csv(local_copy, index=False)
         logger.info(f'found {len(mirong_sample)} objects in MIRONG Sample')
+
         mirong_sample.drop(columns=['ra', 'dec'], inplace=True)
-        mirong_sample.to_csv(local_copy, index=False)
+        logger.debug(f'saving to {mirong_path}')
+        mirong_path.parent.mkdir(parents=True, exist_ok=True)
+        mirong_sample.to_csv(mirong_path, index=False)
 
     else:
-        logger.debug(f'loading {local_copy}')
-        mirong_sample = pd.read_csv(local_copy)
+        logger.debug(f'loading {mirong_path}')
+        mirong_sample = pd.read_csv(mirong_path)
 
     return mirong_sample
 
@@ -150,6 +157,24 @@ class PanSTARRSQueryError(Exception):
     pass
 
 
+def load_cache_or_download(url):
+    logger.debug(f"loading or downloading {url}")
+    h = hashlib.md5(url.encode()).hexdigest()
+    cache_dir = get_directories()['cache_dir']
+    cache_file = cache_dir / (h + ".cache")
+    logger.debug(f"cache file is {cache_file}")
+    if not cache_file.is_file():
+        logger.debug(f"downloading {url}")
+        r = requests.get(url)
+        with open(cache_file, 'wb') as f:
+            f.write(r.content)
+        return r.content
+    else:
+        logger.debug(f"loading {cache_file}")
+        with open(cache_file, 'rb') as f:
+            return f.read()
+
+
 def annotate_not_available(ax):
     xlim = ax.get_xlim()
     ylim = ax.get_ylim()
@@ -170,7 +195,8 @@ def getimages(ra, dec, filters="grizy"):
 
     service = "https://ps1images.stsci.edu/cgi-bin/ps1filenames.py"
     url = f"{service}?ra={ra}&dec={dec}&filters={filters}"
-    table = Table.read(url, format='ascii')
+    content = load_cache_or_download(url)
+    table = Table.read(content.decode(), format='ascii')
     return table
 
 
@@ -231,8 +257,8 @@ def getcolorim(ra, dec, size=240, output_size=None, filters="grizy", format="jpg
     if format not in ("jpg", "png"):
         raise ValueError("format must be jpg or png")
     url = geturl(ra, dec, size=size, filters=filters, output_size=output_size, format=format, color=True)
-    r = requests.get(url)
-    im = Image.open(BytesIO(r.content))
+    content = load_cache_or_download(url)
+    im = Image.open(BytesIO(content))
     return im
 
 
@@ -253,8 +279,8 @@ def getgrayim(ra, dec, size=240, output_size=None, filter="g", format="jpg"):
     if filter not in list("grizy"):
         raise ValueError("filter must be one of grizy")
     url = geturl(ra, dec, size=size, filters=filter, output_size=output_size, format=format)
-    r = requests.get(url[0])
-    im = Image.open(BytesIO(r.content))
+    content = load_cache_or_download(url[0])
+    im = Image.open(BytesIO(content))
     return im
 
 
@@ -446,3 +472,47 @@ class StableTAPService(vo.dal.TAPService):
 #######################################################
 #            END CUSTOM TAP Service                   #
 ###########################################################################################################
+
+
+###########################################################################################################
+#            START CUSTOM TAP Service                 #
+#######################################################
+
+
+class ErrorQueue(Queue):
+    """Queue subclass whose join() re-raises exceptions from worker threads."""
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.error_queue = Queue()
+
+    def report_error(self, exc_info):
+        """Called by workers to push an exception into the error queue."""
+        self.error_queue.put(exc_info)
+        # Also decrement unfinished_tasks, so join() won't block forever
+        with self.all_tasks_done:
+            self.unfinished_tasks = max(0, self.unfinished_tasks - 1)
+            self.all_tasks_done.notify_all()
+
+    def join(self):
+        """Wait until all tasks are done, or raise if a worker failed."""
+        with self.all_tasks_done:
+            while self.unfinished_tasks:
+                if not self.error_queue.empty():
+                    exc_info = self.error_queue.get()
+                    raise exc_info[1].with_traceback(exc_info[2])
+                self.all_tasks_done.wait()
+
+
+class ExceptionSafeThread(Thread):
+    """Thread subclass that reports uncaught exceptions to the ErrorQueue."""
+
+    def __init__(self, error_queue: ErrorQueue, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.error_queue = error_queue
+
+    def run(self):
+        try:
+            super().run()
+        except Exception:
+            self.error_queue.report_error(sys.exc_info())