timewise 0.4.12__tar.gz → 0.5.3__tar.gz

{timewise-0.4.12 → timewise-0.5.3}/PKG-INFO

@@ -1,37 +1,41 @@
 Metadata-Version: 2.3
 Name: timewise
-Version: 0.4.12
+Version: 0.5.3
 Summary: A small package to download infrared data from the WISE satellite
-Home-page: https://github.com/JannisNe/timewise
 License: MIT
 Author: Jannis Necker
 Author-email: jannis.necker@gmail.com
-Requires-Python: >=3.8,<3.12
+Requires-Python: >=3.9,<3.12
 Classifier: License :: OSI Approved :: MIT License
 Classifier: Programming Language :: Python :: 3
-Classifier: Programming Language :: Python :: 3.8
 Classifier: Programming Language :: Python :: 3.9
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
-Requires-Dist: astropy (>=5.1,<6.0)
+Provides-Extra: dev
+Provides-Extra: docs
+Requires-Dist: astropy (>=5.1,<6.0.0)
 Requires-Dist: backoff (>=2.1.2,<3.0.0)
+Requires-Dist: coveralls (>=3.3.1,<4.0.0) ; extra == "dev"
 Requires-Dist: jupyter[jupyter] (>=1.0.0,<2.0.0)
 Requires-Dist: jupyterlab[jupyter] (>=4.0.6,<5.0.0)
 Requires-Dist: matplotlib (>=3.5.3,<4.0.0)
+Requires-Dist: myst-parser (>=1,<3) ; extra == "docs"
 Requires-Dist: numpy (>=1.23.2,<2.0.0)
 Requires-Dist: pandas (>=1.4.3,<3.0.0)
 Requires-Dist: pydantic (>=1.9.0,<2.0.0)
-Requires-Dist: pyvo (>=1.4.1,<2.0.0)
+Requires-Dist: pytest (>=7.2.2,<8.0.0) ; extra == "dev"
+Requires-Dist: pyvo (>=1.7.0,<2.0.0)
 Requires-Dist: requests (>=2.28.1,<3.0.0)
-Requires-Dist: scikit-image (>=0.19.3,<0.21.0)
+Requires-Dist: scikit-image (>=0.19.3,<0.22.0)
 Requires-Dist: scikit-learn (>=1.3.0,<2.0.0)
-Requires-Dist: seaborn (>=0.11.2,<0.13.0)
+Requires-Dist: seaborn (>=0.11.2,<0.14.0)
+Requires-Dist: sphinx-rtd-theme (>=1.3.0,<2.0.0) ; extra == "docs"
 Requires-Dist: tqdm (>=4.64.0,<5.0.0)
 Requires-Dist: urllib3 (==1.26.15)
 Requires-Dist: virtualenv (>=20.16.3,<21.0.0)
 Project-URL: Bug Tracker, https://github.com/JannisNe/timewise/issues
 Project-URL: Documentation, https://timewise.readthedocs.io/en/latest
-Project-URL: Repository, https://github.com/JannisNe/timewise
+Project-URL: Homepage, https://github.com/JannisNe/timewise
 Description-Content-Type: text/markdown
 
 [![CI](https://github.com/JannisNe/timewise/actions/workflows/continous_integration.yml/badge.svg)](https://github.com/JannisNe/timewise/actions/workflows/continous_integration.yml)

timewise-0.5.3/pyproject.toml

@@ -0,0 +1,50 @@
+[build-system]
+requires = ["poetry-core>=2.0.0"]
+build-backend = "poetry.core.masonry.api"
+
+[project]
+name = "timewise"
+version = "0.5.3"
+description = "A small package to download infrared data from the WISE satellite"
+authors = [
+    { name = "Jannis Necker", email = "jannis.necker@gmail.com" },
+]
+license = { text = "MIT" }
+readme = "README.md"
+requires-python = ">=3.9,<3.12"
+dependencies = [
+    "tqdm>=4.64.0,<5.0.0",
+    "requests>=2.28.1,<3.0.0",
+    "pandas>=1.4.3,<3.0.0",
+    "numpy>=1.23.2,<2.0.0",
+    "pyvo>=1.7.0,<2.0.0",
+    "astropy>=5.1,<6.0.0",
+    "matplotlib>=3.5.3,<4.0.0",
+    "scikit-image>=0.19.3,<0.22.0",
+    "backoff>=2.1.2,<3.0.0",
+    "virtualenv>=20.16.3,<21.0.0",
+    "seaborn>=0.11.2,<0.14.0",
+    "urllib3==1.26.15",
+    "pydantic>=1.9.0,<2.0.0",
+    "scikit-learn>=1.3.0,<2.0.0",
+    "jupyterlab[jupyter]>=4.0.6,<5.0.0",
+    "jupyter[jupyter]>=1.0.0,<2.0.0",
+]
+
+[project.scripts]
+timewise = "timewise.cli:timewise_cli"
+
+[project.urls]
+Homepage = "https://github.com/JannisNe/timewise"
+Documentation = "https://timewise.readthedocs.io/en/latest"
+"Bug Tracker" = "https://github.com/JannisNe/timewise/issues"
+
+[project.optional-dependencies]
+dev = [
+    "coveralls>=3.3.1,<4.0.0",
+    "pytest>=7.2.2,<8.0.0",
+]
+docs = [
+    "myst-parser>=1,<3",
+    "sphinx-rtd-theme>=1.3.0,<2.0.0",
+]

{timewise-0.4.12 → timewise-0.5.3}/timewise/__init__.py

@@ -2,4 +2,4 @@ from timewise.wise_data_by_visit import WiseDataByVisit
 from timewise.wise_bigdata_desy_cluster import WISEDataDESYCluster
 from timewise.parent_sample_base import ParentSampleBase
 
-__version__ = "0.4.12"
+__version__ = "0.5.3"

{timewise-0.4.12 → timewise-0.5.3}/timewise/big_parent_sample.py

@@ -1,5 +1,4 @@
 import gc
-import os
 import pickle
 import threading
 import time
@@ -30,7 +29,7 @@ class BigParentSampleBase(ParentSampleBase):
         self._keep_df_in_memory = keep_file_in_memory
         self._time_when_df_was_used_last = time.time()
         self._df = None
-        self._cache_file = os.path.join(self.cache_dir, "cache.pkl")
+        self._cache_file = self.cache_dir / "cache.pkl"
         self._lock_cache_file = False
 
         self._clean_thread = threading.Thread(target=self._periodically_drop_df_to_disk, daemon=True, name='ParentSampleCleanThread').start()
@@ -50,7 +49,7 @@ class BigParentSampleBase(ParentSampleBase):
 
         if isinstance(self._df, type(None)):
 
-            if os.path.isfile(self._cache_file):
+            if self._cache_file.is_file():
                 logger.debug(f'loading from {self._cache_file}')
                 self._wait_for_unlock_cache_file()
                 self._lock_cache_file = True
@@ -97,9 +96,9 @@ class BigParentSampleBase(ParentSampleBase):
         logger.debug('stopped clean thread')
 
     def __del__(self):
-        if hasattr(self, "_cache_file") and os.path.isfile(self._cache_file):
+        if hasattr(self, "_cache_file") and self._cache_file.is_file():
             logger.debug(f'removing {self._cache_file}')
-            os.remove(self._cache_file)
+            self._cache_file.unlink()
 
         if hasattr(self, "clean_thread"):
             logger.debug(f'stopping clean thread')

{timewise-0.4.12 → timewise-0.5.3}/timewise/config_loader.py

@@ -1,11 +1,11 @@
 import logging
 import yaml
 import json
-import os
 import inspect
 from pydantic import BaseModel, validator
 import pandas as pd
 import importlib
+from pathlib import Path
 
 from timewise.parent_sample_base import ParentSampleBase
 from timewise.wise_data_base import WISEDataBase
@@ -80,7 +80,7 @@ class TimewiseConfigLoader(BaseModel):
     @validator("filename")
     def validate_file(cls, v: str):
         if v is not None:
-            if not os.path.isfile(v):
+            if not Path(v).is_file():
                 raise ValueError(f"No file {v}!")
         return v
 

timewise-0.5.3/timewise/general.py

@@ -0,0 +1,52 @@
+import logging
+import os
+from pathlib import Path
+
+
+# Setting up the Logger
+main_logger = logging.getLogger('timewise')
+logger_format = logging.Formatter(
+    '%(levelname)s:%(name)s:%(funcName)s - [%(threadName)s] - %(asctime)s: \n\t%(message)s', "%H:%M:%S"
+)
+stream_handler = logging.StreamHandler()
+stream_handler.setFormatter(logger_format)
+main_logger.addHandler(stream_handler)
+main_logger.propagate = False  # do not propagate to root logger
+
+logger = logging.getLogger(__name__)
+
+
+def get_directories() -> dict[str, Path | None]:
+    # Setting up data directory
+    DATA_DIR_KEY = 'TIMEWISE_DATA'
+    if DATA_DIR_KEY in os.environ:
+        data_dir = Path(os.environ[DATA_DIR_KEY]).expanduser()
+    else:
+        logger.warning(f'{DATA_DIR_KEY} not set! Using home directory.')
+        data_dir = Path('~/').expanduser()
+
+    BIGDATA_DIR_KEY = 'TIMEWISE_BIGDATA'
+    if BIGDATA_DIR_KEY in os.environ:
+        bigdata_dir = Path(os.environ[BIGDATA_DIR_KEY]).expanduser()
+        logger.info(f"Using bigdata directory {bigdata_dir}")
+    else:
+        bigdata_dir = None
+        logger.info(f"No bigdata directory set.")
+
+    output_dir = data_dir / 'output'
+    plots_dir = output_dir / 'plots'
+    cache_dir = data_dir / 'cache'
+
+    return {
+        'data_dir': data_dir,
+        'bigdata_dir': bigdata_dir,
+        'output_dir': output_dir,
+        'plots_dir': plots_dir,
+        'cache_dir': cache_dir
+    }
+
+
+def backoff_hndlr(details):
+    logger.info("Backing off {wait:0.1f} seconds after {tries} tries "
+                "calling function {target} with args {args} and kwargs "
+                "{kwargs}".format(**details))

{timewise-0.4.12 → timewise-0.5.3}/timewise/parent_sample_base.py

@@ -3,7 +3,7 @@ import pandas as pd
 import numpy as np
 import logging
 
-from timewise.general import cache_dir, plots_dir
+from timewise.general import get_directories
 from timewise.utils import plot_sdss_cutout, plot_panstarrs_cutout
 
 
@@ -26,14 +26,14 @@ class ParentSampleBase(abc.ABC):
 
     def __init__(self, base_name):
         # set up directories
-        self.cache_dir = os.path.join(cache_dir, base_name)
-        self.plots_dir = os.path.join(plots_dir, base_name)
+        d = get_directories()
+        self.cache_dir = d["cache_dir"] / base_name
+        self.plots_dir = d["plots_dir"] / base_name
 
         for d in [self.cache_dir, self.plots_dir]:
-            if not os.path.isdir(d):
-                os.makedirs(d)
+            d.parent.mkdir(parents=True, exist_ok=True)
 
-        self.local_sample_copy = os.path.join(self.cache_dir, 'sample.csv')
+        self.local_sample_copy = self.cache_dir / 'sample.csv'
 
     def plot_cutout(self, ind, arcsec=20, interactive=False, **kwargs):
         """
@@ -54,9 +54,10 @@ class ParentSampleBase(abc.ABC):
 
         fn = kwargs.pop(
             "fn",
-            [os.path.join(self.plots_dir, f"{i}_{r[self.default_keymap['id']]}.pdf")
+            [self.plots_dir / f"{i}_{r[self.default_keymap['id']]}.pdf"
              for i, r in sel.iterrows()]
         )
+        self.plots_dir.mkdir(parents=True, exist_ok=True)
 
         logger.debug(f"\nRA: {ra}\nDEC: {dec}\nTITLE: {title}\nFN: {fn}")
         ou = list()
@@ -84,4 +85,5 @@ class ParentSampleBase(abc.ABC):
 
     def save_local(self):
         logger.debug(f"saving under {self.local_sample_copy}")
+        self.local_sample_copy.parent.mkdir(parents=True, exist_ok=True)
         self.df.to_csv(self.local_sample_copy)

{timewise-0.4.12 → timewise-0.5.3}/timewise/utils.py

@@ -13,14 +13,19 @@ from astropy.table import Table
 from PIL import Image
 from io import BytesIO
 import hashlib
+from threading import Thread
+from queue import Queue
+import sys
 
-
-from timewise.general import cache_dir, backoff_hndlr
+from timewise.general import backoff_hndlr, get_directories
 
 
 logger = logging.getLogger(__name__)
 mirong_url = 'http://staff.ustc.edu.cn/~jnac/data_public/wisevar.txt'
-local_copy = os.path.join(cache_dir, 'mirong_sample.csv')
+
+
+def get_mirong_path():
+    return get_directories()['cache_dir'] / 'mirong_sample.csv'
 
 
 @cache
@@ -30,7 +35,8 @@ def get_2d_gaussian_correction(cl):
 
 def get_mirong_sample():
 
-    if not os.path.isfile(local_copy):
+    mirong_path = get_mirong_path()
+    if not mirong_path.is_file():
 
         logger.info(f'getting MIRONG sample from {mirong_url}')
         r = requests.get(mirong_url)
@@ -45,16 +51,16 @@ def get_mirong_sample():
         mirong_sample = pd.DataFrame(lll[1:-1], columns=lll[0])
         mirong_sample['ra'] = mirong_sample['RA']
         mirong_sample['dec'] = mirong_sample['DEC']
-        logger.debug(f'saving to {local_copy}')
-
-        mirong_sample.to_csv(local_copy, index=False)
         logger.info(f'found {len(mirong_sample)} objects in MIRONG Sample')
+
         mirong_sample.drop(columns=['ra', 'dec'], inplace=True)
-        mirong_sample.to_csv(local_copy, index=False)
+        logger.debug(f'saving to {mirong_path}')
+        mirong_path.parent.mkdir(parents=True, exist_ok=True)
+        mirong_sample.to_csv(mirong_path, index=False)
 
     else:
-        logger.debug(f'loading {local_copy}')
-        mirong_sample = pd.read_csv(local_copy)
+        logger.debug(f'loading {mirong_path}')
+        mirong_sample = pd.read_csv(mirong_path)
 
     return mirong_sample
 
@@ -154,9 +160,10 @@ class PanSTARRSQueryError(Exception):
 def load_cache_or_download(url):
     logger.debug(f"loading or downloading {url}")
     h = hashlib.md5(url.encode()).hexdigest()
-    cache_file = os.path.join(cache_dir, h + ".cache")
+    cache_dir = get_directories()['cache_dir']
+    cache_file = cache_dir / (h + ".cache")
     logger.debug(f"cache file is {cache_file}")
-    if not os.path.isfile(cache_file):
+    if not cache_file.is_file():
         logger.debug(f"downloading {url}")
         r = requests.get(url)
         with open(cache_file, 'wb') as f:
@@ -424,9 +431,48 @@ def get_excess_variance(y, y_err, mu):
 
 class StableAsyncTAPJob(vo.dal.AsyncTAPJob):
     """
-    Implements backoff for call of phase which otherwise breaks the code if there are connection issues
+    Implements backoff for call of phase which otherwise breaks the code if there are connection issues.
+    Also stores the response of TapQuery.submit() under self.submit_response for debugging
     """
 
+    def __init__(self, url, *, session=None, delete=True):
+        super(StableAsyncTAPJob, self).__init__(url, session=session, delete=delete)
+        self.submit_response = None
+
+    @classmethod
+    def create(
+            cls, baseurl, query, *, language="ADQL", maxrec=None, uploads=None,
+            session=None, **keywords):
+        """
+        creates a async tap job on the server under ``baseurl``
+        Raises requests.HTTPError if TAPQuery.submit() failes.
+
+        Parameters
+        ----------
+        baseurl : str
+            the TAP baseurl
+        query : str
+            the query string
+        language : str
+            specifies the query language, default ADQL.
+            useful for services which allow to use the backend query language.
+        maxrec : int
+            the maximum records to return. defaults to the service default
+        uploads : dict
+            a mapping from table names to objects containing a votable
+        session : object
+           optional session to use for network requests
+        """
+        tapquery = vo.dal.TAPQuery(
+            baseurl, query, mode="async", language=language, maxrec=maxrec,
+            uploads=uploads, session=session, **keywords)
+        response = tapquery.submit()
+        response.raise_for_status()
+        job = cls(response.url, session=session)
+        job._client_set_maxrec = maxrec
+        job.submit_response = response
+        return job
+
     @property
     @backoff.on_exception(
         backoff.expo,
@@ -446,6 +492,7 @@ class StableTAPService(vo.dal.TAPService):
     def submit_job(
             self,
             query,
+            *,
             language="ADQL",
             maxrec=None,
             uploads=None,
@@ -454,10 +501,10 @@ class StableTAPService(vo.dal.TAPService):
         return StableAsyncTAPJob.create(
             self.baseurl,
             query,
-            language,
-            maxrec,
-            uploads,
-            self._session,
+            language=language,
+            maxrec=maxrec,
+            uploads=uploads,
+            session=self._session,
             **keywords
         )
 
@@ -465,3 +512,47 @@ class StableTAPService(vo.dal.TAPService):
 #######################################################
 #            END CUSTOM TAP Service                   #
 ###########################################################################################################
+
+
+###########################################################################################################
+#            START CUSTOM TAP Service                 #
+#######################################################
+
+
+class ErrorQueue(Queue):
+    """Queue subclass whose join() re-raises exceptions from worker threads."""
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.error_queue = Queue()
+
+    def report_error(self, exc_info):
+        """Called by workers to push an exception into the error queue."""
+        self.error_queue.put(exc_info)
+        # Also decrement unfinished_tasks, so join() won't block forever
+        with self.all_tasks_done:
+            self.unfinished_tasks = max(0, self.unfinished_tasks - 1)
+            self.all_tasks_done.notify_all()
+
+    def join(self):
+        """Wait until all tasks are done, or raise if a worker failed."""
+        with self.all_tasks_done:
+            while self.unfinished_tasks:
+                if not self.error_queue.empty():
+                    exc_info = self.error_queue.get()
+                    raise exc_info[1].with_traceback(exc_info[2])
+                self.all_tasks_done.wait()
+
+
+class ExceptionSafeThread(Thread):
+    """Thread subclass that reports uncaught exceptions to the ErrorQueue."""
+
+    def __init__(self, error_queue: ErrorQueue, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.error_queue = error_queue
+
+    def run(self):
+        try:
+            super().run()
+        except Exception:
+            self.error_queue.report_error(sys.exc_info())