timewise 0.4.12__tar.gz → 0.5.2__tar.gz

{timewise-0.4.12 → timewise-0.5.2}/PKG-INFO

@@ -1,8 +1,7 @@
 Metadata-Version: 2.3
 Name: timewise
-Version: 0.4.12
+Version: 0.5.2
 Summary: A small package to download infrared data from the WISE satellite
-Home-page: https://github.com/JannisNe/timewise
 License: MIT
 Author: Jannis Necker
 Author-email: jannis.necker@gmail.com
@@ -13,25 +12,31 @@ Classifier: Programming Language :: Python :: 3.8
 Classifier: Programming Language :: Python :: 3.9
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
-Requires-Dist: astropy (>=5.1,<6.0)
+Provides-Extra: dev
+Provides-Extra: docs
+Requires-Dist: astropy (>=5.1,<6.0.0)
 Requires-Dist: backoff (>=2.1.2,<3.0.0)
+Requires-Dist: coveralls (>=3.3.1,<4.0.0) ; extra == "dev"
 Requires-Dist: jupyter[jupyter] (>=1.0.0,<2.0.0)
 Requires-Dist: jupyterlab[jupyter] (>=4.0.6,<5.0.0)
 Requires-Dist: matplotlib (>=3.5.3,<4.0.0)
+Requires-Dist: myst-parser (>=1,<3) ; extra == "docs"
 Requires-Dist: numpy (>=1.23.2,<2.0.0)
 Requires-Dist: pandas (>=1.4.3,<3.0.0)
 Requires-Dist: pydantic (>=1.9.0,<2.0.0)
+Requires-Dist: pytest (>=7.2.2,<8.0.0) ; extra == "dev"
 Requires-Dist: pyvo (>=1.4.1,<2.0.0)
 Requires-Dist: requests (>=2.28.1,<3.0.0)
-Requires-Dist: scikit-image (>=0.19.3,<0.21.0)
+Requires-Dist: scikit-image (>=0.19.3,<0.22.0)
 Requires-Dist: scikit-learn (>=1.3.0,<2.0.0)
-Requires-Dist: seaborn (>=0.11.2,<0.13.0)
+Requires-Dist: seaborn (>=0.11.2,<0.14.0)
+Requires-Dist: sphinx-rtd-theme (>=1.3.0,<2.0.0) ; extra == "docs"
 Requires-Dist: tqdm (>=4.64.0,<5.0.0)
 Requires-Dist: urllib3 (==1.26.15)
 Requires-Dist: virtualenv (>=20.16.3,<21.0.0)
 Project-URL: Bug Tracker, https://github.com/JannisNe/timewise/issues
 Project-URL: Documentation, https://timewise.readthedocs.io/en/latest
-Project-URL: Repository, https://github.com/JannisNe/timewise
+Project-URL: Homepage, https://github.com/JannisNe/timewise
 Description-Content-Type: text/markdown
 
 [![CI](https://github.com/JannisNe/timewise/actions/workflows/continous_integration.yml/badge.svg)](https://github.com/JannisNe/timewise/actions/workflows/continous_integration.yml)

timewise-0.5.2/pyproject.toml

@@ -0,0 +1,50 @@
+[build-system]
+requires = ["poetry-core>=2.0.0"]
+build-backend = "poetry.core.masonry.api"
+
+[project]
+name = "timewise"
+version = "0.5.2"
+description = "A small package to download infrared data from the WISE satellite"
+authors = [
+    { name = "Jannis Necker", email = "jannis.necker@gmail.com" },
+]
+license = { text = "MIT" }
+readme = "README.md"
+requires-python = ">=3.8,<3.12"
+dependencies = [
+    "tqdm>=4.64.0,<5.0.0",
+    "requests>=2.28.1,<3.0.0",
+    "pandas>=1.4.3,<3.0.0",
+    "numpy>=1.23.2,<2.0.0",
+    "pyvo>=1.4.1,<2.0.0",
+    "astropy>=5.1,<6.0.0",
+    "matplotlib>=3.5.3,<4.0.0",
+    "scikit-image>=0.19.3,<0.22.0",
+    "backoff>=2.1.2,<3.0.0",
+    "virtualenv>=20.16.3,<21.0.0",
+    "seaborn>=0.11.2,<0.14.0",
+    "urllib3==1.26.15",
+    "pydantic>=1.9.0,<2.0.0",
+    "scikit-learn>=1.3.0,<2.0.0",
+    "jupyterlab[jupyter]>=4.0.6,<5.0.0",
+    "jupyter[jupyter]>=1.0.0,<2.0.0",
+]
+
+[project.scripts]
+timewise = "timewise.cli:timewise_cli"
+
+[project.urls]
+Homepage = "https://github.com/JannisNe/timewise"
+Documentation = "https://timewise.readthedocs.io/en/latest"
+"Bug Tracker" = "https://github.com/JannisNe/timewise/issues"
+
+[project.optional-dependencies]
+dev = [
+    "coveralls>=3.3.1,<4.0.0",
+    "pytest>=7.2.2,<8.0.0",
+]
+docs = [
+    "myst-parser>=1,<3",
+    "sphinx-rtd-theme>=1.3.0,<2.0.0",
+]

{timewise-0.4.12 → timewise-0.5.2}/timewise/__init__.py

@@ -1,5 +1,3 @@
 from timewise.wise_data_by_visit import WiseDataByVisit
 from timewise.wise_bigdata_desy_cluster import WISEDataDESYCluster
 from timewise.parent_sample_base import ParentSampleBase
-
-__version__ = "0.4.12"

{timewise-0.4.12 → timewise-0.5.2}/timewise/big_parent_sample.py

@@ -1,5 +1,4 @@
 import gc
-import os
 import pickle
 import threading
 import time
@@ -30,7 +29,7 @@ class BigParentSampleBase(ParentSampleBase):
         self._keep_df_in_memory = keep_file_in_memory
         self._time_when_df_was_used_last = time.time()
         self._df = None
-        self._cache_file = os.path.join(self.cache_dir, "cache.pkl")
+        self._cache_file = self.cache_dir / "cache.pkl"
         self._lock_cache_file = False
 
         self._clean_thread = threading.Thread(target=self._periodically_drop_df_to_disk, daemon=True, name='ParentSampleCleanThread').start()
@@ -50,7 +49,7 @@ class BigParentSampleBase(ParentSampleBase):
 
         if isinstance(self._df, type(None)):
 
-            if os.path.isfile(self._cache_file):
+            if self._cache_file.is_file():
                 logger.debug(f'loading from {self._cache_file}')
                 self._wait_for_unlock_cache_file()
                 self._lock_cache_file = True
@@ -97,9 +96,9 @@ class BigParentSampleBase(ParentSampleBase):
         logger.debug('stopped clean thread')
 
     def __del__(self):
-        if hasattr(self, "_cache_file") and os.path.isfile(self._cache_file):
+        if hasattr(self, "_cache_file") and self._cache_file.is_file():
             logger.debug(f'removing {self._cache_file}')
-            os.remove(self._cache_file)
+            self._cache_file.unlink()
 
         if hasattr(self, "clean_thread"):
             logger.debug(f'stopping clean thread')

{timewise-0.4.12 → timewise-0.5.2}/timewise/config_loader.py

@@ -1,11 +1,11 @@
 import logging
 import yaml
 import json
-import os
 import inspect
 from pydantic import BaseModel, validator
 import pandas as pd
 import importlib
+from pathlib import Path
 
 from timewise.parent_sample_base import ParentSampleBase
 from timewise.wise_data_base import WISEDataBase
@@ -80,7 +80,7 @@ class TimewiseConfigLoader(BaseModel):
     @validator("filename")
     def validate_file(cls, v: str):
         if v is not None:
-            if not os.path.isfile(v):
+            if not Path(v).is_file():
                 raise ValueError(f"No file {v}!")
         return v
 

timewise-0.5.2/timewise/general.py

@@ -0,0 +1,52 @@
+import logging
+import os
+from pathlib import Path
+
+
+# Setting up the Logger
+main_logger = logging.getLogger('timewise')
+logger_format = logging.Formatter(
+    '%(levelname)s:%(name)s:%(funcName)s - [%(threadName)s] - %(asctime)s: \n\t%(message)s', "%H:%M:%S"
+)
+stream_handler = logging.StreamHandler()
+stream_handler.setFormatter(logger_format)
+main_logger.addHandler(stream_handler)
+main_logger.propagate = False  # do not propagate to root logger
+
+logger = logging.getLogger(__name__)
+
+
+def get_directories() -> dict[str, Path | None]:
+    # Setting up data directory
+    DATA_DIR_KEY = 'TIMEWISE_DATA'
+    if DATA_DIR_KEY in os.environ:
+        data_dir = Path(os.environ[DATA_DIR_KEY]).expanduser()
+    else:
+        logger.warning(f'{DATA_DIR_KEY} not set! Using home directory.')
+        data_dir = Path('~/').expanduser()
+
+    BIGDATA_DIR_KEY = 'TIMEWISE_BIGDATA'
+    if BIGDATA_DIR_KEY in os.environ:
+        bigdata_dir = Path(os.environ[BIGDATA_DIR_KEY]).expanduser()
+        logger.info(f"Using bigdata directory {bigdata_dir}")
+    else:
+        bigdata_dir = None
+        logger.info(f"No bigdata directory set.")
+
+    output_dir = data_dir / 'output'
+    plots_dir = output_dir / 'plots'
+    cache_dir = data_dir / 'cache'
+
+    return {
+        'data_dir': data_dir,
+        'bigdata_dir': bigdata_dir,
+        'output_dir': output_dir,
+        'plots_dir': plots_dir,
+        'cache_dir': cache_dir
+    }
+
+
+def backoff_hndlr(details):
+    logger.info("Backing off {wait:0.1f} seconds after {tries} tries "
+                "calling function {target} with args {args} and kwargs "
+                "{kwargs}".format(**details))

{timewise-0.4.12 → timewise-0.5.2}/timewise/parent_sample_base.py

@@ -3,7 +3,7 @@ import pandas as pd
 import numpy as np
 import logging
 
-from timewise.general import cache_dir, plots_dir
+from timewise.general import get_directories
 from timewise.utils import plot_sdss_cutout, plot_panstarrs_cutout
 
 
@@ -26,14 +26,14 @@ class ParentSampleBase(abc.ABC):
 
     def __init__(self, base_name):
         # set up directories
-        self.cache_dir = os.path.join(cache_dir, base_name)
-        self.plots_dir = os.path.join(plots_dir, base_name)
+        d = get_directories()
+        self.cache_dir = d["cache_dir"] / base_name
+        self.plots_dir = d["plots_dir"] / base_name
 
         for d in [self.cache_dir, self.plots_dir]:
-            if not os.path.isdir(d):
-                os.makedirs(d)
+            d.parent.mkdir(parents=True, exist_ok=True)
 
-        self.local_sample_copy = os.path.join(self.cache_dir, 'sample.csv')
+        self.local_sample_copy = self.cache_dir / 'sample.csv'
 
     def plot_cutout(self, ind, arcsec=20, interactive=False, **kwargs):
         """
@@ -54,9 +54,10 @@ class ParentSampleBase(abc.ABC):
 
         fn = kwargs.pop(
             "fn",
-            [os.path.join(self.plots_dir, f"{i}_{r[self.default_keymap['id']]}.pdf")
+            [self.plots_dir / f"{i}_{r[self.default_keymap['id']]}.pdf"
              for i, r in sel.iterrows()]
         )
+        self.plots_dir.mkdir(parents=True, exist_ok=True)
 
         logger.debug(f"\nRA: {ra}\nDEC: {dec}\nTITLE: {title}\nFN: {fn}")
         ou = list()
@@ -84,4 +85,5 @@ class ParentSampleBase(abc.ABC):
 
     def save_local(self):
         logger.debug(f"saving under {self.local_sample_copy}")
+        self.local_sample_copy.parent.mkdir(parents=True, exist_ok=True)
         self.df.to_csv(self.local_sample_copy)

{timewise-0.4.12 → timewise-0.5.2}/timewise/utils.py

@@ -13,14 +13,19 @@ from astropy.table import Table
 from PIL import Image
 from io import BytesIO
 import hashlib
+from threading import Thread
+from queue import Queue
+import sys
 
-
-from timewise.general import cache_dir, backoff_hndlr
+from timewise.general import backoff_hndlr, get_directories
 
 
 logger = logging.getLogger(__name__)
 mirong_url = 'http://staff.ustc.edu.cn/~jnac/data_public/wisevar.txt'
-local_copy = os.path.join(cache_dir, 'mirong_sample.csv')
+
+
+def get_mirong_path():
+    return get_directories()['cache_dir'] / 'mirong_sample.csv'
 
 
 @cache
@@ -30,7 +35,8 @@ def get_2d_gaussian_correction(cl):
 
 def get_mirong_sample():
 
-    if not os.path.isfile(local_copy):
+    mirong_path = get_mirong_path()
+    if not mirong_path.is_file():
 
         logger.info(f'getting MIRONG sample from {mirong_url}')
         r = requests.get(mirong_url)
@@ -45,16 +51,16 @@ def get_mirong_sample():
         mirong_sample = pd.DataFrame(lll[1:-1], columns=lll[0])
         mirong_sample['ra'] = mirong_sample['RA']
         mirong_sample['dec'] = mirong_sample['DEC']
-        logger.debug(f'saving to {local_copy}')
-
-        mirong_sample.to_csv(local_copy, index=False)
         logger.info(f'found {len(mirong_sample)} objects in MIRONG Sample')
+
         mirong_sample.drop(columns=['ra', 'dec'], inplace=True)
-        mirong_sample.to_csv(local_copy, index=False)
+        logger.debug(f'saving to {mirong_path}')
+        mirong_path.parent.mkdir(parents=True, exist_ok=True)
+        mirong_sample.to_csv(mirong_path, index=False)
 
     else:
-        logger.debug(f'loading {local_copy}')
-        mirong_sample = pd.read_csv(local_copy)
+        logger.debug(f'loading {mirong_path}')
+        mirong_sample = pd.read_csv(mirong_path)
 
     return mirong_sample
 
@@ -154,9 +160,10 @@ class PanSTARRSQueryError(Exception):
 def load_cache_or_download(url):
     logger.debug(f"loading or downloading {url}")
     h = hashlib.md5(url.encode()).hexdigest()
-    cache_file = os.path.join(cache_dir, h + ".cache")
+    cache_dir = get_directories()['cache_dir']
+    cache_file = cache_dir / (h + ".cache")
     logger.debug(f"cache file is {cache_file}")
-    if not os.path.isfile(cache_file):
+    if not cache_file.is_file():
         logger.debug(f"downloading {url}")
         r = requests.get(url)
         with open(cache_file, 'wb') as f:
@@ -465,3 +472,47 @@ class StableTAPService(vo.dal.TAPService):
 #######################################################
 #            END CUSTOM TAP Service                   #
 ###########################################################################################################
+
+
+###########################################################################################################
+#            START CUSTOM TAP Service                 #
+#######################################################
+
+
+class ErrorQueue(Queue):
+    """Queue subclass whose join() re-raises exceptions from worker threads."""
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.error_queue = Queue()
+
+    def report_error(self, exc_info):
+        """Called by workers to push an exception into the error queue."""
+        self.error_queue.put(exc_info)
+        # Also decrement unfinished_tasks, so join() won't block forever
+        with self.all_tasks_done:
+            self.unfinished_tasks = max(0, self.unfinished_tasks - 1)
+            self.all_tasks_done.notify_all()
+
+    def join(self):
+        """Wait until all tasks are done, or raise if a worker failed."""
+        with self.all_tasks_done:
+            while self.unfinished_tasks:
+                if not self.error_queue.empty():
+                    exc_info = self.error_queue.get()
+                    raise exc_info[1].with_traceback(exc_info[2])
+                self.all_tasks_done.wait()
+
+
+class ExceptionSafeThread(Thread):
+    """Thread subclass that reports uncaught exceptions to the ErrorQueue."""
+
+    def __init__(self, error_queue: ErrorQueue, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.error_queue = error_queue
+
+    def run(self):
+        try:
+            super().run()
+        except Exception:
+            self.error_queue.report_error(sys.exc_info())

{timewise-0.4.12 → timewise-0.5.2}/timewise/wise_bigdata_desy_cluster.py

@@ -1,5 +1,4 @@
 import getpass
-import glob
 import os
 import json
 import subprocess
@@ -15,6 +14,7 @@ import shutil
 import gc
 import tqdm
 import sys
+from pathlib import Path
 
 from functools import cache
 from scipy.stats import chi2, f
@@ -28,8 +28,9 @@ import logging
 
 from typing import List
 
-from timewise.general import data_dir, bigdata_dir, backoff_hndlr
+from timewise.general import get_directories, backoff_hndlr
 from timewise.wise_data_by_visit import WiseDataByVisit
+from timewise.utils import StableAsyncTAPJob, ErrorQueue, ExceptionSafeThread
 
 
 logger = logging.getLogger(__name__)
@@ -41,9 +42,9 @@ class WISEDataDESYCluster(WiseDataByVisit):
     In addition to the attributes of `WiseDataByVisit` this class has the following attributes:
 
     :param executable_filename: the filename of the executable that will be submitted to the cluster
-    :type executable_filename: str
+    :type executable_filename: Path
     :param submit_file_filename: the filename of the submit file that will be submitted to the cluster
-    :type submit_file_filename: str
+    :type submit_file_filename: Path
     :param job_id: the job id of the submitted job
     :type job_id: str
     :param cluster_jobID_map: a dictionary mapping the chunk number to the cluster job id
@@ -51,13 +52,16 @@ class WISEDataDESYCluster(WiseDataByVisit):
     :param clusterJob_chunk_map: a dictionary mapping the cluster job id to the chunk number
     :type clusterJob_chunk_map: dict
     :param cluster_info_file: the filename of the file that stores the cluster info, loaded by the cluster jobs
-    :type cluster_info_file: str
+    :type cluster_info_file: Path
     :param start_time: the time when the download started
     :type start_time: float
     """
     status_cmd = f'qstat -u {getpass.getuser()}'
     # finding the file that contains the setup function
-    BASHFILE = os.getenv('TIMEWISE_DESY_CLUSTER_BASHFILE', os.path.expanduser('~/.bashrc'))
+    if (env_file := os.getenv('TIMEWISE_DESY_CLUSTER_BASHFILE')) is not None:
+        BASHFILE = Path(env_file)
+    else:
+        BASHFILE = Path("~/.bashrc").expanduser()
 
     def __init__(
             self,
@@ -89,13 +93,14 @@ class WISEDataDESYCluster(WiseDataByVisit):
 
         # set up cluster stuff
         self._status_output = None
-        self.executable_filename = os.path.join(self.cluster_dir, "run_timewise.sh")
-        self.submit_file_filename = os.path.join(self.cluster_dir, "submit_file.submit")
+        directories = get_directories()
+        self.executable_filename = self.cluster_dir / "run_timewise.sh"
+        self.submit_file_filename = self.cluster_dir / "submit_file.submit"
         self.job_id = None
 
         self.cluster_jobID_map = None
         self.clusterJob_chunk_map = None
-        self.cluster_info_file = os.path.join(self.cluster_dir, 'cluster_info.pkl')
+        self.cluster_info_file = self.cluster_dir / 'cluster_info.pkl'
         self._overwrite = True
 
         # these attributes will be set later and are used to pass them to the threads
@@ -121,9 +126,10 @@ class WISEDataDESYCluster(WiseDataByVisit):
         fn = super(WISEDataDESYCluster, self)._data_product_filename(service, chunk_number=chunk_number, jobID=jobID)
 
         if use_bigdata_dir:
-            fn = fn.replace(data_dir, bigdata_dir)
+            d = get_directories()
+            fn = str(fn).replace(str(d["data_dir"]), str(d["bigdata_dir"]))
 
-        return fn + ".gz"
+        return Path(str(fn) + ".gz")
 
     def load_data_product(
             self,
@@ -199,7 +205,7 @@ class WISEDataDESYCluster(WiseDataByVisit):
     def get_sample_photometric_data(self, max_nTAPjobs=8, perc=1, tables=None, chunks=None,
                                     cluster_jobs_per_chunk=100, wait=5, remove_chunks=False,
                                     query_type='positional', overwrite=True,
-                                    storage_directory=bigdata_dir,
+                                    storage_directory=None,
                                     node_memory='8G',
                                     skip_download=False,
                                     skip_input=False,
@@ -225,8 +231,8 @@ class WISEDataDESYCluster(WiseDataByVisit):
         :type query_type: str
         :param overwrite: overwrite already existing lightcurves and metadata
         :type overwrite: bool
-        :param storage_directory: move binned files and raw data here after work is done
-        :type storage_directory: str
+        :param storage_directory: move binned files and raw data here after work is done, defaults to TIMEWISE_BIGDATA_DIR
+        :type storage_directory: str | Path
         :param node_memory: memory per node on the cluster, default is 8G
         :type node_memory: str
         :param skip_download: if True, assume data is already downloaded, only do binning in that case
@@ -281,25 +287,29 @@ class WISEDataDESYCluster(WiseDataByVisit):
         self.clear_cluster_log_dir()
         self._save_cluster_info()
         self._overwrite = overwrite
-        self._storage_dir = storage_directory
+        self._storage_dir = get_directories()['bigdata_dir'] if storage_directory is None else Path(storage_directory)
 
         # --------------------------- set up queues --------------------------- #
 
         self.queue = queue.Queue()
-        self._tap_queue = queue.Queue()
-        self._cluster_queue = queue.Queue()
+        self._tap_queue = ErrorQueue()
+        self._cluster_queue = ErrorQueue()
         self._io_queue = queue.PriorityQueue()
         self._io_queue_done = queue.Queue()
-        self._combining_queue = queue.Queue()
+        self._combining_queue = ErrorQueue()
 
         # --------------------------- starting threads --------------------------- #
 
-        tap_threads = [threading.Thread(target=self._tap_thread, daemon=True, name=f"TAPThread{_}")
-                       for _ in range(max_nTAPjobs)]
-        cluster_threads = [threading.Thread(target=self._cluster_thread, daemon=True, name=f"ClusterThread{_}")
-                           for _ in range(max_nTAPjobs)]
+        tap_threads = [
+            ExceptionSafeThread(error_queue=self._tap_queue, target=self._tap_thread, daemon=True, name=f"TAPThread{_}")
+            for _ in range(max_nTAPjobs)
+        ]
+        cluster_threads = [
+            ExceptionSafeThread(error_queue=self._cluster_queue, target=self._cluster_thread, daemon=True, name=f"ClusterThread{_}")
+            for _ in range(max_nTAPjobs)
+        ]
         io_thread = threading.Thread(target=self._io_thread, daemon=True, name="IOThread")
-        combining_thread = threading.Thread(target=self._combining_thread, daemon=True, name="CombiningThread")
+        combining_thread = ExceptionSafeThread(error_queue=self._combining_queue, target=self._combining_thread, daemon=True, name="CombiningThread")
         status_thread = threading.Thread(target=self._status_thread, daemon=True, name='StatusThread')
 
         for t in tap_threads + cluster_threads + [io_thread, combining_thread]:
@@ -349,13 +359,13 @@ class WISEDataDESYCluster(WiseDataByVisit):
     )
     def _wait_for_job(self, t, i):
         logger.info(f"Waiting on {i}th query of {t} ........")
-        _job = self.tap_jobs[t][i]
+        _job = StableAsyncTAPJob(url=self.tap_jobs[t][i])
         _job.wait()
         logger.info(f'{i}th query of {t}: Done!')
 
     def _get_results_from_job(self, t, i):
         logger.debug(f"getting results for {i}th query of {t} .........")
-        _job = self.tap_jobs[t][i]
+        _job = StableAsyncTAPJob(url=self.tap_jobs[t][i])
         lightcurve = _job.fetch_result().to_table().to_pandas()
         fn = self._chunk_photometry_cache_filename(t, i)
         table_nice_name = self.get_db_name(t, nice=True)
@@ -431,8 +441,8 @@ class WISEDataDESYCluster(WiseDataByVisit):
                 # --------------  get results of TAP job for chunk i-1 ------------- #
                 if i > 0:
                     t_before = tables[i - 1]
-
-                    if self.tap_jobs[t_before][chunk].phase == "COMPLETED":
+                    phase = StableAsyncTAPJob(url=self.tap_jobs[t_before][chunk]).phase
+                    if phase == "COMPLETED":
                         result_method = "_get_results_from_job"
                         result_args = [t_before, chunk]
                         self._io_queue.put((2, result_method, result_args))
@@ -441,7 +451,7 @@ class WISEDataDESYCluster(WiseDataByVisit):
                     else:
                         logger.warning(
                             f"No completion for {chunk}th query of {t_before}! "
-                            f"Phase is {self.tap_jobs[t_before][chunk].phase}!"
+                            f"Phase is {phase}!"
                         )
                         submit_to_cluster = False
 
@@ -466,19 +476,16 @@ class WISEDataDESYCluster(WiseDataByVisit):
             gc.collect()
 
     def _move_file_to_storage(self, filename):
-        dst_fn = filename.replace(data_dir, self._storage_dir)
-
-        dst_dir = os.path.dirname(dst_fn)
-        if not os.path.isdir(dst_dir):
-            logger.debug(f"making directory {dst_dir}")
-            os.makedirs(dst_dir)
+        data_dir = str(get_directories()['data_dir'])
+        dst_fn = Path(str(filename).replace(str(data_dir), str(self._storage_dir)))
+        dst_fn.parent.mkdir(parents=True, exist_ok=True)
 
         logger.debug(f"copy {filename} to {dst_fn}")
 
         try:
             shutil.copy2(filename, dst_fn)
 
-            if os.path.getsize(filename) == os.path.getsize(dst_fn):
+            if Path(filename).stat().st_size == dst_fn.stat().st_size:
                 logger.debug(f"copy successful, removing {filename}")
                 os.remove(filename)
             else:
@@ -509,8 +516,8 @@ class WISEDataDESYCluster(WiseDataByVisit):
                 self.wait_for_job(job_id)
                 logger.debug(f'cluster done for chunk {chunk} (Cluster job {job_id}).')
 
-                log_files = glob.glob(f"./{job_id}_*")
-                log_files_abs = [os.path.abspath(p) for p in log_files]
+                log_files = Path("./").glob(f"{job_id}_*")
+                log_files_abs = [p.absolute() for p in log_files]
                 logger.debug(f"moving {len(log_files_abs)} log files to {self.cluster_log_dir}")
                 for f in log_files_abs:
                     shutil.move(f, self.cluster_log_dir)
@@ -708,9 +715,9 @@ class WISEDataDESYCluster(WiseDataByVisit):
         """
         Clears the directory where cluster logs are stored
         """
-        fns = os.listdir(self.cluster_log_dir)
+        fns = self.cluster_log_dir.glob("*")
         for fn in fns:
-            os.remove(os.path.join(self.cluster_log_dir, fn))
+            (self.cluster_log_dir / fn).unlink()
 
     def make_executable_file(self):
         """
@@ -729,8 +736,8 @@ class WISEDataDESYCluster(WiseDataByVisit):
             f'--mask_by_position $2'
         )
 
-        logger.debug("writing executable to " + self.executable_filename)
-        with open(self.executable_filename, "w") as f:
+        logger.debug("writing executable to " + str(self.executable_filename))
+        with self.executable_filename.open("w") as f:
             f.write(txt)
 
     def get_submit_file_filename(self, ids):
@@ -744,7 +751,7 @@ class WISEDataDESYCluster(WiseDataByVisit):
         """
         ids = np.atleast_1d(ids)
         ids_string = f"{min(ids)}-{max(ids)}"
-        return os.path.join(self.cluster_dir, f"ids{ids_string}.submit")
+        return self.cluster_dir / f"ids{ids_string}.submit"
 
     def make_submit_file(
             self,
@@ -764,6 +771,9 @@ class WISEDataDESYCluster(WiseDataByVisit):
         """
 
         q = "1 job_id in " + ", ".join(np.atleast_1d(job_ids).astype(str))
+        d = get_directories()
+        data_dir = str(d['data_dir'])
+        bigdata_dir = str(d['bigdata_dir'])
 
         text = (
             f"executable = {self.executable_filename} \n"
@@ -939,7 +949,7 @@ class WISEDataDESYCluster(WiseDataByVisit):
         _lc = lc if plot_binned else None
 
         if not fn:
-            fn = os.path.join(self.plots_dir, f"{parent_sample_idx}_{lum_key}.pdf")
+            fn = self.plots_dir / f"{parent_sample_idx}_{lum_key}.pdf"
 
         return self._plot_lc(lightcurve=_lc, unbinned_lc=unbinned_lc, interactive=interactive, fn=fn, ax=ax,
                              save=save, lum_key=lum_key, **kwargs)
@@ -1173,10 +1183,8 @@ class WISEDataDESYCluster(WiseDataByVisit):
                 chunk_str = "chunks_" + "_".join([str(c) for c in chunks]) \
                     if len(chunks) != self.n_chunks \
                     else "all_chunks"
-                fn = os.path.join(self.plots_dir, f"chi2_plots", lum_key, f"{n}_datapoints_{kind}_{chunk_str}.pdf")
-                d = os.path.dirname(fn)
-                if not os.path.isdir(d):
-                    os.makedirs(d)
+                fn = self.plots_dir / f"chi2_plots" / lum_key / f"{n}_datapoints_{kind}_{chunk_str}.pdf"
+                fn.parent.mkdir(parents=True, exist_ok=True)
                 logger.debug(f"saving under {fn}")
                 fig.savefig(fn)
 
@@ -1337,10 +1345,8 @@ class WISEDataDESYCluster(WiseDataByVisit):
             chunk_str = "chunks_" + "_".join([str(c) for c in chunks]) \
                 if len(chunks) != self.n_chunks \
                 else "all_chunks"
-            fn = os.path.join(self.plots_dir, f"coverage_plots", lum_key, f"{chunk_str}.pdf")
-            d = os.path.dirname(fn)
-            if not os.path.isdir(d):
-                os.makedirs(d)
+            fn = self.plots_dir / f"coverage_plots" / lum_key / f"{chunk_str}.pdf"
+            fn.parent.mkdir(parents=True, exist_ok=True)
             logger.debug(f"saving under {fn}")
             fig.savefig(fn)
 

{timewise-0.4.12 → timewise-0.5.2}/timewise/wise_data_base.py

@@ -1,4 +1,6 @@
 import abc
+import sys
+
 import backoff
 import copy
 import json
@@ -11,6 +13,7 @@ import subprocess
 import threading
 import time
 import tqdm
+from pathlib import Path
 
 import astropy.units as u
 import matplotlib.pyplot as plt
@@ -25,8 +28,8 @@ from astropy.table import Table
 from astropy.coordinates.angle_utilities import angular_separation, position_angle
 from sklearn.cluster import HDBSCAN
 
-from timewise.general import cache_dir, plots_dir, output_dir, logger_format, backoff_hndlr
-from timewise.utils import StableTAPService
+from timewise.general import get_directories, logger_format, backoff_hndlr
+from timewise.utils import StableAsyncTAPJob, StableTAPService
 
 logger = logging.getLogger(__name__)
 
@@ -53,20 +56,20 @@ class WISEDataBase(abc.ABC):
     :param parent_sample_default_entries: default entries for the parent sample
     :type parent_sample_default_entries: dict
     :param cache_dir: directory for cached data
-    :type cache_dir: str
+    :type cache_dir: Path
     :param cluster_dir: directory for cluster data
     :param cluster_log_dir: directory for cluster logs
-    :type cluster_dir: str
+    :type cluster_dir: Path
     :param output_dir: directory for output data
-    :type output_dir: str
+    :type output_dir: Path
     :param lightcurve_dir: directory for lightcurve data
-    :type lightcurve_dir: str
+    :type lightcurve_dir: Path
     :param plots_dir: directory for plots
-    :type plots_dir: str
+    :type plots_dir: Path
     :param submit_file: file for cluster submission
-    :type submit_file: str
-    :param tap_jobs: TAP jobs
-    :type tap_jobs: list[pyvo.dal.tap.TAPJob]
+    :type submit_file: Path
+    :param tap_jobs: TAP job URLs
+    :type tap_jobs: list[str]
     :param queue: queue for cluster jobs
     :type queue: multiprocessing.Queue
     :param clear_unbinned_photometry_when_binning: whether to clear unbinned photometry when binning
@@ -191,9 +194,11 @@ class WISEDataBase(abc.ABC):
         'W2': 0.280
     }
 
-    _this_dir = os.path.abspath(os.path.dirname(__file__))
-    magnitude_zeropoints_corrections = ascii.read(f'{_this_dir}/wise_flux_conversion_correction.dat',
-                                                  delimiter='\t').to_pandas()
+    _this_dir = Path(__file__).absolute().parent
+    magnitude_zeropoints_corrections = ascii.read(
+        _this_dir / 'wise_flux_conversion_correction.dat',
+        delimiter='\t'
+    ).to_pandas()
 
     band_wavelengths = {
         'W1': 3.368 * 1e-6 * u.m,
@@ -213,7 +218,7 @@ class WISEDataBase(abc.ABC):
     parent_sample_wise_skysep_key = 'sep_to_WISE_source'
 
     def __init__(self,
-                 base_name,
+                 base_name: str,
                  parent_sample_class,
                  min_sep_arcsec,
                  n_chunks):
@@ -228,6 +233,8 @@ class WISEDataBase(abc.ABC):
         :type min_sep_arcsec: float
         :param n_chunks: number of chunks in declination
         :type n_chunks: int
+        :param tap_url_cache_name: TAP job URLs are stored here to be able to resume them
+        :type tap_url_cache_name: str
         """
 
         #######################################################################################
@@ -253,26 +260,26 @@ class WISEDataBase(abc.ABC):
         # --------------------------- ^^^^ set up parent sample ^^^^ --------------------------- #
 
         # set up directories
-        self.cache_dir = os.path.join(cache_dir, base_name)
-        self._cache_photometry_dir = os.path.join(self.cache_dir, "photometry")
-        self.cluster_dir = os.path.join(self.cache_dir, 'cluster')
-        self.cluster_log_dir = os.path.join(self.cluster_dir, 'logs')
-        self.output_dir = os.path.join(output_dir, base_name)
-        self.lightcurve_dir = os.path.join(self.output_dir, "lightcurves")
-        self.plots_dir = os.path.join(plots_dir, base_name)
+        directories = get_directories()  # type: dict[str, Path]
+        self.cache_dir = directories['cache_dir'] / base_name
+        self._cache_photometry_dir = self.cache_dir / "photometry"
+        self.cluster_dir = self.cache_dir / 'cluster'
+        self.cluster_log_dir = self.cluster_dir / 'logs'
+        self.output_dir = directories["output_dir"] / base_name
+        self.lightcurve_dir = self.output_dir / "lightcurves"
+        self.plots_dir = directories["plots_dir"] / base_name
+        self.tap_jobs_cache_dir = self.cache_dir / 'tap_cache'
 
         for d in [self.cache_dir, self._cache_photometry_dir, self.cluster_dir, self.cluster_log_dir,
                   self.output_dir, self.lightcurve_dir, self.plots_dir]:
-            if not os.path.isdir(d):
-                logger.debug(f"making directory {d}")
-                os.makedirs(d)
+            d.mkdir(parents=True, exist_ok=True)
 
-        file_handler = logging.FileHandler(filename=self.cache_dir + '/log.err', mode="a")
+        file_handler = logging.FileHandler(filename=self.cache_dir / 'log.err', mode="a")
         file_handler.setLevel("WARNING")
         file_handler.setFormatter(logger_format)
         logger.addHandler(file_handler)
 
-        self.submit_file = os.path.join(self.cluster_dir, 'submit.txt')
+        self.submit_file = self.cluster_dir / 'submit.txt'
 
         # set up result attributes
         self._split_chunk_key = '__chunk'
@@ -497,7 +504,7 @@ class WISEDataBase(abc.ABC):
         if err_msg:
             logger.error(err_msg.decode())
         process.terminate()
-        if os.path.isfile(out_file):
+        if Path(out_file).is_file():
             return 1
         else:
             return 0
@@ -566,8 +573,8 @@ class WISEDataBase(abc.ABC):
 
         dec_intervall_mask = self.chunk_map == chunk_number
         logger.debug(f"Any selected: {np.any(dec_intervall_mask)}")
-        _parent_sample_declination_band_file = os.path.join(self.cache_dir, f"parent_sample_chunk{chunk_number}.xml")
-        _output_file = os.path.join(self.cache_dir, f"parent_sample_chunk{chunk_number}.tbl")
+        _parent_sample_declination_band_file = self.cache_dir / f"parent_sample_chunk{chunk_number}.xml"
+        _output_file = self.cache_dir / f"parent_sample_chunk{chunk_number}.tbl"
 
         additional_keys = (
             "," + ",".join(additional_columns)
@@ -637,7 +644,7 @@ class WISEDataBase(abc.ABC):
     # START GET PHOTOMETRY DATA       #
     ###################################
 
-    def get_photometric_data(self, tables=None, perc=1, wait=0, service=None, nthreads=100,
+    def get_photometric_data(self, tables=None, perc=1, service=None, nthreads=100,
                              chunks=None, overwrite=True, remove_chunks=False, query_type='positional',
                              skip_download=False, mask_by_position=False):
         """
@@ -645,6 +652,11 @@ class WISEDataBase(abc.ABC):
 
             </path/to/timewise/data/dir>/output/<base_name>/lightcurves/binned_lightcurves_<service>.json
 
+        If service is 'tap' then the process exists on the first call to give the jobs running on the IRSA
+        servers some time. The job infos are cached and loaded on the next function call. `timewise` will
+        then wait on the jobs to finish. If the process is terminated via the keyboard during the waiting
+        the TAP connections will also be cached to be resumed at a later time.
+
         :param remove_chunks: remove single chunk files after binning
         :type remove_chunks: bools
         :param overwrite: overwrite already existing lightcurves and metadata
@@ -657,8 +669,6 @@ class WISEDataBase(abc.ABC):
         :type nthreads: int
         :param service: either of 'gator' or 'tap', selects base on elements per chunk by default
         :type service: str
-        :param wait: time in hours to wait after submitting TAP jobs
-        :type wait: float
         :param chunks: containing indices of chunks to download
         :type chunks: list-like
         :param query_type: 'positional': query photometry based on distance from object, 'by_allwise_id': select all photometry points within a radius of 50 arcsec with the corresponding AllWISE ID
@@ -667,6 +677,8 @@ class WISEDataBase(abc.ABC):
         :type skip_download: bool
         :param mask_by_position: if `True` mask single exposures that are too far away from the bulk
         :type mask_by_position: bool
+        :return: The status of the processing
+        :rtype: bool
         """
 
         mag = True
@@ -703,32 +715,40 @@ class WISEDataBase(abc.ABC):
                          f"from {tables}")
 
             if service == 'tap':
-                self._query_for_photometry(tables, chunks, wait, mag, flux, nthreads, query_type)
+                done = self._query_for_photometry(tables, chunks, mag, flux, nthreads, query_type)
+                if not done:
+                    logger.info("Some TAP jobs still running. Exit here and resume later.")
+                    return False
 
             elif service == 'gator':
                 self._query_for_photometry_gator(tables, chunks, mag, flux, nthreads)
+            else:
+                raise ValueError(f"Unknown service {service}! Choose one of 'tap' or 'gator'")
 
         else:
             logger.info("skipping download, assume data is already downloaded.")
 
+        logger.info("Download done, processing lightcurves")
         self._select_individual_lightcurves_and_bin(service=service, chunks=chunks, mask_by_position=mask_by_position)
         for c in chunks:
             self.calculate_metadata(service=service, chunk_number=c, overwrite=True)
 
         self._combine_data_products(service=service, remove=remove_chunks, overwrite=overwrite)
 
+        return True
+
     def _data_product_filename(self, service, chunk_number=None, jobID=None):
 
         n = "timewise_data_product_"
 
         if (chunk_number is None) and (jobID is None):
-            return os.path.join(self.lightcurve_dir, f"{n}{service}.json")
+            return self.lightcurve_dir / f"{n}{service}.json"
         else:
             fn = f"{n}{service}{self._split_chunk_key}{chunk_number}"
             if (chunk_number is not None) and (jobID is None):
-                return os.path.join(self._cache_photometry_dir, fn + ".json")
+                return self._cache_photometry_dir / (fn + ".json")
             else:
-                return os.path.join(self._cache_photometry_dir, fn + f"_{jobID}.json")
+                return self._cache_photometry_dir / (fn + f"_{jobID}.json")
 
     @staticmethod
     def _verify_contains_lightcurves(data_product):
@@ -880,7 +900,7 @@ class WISEDataBase(abc.ABC):
         _ending = '.xml' if gator_input else'.tbl'
         fn = f"{self._cached_raw_photometry_prefix}_{table_name}{_additional_neowise_query}{_gator_input}" \
              f"{self._split_chunk_key}{chunk_number}{_ending}"
-        return os.path.join(self._cache_photometry_dir, fn)
+        return self._cache_photometry_dir / fn
 
     def _thread_query_photometry_gator(self, chunk_number, table_name, mag, flux):
         _infile = self._gator_chunk_photometry_cache_filename(table_name, chunk_number, gator_input=True)
@@ -943,9 +963,11 @@ class WISEDataBase(abc.ABC):
         for t in threads:
             t.join()
 
+        return True
+
     def _get_unbinned_lightcurves_gator(self, chunk_number, clear=False):
         # load only the files for this chunk
-        fns = [os.path.join(self._cache_photometry_dir, fn)
+        fns = [self._cache_photometry_dir / fn
                for fn in os.listdir(self._cache_photometry_dir)
                if (fn.startswith(self._cached_raw_photometry_prefix) and
                    fn.endswith(f"{self._split_chunk_key}{chunk_number}.tbl"))
@@ -957,12 +979,12 @@ class WISEDataBase(abc.ABC):
         for fn in fns:
             data_table = Table.read(fn, format='ipac').to_pandas()
 
-            t = 'allwise_p3as_mep' if 'allwise' in fn else 'neowiser_p1bs_psd'
+            t = 'allwise_p3as_mep' if 'allwise' in str(fn) else 'neowiser_p1bs_psd'
             nice_name = self.get_db_name(t, nice=True)
             cols = {'index_01': self._tap_orig_id_key}
             cols.update(self.photometry_table_keymap[nice_name]['mag'])
             cols.update(self.photometry_table_keymap[nice_name]['flux'])
-            if 'allwise' in fn:
+            if 'allwise' in str(fn):
                 cols['cntr_mf'] = 'allwise_cntr'
 
             data_table = data_table.rename(columns=cols)
@@ -982,6 +1004,62 @@ class WISEDataBase(abc.ABC):
     # START using TAP to get photometry        #
     # ---------------------------------------- #
 
+    @property
+    def tap_cache_filenames(self):
+        return (
+            self.tap_jobs_cache_dir / f"tap_jobs.json",
+            self.tap_jobs_cache_dir / f"queue.json"
+        )
+
+    def dump_tap_cache(self):
+        self.tap_jobs_cache_dir.mkdir(parents=True, exist_ok=True)
+
+        tap_jobs_fn, queue_fn = self.tap_cache_filenames
+        logger.debug(f"saving TAP jobs to {tap_jobs_fn}")
+        tap_jobs_fn.parent.mkdir(parents=True, exist_ok=True)
+        with tap_jobs_fn.open("w") as f:
+            json.dump(self.tap_jobs, f, indent=4)
+
+        queue_fn.parent.mkdir(parents=True, exist_ok=True)
+        logger.debug(f"saving queue to {queue_fn}")
+        with queue_fn.open("w") as f:
+            json.dump(list(self.queue.queue), f, indent=4)
+
+    def load_tap_cache(self):
+        tap_jobs_fn, queue_fn = self.tap_cache_filenames
+
+        logger.debug(f"loading TAP jobs from {tap_jobs_fn}")
+        if tap_jobs_fn.is_file():
+            with tap_jobs_fn.open("r") as f:
+                tap_jobs_json = json.load(f)
+            # JSON keys are always strings while we need the chunk numbers
+            # to be integers in the dictionary
+            self.tap_jobs = {
+                t: {int(i): url for i, url in v.items()} for t, v in tap_jobs_json.items()
+            }
+            logger.debug(f"removing {tap_jobs_fn}")
+            tap_jobs_fn.unlink()
+        else:
+            logger.warning(f"No file {tap_jobs_fn}")
+            self.tap_jobs = None
+
+        logger.debug(f"loading queue from {queue_fn}")
+        if queue_fn.is_file():
+            with queue_fn.open("r") as f:
+                ql = json.load(f)
+                logger.debug(f"loaded {len(ql)} queue elements")
+                self.queue = queue.Queue()
+                for q in ql:
+                    self.queue.put(q)
+                logger.debug(f"removing {queue_fn}")
+                queue_fn.unlink()
+        else:
+            logger.warning(f"No file {queue_fn}")
+            self.queue = None
+
+        cache_exists = (self.tap_jobs is not None) and (self.queue is not None)
+        return cache_exists
+
     def _get_photometry_query_string(self, table_name, mag, flux, query_type):
         """
         Construct a query string to submit to IRSA
@@ -1075,7 +1153,7 @@ class WISEDataBase(abc.ABC):
 
                 logger.info(f'submitted job for {t} for chunk {i}: ')
                 logger.debug(f'Job: {job.url}; {job.phase}')
-                self.tap_jobs[t][i] = job
+                self.tap_jobs[t][i] = job.url
                 self.queue.put((t, i))
                 break
 
@@ -1091,7 +1169,7 @@ class WISEDataBase(abc.ABC):
         _additional_neowise_query = '_neowise_gator' if additional_neowise_query else ''
         fn = f"{self._cached_raw_photometry_prefix}_{table_name}{_additional_neowise_query}" \
              f"{self._split_chunk_key}{chunk_number}.csv"
-        return os.path.join(self._cache_photometry_dir, fn)
+        return self._cache_photometry_dir / fn
 
     @staticmethod
     def _give_up_tap(e):
@@ -1107,7 +1185,7 @@ class WISEDataBase(abc.ABC):
     def _thread_wait_and_get_results(self, t, i):
         logger.info(f"Waiting on {i}th query of {t} ........")
 
-        _job = self.tap_jobs[t][i]
+        _job = StableAsyncTAPJob(url=self.tap_jobs[t][i])
         _job.wait()
         logger.info(f'{i}th query of {t}: Done!')
 
@@ -1136,7 +1214,7 @@ class WISEDataBase(abc.ABC):
                 logger.debug(f"No more queue. exiting")
                 break
 
-            job = self.tap_jobs[t][i]
+            job = StableAsyncTAPJob(url=self.tap_jobs[t][i])
 
             _ntries = 10
             while True:
@@ -1179,38 +1257,50 @@ class WISEDataBase(abc.ABC):
 
         try:
             self.queue.join()
+            logger.info('all tap_jobs done!')
         except KeyboardInterrupt:
-            pass
+            self.dump_tap_cache()
+            return False
+        finally:
+            for i, t in enumerate(threads):
+                logger.debug(f"{i}th thread alive: {t.is_alive()}")
+            for t in threads:
+                t.join()
+            self.tap_jobs = None
+            del threads
 
-        logger.info('all tap_jobs done!')
-        for i, t in enumerate(threads):
-            logger.debug(f"{i}th thread alive: {t.is_alive()}")
+        return True
 
-        for t in threads:
-            t.join()
-        self.tap_jobs = None
-        del threads
+    def _query_for_photometry(self, tables, chunks, mag, flux, nthreads, query_type):
+        # ----------------------------------------------------------------------
+        #     Load TAP cache if it exists
+        # ----------------------------------------------------------------------
+        cache_exists = self.load_tap_cache()
 
-    def _query_for_photometry(self, tables, chunks, wait, mag, flux, nthreads, query_type):
         # ----------------------------------------------------------------------
         #     Do the query
         # ----------------------------------------------------------------------
-        self.tap_jobs = dict()
-        self.queue = queue.Queue()
-        tables = np.atleast_1d(tables)
-
-        for t in tables:
-            self.tap_jobs[t] = dict()
-            for i in chunks:
-                self._submit_job_to_TAP(i, t, mag, flux, query_type)
-                time.sleep(5)
+        if not cache_exists:
+            self.tap_jobs = dict()
+            self.queue = queue.Queue() if self.queue is None else self.queue
+            tables = np.atleast_1d(tables)
+
+            for t in tables:
+                self.tap_jobs[t] = dict()
+                for i in chunks:
+                    self._submit_job_to_TAP(i, t, mag, flux, query_type)
+                    time.sleep(5)
+
+            logger.info(f'added {self.queue.qsize()} tasks to queue')
+            self.dump_tap_cache()
+            logger.info(f"wait some time to give tap_jobs some time")
+            return False
 
-        logger.info(f'added {self.queue.qsize()} tasks to queue')
-        logger.info(f"wait for {wait} hours to give tap_jobs some time")
-        time.sleep(wait * 3600)
+        logger.info(f'starting worker threads to retrieve results, {self.queue.qsize()} tasks in queue')
         nthreads = min(len(tables) * len(chunks), nthreads)
-        self._run_tap_worker_threads(nthreads)
+        success = self._run_tap_worker_threads(nthreads)
         self.queue = None
+        return success
 
     # ----------------------------------------------------------------------
     #     select individual lightcurves and bin
@@ -1263,7 +1353,7 @@ class WISEDataBase(abc.ABC):
         :type clear: bool, optional
         """
         # load only the files for this chunk
-        fns = [os.path.join(self._cache_photometry_dir, fn)
+        fns = [self._cache_photometry_dir / fn
                for fn in os.listdir(self._cache_photometry_dir)
                if (fn.startswith(self._cached_raw_photometry_prefix) and fn.endswith(
                 f"{self._split_chunk_key}{chunk_number}.csv"
@@ -1715,9 +1805,9 @@ class WISEDataBase(abc.ABC):
         """
 
         logger.info(f"getting position masks for {service}, chunk {chunk_number}")
-        fn = os.path.join(self.cache_dir, "position_masks", f"{service}_chunk{chunk_number}.json")
+        fn = self.cache_dir / "position_masks" / f"{service}_chunk{chunk_number}.json"
 
-        if not os.path.isfile(fn):
+        if not fn.is_file():
             logger.debug(f"No file {fn}. Calculating position masks.")
 
             if service == "tap":
@@ -1746,10 +1836,7 @@ class WISEDataBase(abc.ABC):
                 if len(bad_indices) > 0:
                     position_masks[str(i)] = bad_indices
 
-            d = os.path.dirname(fn)
-            if not os.path.isdir(d):
-                os.makedirs(d, exist_ok=True)
-
+            fn.parent.mkdir(exist_ok=True, parents=True)
             with open(fn, "w") as f:
                 json.dump(position_masks, f)
 
@@ -1820,7 +1907,7 @@ class WISEDataBase(abc.ABC):
         _lc = lc if plot_binned else None
 
         if not fn:
-            fn = os.path.join(self.plots_dir, f"{parent_sample_idx}_{lum_key}.pdf")
+            fn = self.plots_dir / f"{parent_sample_idx}_{lum_key}.pdf"
 
         return self._plot_lc(lightcurve=_lc, unbinned_lc=unbinned_lc, interactive=interactive, fn=fn, ax=ax,
                              save=save, lum_key=lum_key, **kwargs)

{timewise-0.4.12 → timewise-0.5.2}/timewise/wise_data_by_visit.py

@@ -662,7 +662,7 @@ class WiseDataByVisit(WISEDataBase):
 
         if save:
             if fn is None:
-                fn = os.path.join(self.plots_dir, f"{ind}_binning_diag_{which}cutout.pdf")
+                fn = self.plots_dir / f"{ind}_binning_diag_{which}cutout.pdf"
             logger.debug(f"saving under {fn}")
             fig.savefig(fn)
 

timewise-0.4.12/pyproject.toml

@@ -1,50 +0,0 @@
-[tool.poetry]
-name = "timewise"
-version = "0.4.12"
-description = "A small package to download infrared data from the WISE satellite"
-authors = ["Jannis Necker <jannis.necker@gmail.com>"]
-license = "MIT"
-readme = "README.md"
-documentation = "https://timewise.readthedocs.io/en/latest"
-repository = "https://github.com/JannisNe/timewise"
-
-[tool.poetry.scripts]
-timewise = "timewise.cli:timewise_cli"
-
-[tool.poetry.urls]
-"Bug Tracker" = "https://github.com/JannisNe/timewise/issues"
-
-[tool.poetry.dependencies]
-python = ">=3.8, <3.12"
-tqdm = "^4.64.0"
-requests = "^2.28.1"
-pandas = ">=1.4.3,<3.0.0"
-numpy = "^1.23.2"
-pyvo = "^1.4.1"
-astropy = "^5.1"
-matplotlib = "^3.5.3"
-scikit-image = ">=0.19.3,<0.21.0"
-backoff = "^2.1.2"
-virtualenv = "^20.16.3"
-seaborn = ">=0.11.2,<0.13.0"
-urllib3 = "1.26.15"
-pydantic = ">=1.9.0,<2.0.0"
-scikit-learn = "^1.3.0"
-jupyterlab = {version = "^4.0.6", extras = ["jupyter"]}
-jupyter = {version = "^1.0.0", extras = ["jupyter"]}
-
-[tool.poetry.dev-dependencies]
-coveralls = "^3.3.1"
-pytest = "^7.2.2"
-
-[tool.poetry.group.docs]
-optional = true
-
-[tool.poetry.group.docs.dependencies]
-myst-parser = ">=1,<3"
-sphinx-rtd-theme = "^1.3.0"
-
-
-[build-system]
-requires = ["poetry-core>=1.0.0"]
-build-backend = "poetry.core.masonry.api"

timewise-0.4.12/timewise/general.py

@@ -1,44 +0,0 @@
-import logging, os
-
-
-# Setting up the Logger
-main_logger = logging.getLogger('timewise')
-logger_format = logging.Formatter(
-    '%(levelname)s:%(name)s:%(funcName)s - [%(threadName)s] - %(asctime)s: \n\t%(message)s', "%H:%M:%S"
-)
-stream_handler = logging.StreamHandler()
-stream_handler.setFormatter(logger_format)
-main_logger.addHandler(stream_handler)
-main_logger.propagate = False  # do not propagate to root logger
-
-logger = logging.getLogger(__name__)
-
-# Setting up data directory
-DATA_DIR_KEY = 'TIMEWISE_DATA'
-if DATA_DIR_KEY in os.environ:
-    data_dir = os.path.expanduser(os.environ[DATA_DIR_KEY])
-else:
-    logger.warning(f'{DATA_DIR_KEY} not set! Using home directory.')
-    data_dir = os.path.expanduser('~/')
-
-BIGDATA_DIR_KEY = 'TIMEWISE_BIGDATA'
-if BIGDATA_DIR_KEY in os.environ:
-    bigdata_dir = os.path.expanduser(os.environ[BIGDATA_DIR_KEY])
-    logger.info(f"Using bigdata directory {bigdata_dir}")
-else:
-    bigdata_dir = None
-    logger.info(f"No bigdata directory set.")
-
-output_dir = os.path.join(data_dir, 'output')
-plots_dir = os.path.join(output_dir, 'plots')
-cache_dir = os.path.join(data_dir, 'cache')
-
-for d in [data_dir, output_dir, plots_dir, cache_dir]:
-    if not os.path.isdir(d):
-        os.mkdir(os.path.abspath(d))
-
-
-def backoff_hndlr(details):
-    logger.info("Backing off {wait:0.1f} seconds after {tries} tries "
-                "calling function {target} with args {args} and kwargs "
-                "{kwargs}".format(**details))