timewise 0.3.0rc6__tar.gz → 0.3.2__tar.gz

{timewise-0.3.0rc6 → timewise-0.3.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: timewise
-Version: 0.3.0rc6
+Version: 0.3.2
 Summary: A small package to download infrared data from the WISE satellite
 Home-page: https://github.com/JannisNe/timewise
 License: MIT

{timewise-0.3.0rc6 → timewise-0.3.2}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "timewise"
-version = "0.3.0rc6"
+version = "0.3.2"
 description = "A small package to download infrared data from the WISE satellite"
 authors = ["Jannis Necker <jannis.necker@gmail.com>"]
 license = "MIT"

{timewise-0.3.0rc6 → timewise-0.3.2}/timewise/wise_bigdata_desy_cluster.py

@@ -470,21 +470,28 @@ class WISEDataDESYCluster(WiseDataByVisit):
             logger.debug(f"combining chunk {chunk}")
 
             try:
-                self._combine_data_products('tap', chunk_number=chunk, remove=True, overwrite=self._overwrite)
-
-                if self._storage_dir:
-                    filenames_to_move = [
-                        self._data_product_filename(service='tap', chunk_number=chunk),
-                    ]
-
-                    for t in self.photometry_table_keymap.keys():
-                        filenames_to_move.append(self._chunk_photometry_cache_filename(t, chunk))
-
-                    for fn in filenames_to_move:
-                        try:
-                            self._move_file_to_storage(fn)
-                        except shutil.SameFileError as e:
-                            logger.error(f"{e}. Not moving.")
+                success = self._combine_data_products('tap', chunk_number=chunk, remove=True, overwrite=self._overwrite)
+
+                if success:
+                    if self._storage_dir:
+                        filenames_to_move = [
+                            self._data_product_filename(service='tap', chunk_number=chunk),
+                        ]
+
+                        for t in self.photometry_table_keymap.keys():
+                            filenames_to_move.append(self._chunk_photometry_cache_filename(t, chunk))
+    
+                        for fn in filenames_to_move:
+                            try:
+                                self._move_file_to_storage(fn)
+                            except shutil.SameFileError as e:
+                                logger.error(f"{e}. Not moving.")
+
+                else:
+                    msg = f"Chunk {chunk}: Combining data products not successfully!"
+                    if self._storage_dir:
+                        msg += " Not moving files to storage."
+                    logger.warning(msg)
 
             finally:
                 self._combining_queue.task_done()

{timewise-0.3.0rc6 → timewise-0.3.2}/timewise/wise_data_base.py

@@ -678,7 +678,13 @@ class WISEDataBase(abc.ABC):
         with open(fn, "w") as f:
             json.dump(data_product, f, indent=4)
 
-    def _combine_data_products(self, service=None, chunk_number=None, remove=False, overwrite=False):
+    def _combine_data_products(
+            self,
+            service=None,
+            chunk_number=None,
+            remove=False,
+            overwrite=False
+    ):
         if not service:
             logger.info("Combining all lightcuves collected with all services")
             itr = ['service', ['gator', 'tap']]
@@ -688,7 +694,7 @@ class WISEDataBase(abc.ABC):
             itr = ['chunk_number', range(self.n_chunks)]
             kwargs = {'service': service}
         elif chunk_number is not None:
-            logger.info(f"Combining all lightcurves collected eith {service} for chunk {chunk_number}")
+            logger.info(f"Combining all lightcurves collected with {service} for chunk {chunk_number}")
             itr = ['jobID',
                    list(self.clusterJob_chunk_map.index[self.clusterJob_chunk_map.chunk_number == chunk_number])]
             kwargs = {'service': service, 'chunk_number': chunk_number}
@@ -697,11 +703,13 @@ class WISEDataBase(abc.ABC):
 
         lcs = None
         fns = list()
+        missing_files = 0
         for i in itr[1]:
             kw = dict(kwargs)
             kw[itr[0]] = i
             kw['return_filename'] = True
             res = self.load_data_product(**kw)
+
             if not isinstance(res, type(None)):
                 ilcs, ifn = res
                 fns.append(ifn)
@@ -710,11 +718,28 @@ class WISEDataBase(abc.ABC):
                 else:
                     lcs.update(ilcs)
 
-        self._save_data_product(lcs, service=service, chunk_number=chunk_number, overwrite=overwrite)
+            else:
+                missing_files += 1
+
+            if missing_files > 0:
+                msg = f"Missing {missing_files} for {service}"
+                if chunk_number is not None:
+                    msg += f" chunk {chunk_number}"
+                msg += "! Not saving data product"
+                logger.warning(msg)
+                break
+
+        if missing_files == 0:
+            self._save_data_product(lcs, service=service, chunk_number=chunk_number, overwrite=overwrite)
 
-        if remove:
-            for fn in tqdm.tqdm(fns, desc="removing files"):
-                os.remove(fn)
+            if remove:
+                for fn in tqdm.tqdm(fns, desc="removing files"):
+                    os.remove(fn)
+
+            return True
+
+        else:
+            return False
 
     # ----------------------------------------------------------------------------------- #
     # START using GATOR to get photometry        #

{timewise-0.3.0rc6 → timewise-0.3.2}/timewise/wise_data_by_visit.py

@@ -327,12 +327,19 @@ class WiseDataByVisit(WISEDataBase):
                 metadata[Nk] = len(ilc)
 
                 if len(ilc) > 0:
-                    metadata[mean_weighted_ppb_key] = np.average(ilc[llumkey], weights=ilc[ppb_key])
-                    metadata[excess_variance_key], metadata[excess_variance_err_key] = get_excess_variance(
-                        np.array(ilc[llumkey]),
-                        np.array(ilc[errkey]),
-                        np.array(metadata[mean_weighted_ppb_key])
-                    )
+                    # check if ppb sum to more than zero
+                    if sum(ilc[ppb_key]) > 0:
+                        metadata[mean_weighted_ppb_key] = np.average(ilc[llumkey], weights=ilc[ppb_key])
+                        metadata[excess_variance_key], metadata[excess_variance_err_key] = get_excess_variance(
+                            np.array(ilc[llumkey]),
+                            np.array(ilc[errkey]),
+                            np.array(metadata[mean_weighted_ppb_key])
+                        )
+
+                    # if not we can not calculate the excess variance
+                    else:
+                        for k in [mean_weighted_ppb_key, excess_variance_key, excess_variance_err_key]:
+                            metadata[k] = np.nan
 
                     imin = ilc[llumkey].min()
                     imax = ilc[llumkey].max()

timewise-0.3.0rc6/setup.py

@@ -1,43 +0,0 @@
-# -*- coding: utf-8 -*-
-from setuptools import setup
-
-packages = \
-['timewise']
-
-package_data = \
-{'': ['*']}
-
-install_requires = \
-['astropy>=5.1,<6.0',
- 'backoff>=2.1.2,<3.0.0',
- 'coveralls>=3.3.1,<4.0.0',
- 'furo>=2022.6.21,<2023.0.0',
- 'matplotlib>=3.5.3,<4.0.0',
- 'myst-parser>=0.18.0,<0.19.0',
- 'numpy>=1.23.2,<2.0.0',
- 'pandas>=1.4.3,<2.0.0',
- 'pyvo>=1.3,<2.0',
- 'requests>=2.28.1,<3.0.0',
- 'scikit-image>=0.19.3,<0.20.0',
- 'seaborn>=0.11.2,<0.13.0',
- 'tqdm>=4.64.0,<5.0.0',
- 'virtualenv>=20.16.3,<21.0.0']
-
-setup_kwargs = {
-    'name': 'timewise',
-    'version': '0.3.0rc6',
-    'description': 'A small package to download infrared data from the WISE satellite',
-    'long_description': "[![CI](https://github.com/JannisNe/timewise/actions/workflows/continous_integration.yml/badge.svg)](https://github.com/JannisNe/timewise/actions/workflows/continous_integration.yml)\n[![Coverage Status](https://coveralls.io/repos/github/JannisNe/timewise/badge.svg?branch=main)](https://coveralls.io/github/JannisNe/timewise?branch=main)\n[![PyPI version](https://badge.fury.io/py/timewise.svg)](https://badge.fury.io/py/timewise)\n[![Documentation Status](https://readthedocs.org/projects/timewise/badge/?version=latest)](https://timewise.readthedocs.io/en/latest/?badge=latest)\n[![DOI](https://zenodo.org/badge/449677569.svg)](https://zenodo.org/badge/latestdoi/449677569)\n\n\n\n# `timewise` is great, love it!\nDownload infrared lightcurves recorded with the WISE satellite.\n\n## Installation\n\n`timewise` is a python package, installable through `pip`\n```\npip install timewise\n```\n\nIf you would like to contribute just clone the repository. Easy.\n\n\n## Dependencies\n\nAll dependencies are listed in `requirements.txt`. If installing with `pip` they will automatically installed.\nOtherwise you can install them with `pip install -r requirements.txt`.\n\nThere is one package that does not obey! It's `SciServer`! \nIt's used to access SDSS data and plot cutouts. If you want to use this functionality \ninstall [this](https://github.com/sciserver/SciScript-Python) and create an account [here](https://www.sciserver.org).\nAs soon as required you will be required to enter your username and password.\n\n\n## Testing\n You can verify that everything is working (because this package is flawless and works everywhere.) by executing\n the unittest\n```\npython -m unittest discover tests/\n```\n\n## Cite\nIf you you `timewise` please cite [this](https://zenodo.org/badge/latestdoi/449677569).\n\n## Usage\nDetailed documentation can be found [here](https://timewise.readthedocs.io/en/latest/)\n",
-    'author': 'Jannis Necker',
-    'author_email': 'jannis.necker@gmail.com',
-    'maintainer': 'None',
-    'maintainer_email': 'None',
-    'url': 'https://github.com/JannisNe/timewise',
-    'packages': packages,
-    'package_data': package_data,
-    'install_requires': install_requires,
-    'python_requires': '>=3.8,<3.12',
-}
-
-
-setup(**setup_kwargs)