timewise 0.4.4__tar.gz → 0.4.7__tar.gz

{timewise-0.4.4 → timewise-0.4.7}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: timewise
-Version: 0.4.4
+Version: 0.4.7
 Summary: A small package to download infrared data from the WISE satellite
 Home-page: https://github.com/JannisNe/timewise
 License: MIT
@@ -15,6 +15,8 @@ Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
 Requires-Dist: astropy (>=5.1,<6.0)
 Requires-Dist: backoff (>=2.1.2,<3.0.0)
+Requires-Dist: jupyter[jupyter] (>=1.0.0,<2.0.0)
+Requires-Dist: jupyterlab[jupyter] (>=4.0.6,<5.0.0)
 Requires-Dist: matplotlib (>=3.5.3,<4.0.0)
 Requires-Dist: numpy (>=1.23.2,<2.0.0)
 Requires-Dist: pandas (>=1.4.3,<3.0.0)

{timewise-0.4.4 → timewise-0.4.7}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "timewise"
-version = "0.4.4"
+version = "0.4.7"
 description = "A small package to download infrared data from the WISE satellite"
 authors = ["Jannis Necker <jannis.necker@gmail.com>"]
 license = "MIT"
@@ -30,6 +30,8 @@ seaborn = ">=0.11.2,<0.13.0"
 urllib3 = "1.26.15"
 pydantic = ">=1.9.0,<2.0.0"
 scikit-learn = "^1.3.0"
+jupyterlab = {version = "^4.0.6", extras = ["jupyter"]}
+jupyter = {version = "^1.0.0", extras = ["jupyter"]}
 
 [tool.poetry.dev-dependencies]
 coveralls = "^3.3.1"
@@ -39,7 +41,7 @@ pytest = "^7.2.2"
 optional = true
 
 [tool.poetry.group.docs.dependencies]
-myst-parser = "^1.0.0"
+myst-parser = ">=1,<3"
 sphinx-rtd-theme = "^1.3.0"
 
 

{timewise-0.4.4 → timewise-0.4.7}/timewise/__init__.py

@@ -2,4 +2,4 @@ from timewise.wise_data_by_visit import WiseDataByVisit
 from timewise.wise_bigdata_desy_cluster import WISEDataDESYCluster
 from timewise.parent_sample_base import ParentSampleBase
 
-__version__ = "0.4.4"
+__version__ = "0.4.5"

{timewise-0.4.4 → timewise-0.4.7}/timewise/wise_bigdata_desy_cluster.py

@@ -131,7 +131,8 @@ class WISEDataDESYCluster(WiseDataByVisit):
             chunk_number=None,
             jobID=None,
             return_filename=False,
-            use_bigdata_dir=False
+            use_bigdata_dir=False,
+            verify_contains_lightcurves=False
     ):
         fn = self._data_product_filename(
             service,
@@ -144,6 +145,13 @@ class WISEDataDESYCluster(WiseDataByVisit):
         try:
             with gzip.open(fn, 'rt', encoding="utf-8") as fzip:
                 data_product = json.load(fzip)
+
+            if verify_contains_lightcurves:
+                try:
+                    self._verify_contains_lightcurves(data_product)
+                except KeyError as e:
+                    raise KeyError(f"{fn}: {e}")
+
             if return_filename:
                 return data_product, fn
             return data_product

{timewise-0.4.4 → timewise-0.4.7}/timewise/wise_data_base.py

@@ -725,7 +725,21 @@ class WISEDataBase(abc.ABC):
             else:
                 return os.path.join(self._cache_photometry_dir, fn + f"_{jobID}.json")
 
-    def load_data_product(self, service, chunk_number=None, jobID=None, return_filename=False):
+    @staticmethod
+    def _verify_contains_lightcurves(data_product):
+        mask = ["timewise_lightcurve" in data.keys() for data in data_product.values()]
+        if not any(mask):
+            raise KeyError(f"'timewise_lightcurves' in none of the results."
+                           f"Cluster job probably did not finish.")
+
+    def load_data_product(
+            self,
+            service,
+            chunk_number=None,
+            jobID=None,
+            return_filename=False,
+            verify_contains_lightcurves=False
+    ):
         """
         Load data product from disk
 
@@ -736,12 +750,22 @@ class WISEDataBase(abc.ABC):
         :param jobID: jobID to load, if None load the combined file for this chunk
         :type jobID: int, optional
         :param return_filename: return filename of data product, defaults to False
+        :type return_filename: bool, optional
+        :param verify_contains_lightcurves: verify that the data product contains lightcurves, defaults to False
+        :type verify_contains_lightcurves: bool, optional
         """
         fn = self._data_product_filename(service, chunk_number, jobID)
         logger.debug(f"loading {fn}")
         try:
             with open(fn, "r") as f:
                 lcs = json.load(f)
+
+            if verify_contains_lightcurves:
+                try:
+                    self._verify_contains_lightcurves(lcs)
+                except KeyError as e:
+                    raise KeyError(f"{fn}: {e}")
+
             if return_filename:
                 return lcs, fn
             return lcs
@@ -791,33 +815,43 @@ class WISEDataBase(abc.ABC):
 
         lcs = None
         fns = list()
-        missing_files = 0
+        missing_files = False
+        erroneous_files = False
         for i in itr[1]:
             kw = dict(kwargs)
             kw[itr[0]] = i
             kw['return_filename'] = True
-            res = self.load_data_product(**kw)
+            kw["verify_contains_lightcurves"] = True
+
+            try:
+                res = self.load_data_product(**kw)
+                if res is not None:
+                    ilcs, ifn = res
+                    fns.append(ifn)
+                    if isinstance(lcs, type(None)):
+                        lcs = dict(ilcs)
+                    else:
+                        lcs.update(ilcs)
 
-            if not isinstance(res, type(None)):
-                ilcs, ifn = res
-                fns.append(ifn)
-                if isinstance(lcs, type(None)):
-                    lcs = dict(ilcs)
                 else:
-                    lcs.update(ilcs)
+                    missing_files = True
 
-            else:
-                missing_files += 1
-
-            if missing_files > 0:
-                msg = f"Missing {missing_files} for {service}"
-                if chunk_number is not None:
-                    msg += f" chunk {chunk_number}"
-                msg += "! Not saving data product"
-                logger.warning(msg)
+            except KeyError as e:
+                logger.error(e)
+                erroneous_files = True
+
+            if missing_files:
+                logger.warning(f"Missing files for {service}")
+
+            if erroneous_files:
+                logger.warning(f"Erroneous files for {service}")
+
+            if erroneous_files or missing_files:
+                _chunk_str = f" for chunk {chunk_number}" if chunk_number is not None else ""
+                logger.warning(f"Not saving combined data product{_chunk_str}")
                 break
 
-        if missing_files == 0:
+        if not (erroneous_files or missing_files):
             self._save_data_product(lcs, service=service, chunk_number=chunk_number, overwrite=overwrite)
 
             if remove:
@@ -1659,7 +1693,7 @@ class WISEDataBase(abc.ABC):
 
         if return_all:
             return_closest_allwise_mask = list(closest_allwise_mask) if closest_allwise_mask is not None else None
-            return list(bad_indices), cluster_res, return_closest_allwise_mask
+            return list(bad_indices), cluster_res, data_mask, return_closest_allwise_mask
         else:
             return list(bad_indices)
 

{timewise-0.4.4 → timewise-0.4.7}/timewise/wise_data_by_visit.py

@@ -4,6 +4,7 @@ import numpy as np
 import logging
 from scipy import stats
 import matplotlib.pyplot as plt
+from matplotlib.lines import Line2D
 
 from timewise.wise_data_base import WISEDataBase
 from timewise.utils import get_excess_variance
@@ -499,7 +500,7 @@ class WiseDataByVisit(WISEDataBase):
 
         # get a mask indicating outliers based on position
         ra, dec = pos.astype(float)
-        bad_indices_position, cluster_res, allwise_mask = self.calculate_position_mask(
+        bad_indices_position, cluster_res, data_mask, allwise_mask = self.calculate_position_mask(
             lightcurve, ra=ra, dec=dec, return_all=True, whitelist_region=self.whitelist_region.to("arcsec").value
         )
         position_mask = (
@@ -537,8 +538,7 @@ class WiseDataByVisit(WISEDataBase):
         )
 
         # set markers for visits
-        markers = ['o', 'v', '^', '<', '>', '1', '2', '3', '4', '8', 's', 'p', '*', 'h', 'H', '+', 'x', 'D', 'd', '|',
-                   '_', 'P', 'X']
+        markers = list(Line2D.filled_markers) + ["$1$", "$2$", "$3$", "$4$", "$5$", "$6$", "$7$", "$8$", "$9$"]
 
         # calculate ra and dec relative to center of cutout
         ra = (lightcurve.ra - pos[self.parent_sample.default_keymap["ra"]]) * 3600
@@ -547,41 +547,65 @@ class WiseDataByVisit(WISEDataBase):
         # for each visit plot the datapoints on the cutout
         for visit in np.unique(visit_map):
             m = visit_map == visit
-
             label = str(visit)
-            marker = markers[visit]
-            color = f"C{visit}"
+            axs[0].plot([], [], marker=markers[visit], label=label, mec="k", mew=1, mfc="none", ls="")
 
-            for im, _color, _label, zorder in zip(
+            for im, mec, zorder in zip(
                     [position_mask, ~position_mask],
-                    [color, "gray"],
-                    [label, ""],
+                    ["k", "none"],
                     [1, 0]
             ):
                 mask = m & im
-                datapoints = lightcurve[mask]
-                if ("sigra" in datapoints.columns) and ("sigdec" in datapoints.columns):
-                    has_sig = ~datapoints.sigra.isna() & ~datapoints.sigdec.isna()
-                    axs[0].errorbar(
-                        ra[mask][has_sig],
-                        dec[mask][has_sig],
-                        xerr=datapoints.sigra[has_sig] / 3600,
-                        yerr=datapoints.sigdec[has_sig] / 3600,
-                        label=_label,
-                        marker=marker,
-                        ls="",
-                        color=_color,
-                        zorder=zorder
-                    )
-                    axs[0].scatter(
-                        ra[mask][~has_sig],
-                        dec[mask][~has_sig],
-                        marker=marker,
-                        color=_color,
-                        zorder=zorder
+
+                for i_data_mask, i_data in zip([data_mask, ~data_mask], ["data", "other_allwise"]):
+                    datapoints = lightcurve[mask & i_data_mask]
+                    cluster_labels = (
+                        cluster_res.labels_[mask[i_data_mask]] if i_data == "data"
+                        else np.array([-1] * len(datapoints))
                     )
-                else:
-                    axs[0].scatter(ra[mask], dec[mask], label=_label, marker=marker, color=_color, zorder=zorder)
+
+                    for cluster_label in np.unique(cluster_labels):
+                        cluster_label_mask = cluster_labels == cluster_label
+                        datapoints_cluster = datapoints[cluster_label_mask]
+                        # make a marker for the visit and colored by cluster
+                        color = f"C{cluster_label}" if cluster_label != -1 else "grey"
+
+                        if ("sigra" in datapoints_cluster.columns) and ("sigdec" in datapoints_cluster.columns):
+                            has_sig = ~datapoints_cluster.sigra.isna() & ~datapoints_cluster.sigdec.isna()
+                            _ra = ra[mask & i_data_mask][cluster_label_mask]
+                            _dec = dec[mask & i_data_mask][cluster_label_mask]
+
+                            axs[0].errorbar(
+                                _ra[has_sig],
+                                _dec[has_sig],
+                                xerr=datapoints_cluster.sigra[has_sig] / 3600,
+                                yerr=datapoints_cluster.sigdec[has_sig] / 3600,
+                                marker=markers[visit],
+                                ls="",
+                                color=color,
+                                zorder=zorder,
+                                ms=10,
+                                mec=mec,
+                                mew=0.1
+                            )
+                            axs[0].scatter(
+                                _ra[~has_sig],
+                                _dec[~has_sig],
+                                marker=markers[visit],
+                                color=color,
+                                zorder=zorder,
+                                edgecolors=mec,
+                                linewidths=0.1,
+                            )
+                        else:
+                            axs[0].scatter(
+                                ra[mask], dec[mask],
+                                marker=markers[visit],
+                                color=color,
+                                zorder=zorder,
+                                edgecolors=mec,
+                                linewidths=0.1,
+                            )
 
         # for each band indicate the outliers based on brightness with circles
         for b, outlier_mask in outlier_masks.items():