mwdust 1.6__cp313-cp313-macosx_10_13_x86_64.whl

mwdust/Sale14.py

@@ -0,0 +1,217 @@
+###############################################################################
+#
+#   Sale14: extinction model from Sale et al. 2014 2014MNRAS.443.2907S
+#
+###############################################################################
+import os, os.path
+import sys
+import tarfile
+import shutil
+import numpy
+from scipy import interpolate
+from astropy.io import ascii
+from mwdust.util.extCurves import aebv
+from mwdust.util.tools import cos_sphere_dist
+from mwdust.util.download import downloader, dust_dir
+from mwdust.DustMap3D import DustMap3D
+
+_DEGTORAD= numpy.pi/180.
+_saledir= os.path.join(dust_dir,'sale14')
+_ERASESTR= "                                                                                "
+class Sale14(DustMap3D):
+    """extinction model from Sale et al. 2014 2014MNRAS.443.2907S"""
+    def __init__(self,filter=None,sf10=True):
+        """
+        NAME:
+           __init__
+        PURPOSE:
+           Initialize the Sale14 dust map
+        INPUT:
+           filter= filter to return the extinction in
+           sf10= (True) if True, use the Schlafly & Finkbeiner calibrations
+        OUTPUT:
+           object
+        HISTORY:
+           2015-03-08 - Started - Bovy (IAS)
+        """
+        DustMap3D.__init__(self,filter=filter)
+        self._sf10= sf10
+        #Read the maps
+        sys.stdout.write('\r'+"Reading Sale et al. (2014) data file ...\r")
+        sys.stdout.flush()
+        self._saledata= ascii.read(os.path.join(_saledir,
+                                                     'Amap.dat'),
+                                        readme=os.path.join(_saledir,
+                                                            'ReadMe'),
+                                        guess=False, format='cds',
+                                        fill_values=[('', '-999')])
+        sys.stdout.write('\r'+_ERASESTR+'\r')
+        sys.stdout.flush()
+        # Some summaries
+        self._dl= self._saledata['lmax']-self._saledata['lmin']
+        self._db= self._saledata['b_max']-self._saledata['b_min']
+        self._lmin= numpy.amin(self._saledata['lmin'])
+        self._lmax= numpy.amax(self._saledata['lmax'])
+        self._bmin= numpy.amin(self._saledata['b_min'])
+        self._bmax= numpy.amax(self._saledata['b_max'])
+        self._ndistbin= 150
+        self._ds= numpy.linspace(0.05,14.95,self._ndistbin)
+        # For dust_vals
+        self._sintheta= numpy.sin((90.-self._saledata['GLAT'])*_DEGTORAD)
+        self._costheta= numpy.cos((90.-self._saledata['GLAT'])*_DEGTORAD)
+        self._sinphi= numpy.sin(self._saledata['GLON']*_DEGTORAD)
+        self._cosphi= numpy.cos(self._saledata['GLON']*_DEGTORAD)
+        self._intps= numpy.zeros(len(self._saledata),dtype='object') #array to cache interpolated extinctions
+        return None
+
+    def _evaluate(self,l,b,d,_lbIndx=None):
+        """
+        NAME:
+           _evaluate
+        PURPOSE:
+           evaluate the dust-map
+        INPUT:
+           l- Galactic longitude (deg)
+           b- Galactic latitude (deg)
+           d- distance (kpc) can be array
+        OUTPUT:
+           extinction
+        HISTORY:
+           2015-03-08 - Started - Bovy (IAS)
+        """
+        if isinstance(l,numpy.ndarray) or isinstance(b,numpy.ndarray):
+            raise NotImplementedError("array input for l and b for Sale et al. (2014) dust map not implemented")
+        if _lbIndx is None: lbIndx= self._lbIndx(l,b)
+        else: lbIndx= _lbIndx
+        if self._intps[lbIndx] != 0:
+            out= self._intps[lbIndx](d)
+        else:
+            tlbData= self.lbData(l,b)
+            interpData=\
+                interpolate.InterpolatedUnivariateSpline(self._ds,
+                                                         tlbData['a0'],
+                                                         k=1)
+            out= interpData(d)
+            self._intps[lbIndx]= interpData
+        if self._filter is None: # Sale et al. say A0/Aks = 11
+            return out/11./aebv('2MASS Ks',sf10=self._sf10)
+        else: # if sf10, first put ebv on SFD scale
+            return out/11./aebv('2MASS Ks',sf10=self._sf10)\
+                *aebv(self._filter,sf10=self._sf10)
+
+    def dust_vals_disk(self,lcen,bcen,dist,radius):
+        """
+        NAME:
+           dust_vals_disk
+        PURPOSE:
+           return the distribution of extinction within a small disk as samples
+        INPUT:
+           lcen, bcen - Galactic longitude and latitude of the center of the disk (deg)
+           dist - distance in kpc
+           radius - radius of the disk (deg)
+        OUTPUT:
+           (pixarea,extinction) - arrays of pixel-area in sq rad and extinction value
+        HISTORY:
+           2015-03-07 - Written - Bovy (IAS)
+        """
+        # Find all of the (l,b) of the pixels within radius of (lcen,bcen)
+        indx= cos_sphere_dist(self._sintheta,self._costheta,
+                              self._sinphi,self._cosphi,
+                              numpy.sin((90.-bcen)*_DEGTORAD),
+                              numpy.cos((90.-bcen)*_DEGTORAD),
+                              numpy.sin(lcen*_DEGTORAD),
+                              numpy.cos(lcen*_DEGTORAD)) \
+                              >= numpy.cos(radius*_DEGTORAD)
+        ll= self._saledata['GLON'][indx]
+        bb= self._saledata['GLAT'][indx]
+        # Now get the extinctions for these pixels
+        pixarea= []
+        extinction= []
+        for l,b in zip(ll,bb):
+            lbIndx= self._lbIndx(l,b)
+            extinction.append(self._evaluate(l,b,dist,_lbIndx=lbIndx))
+            pixarea.append(self._dl[lbIndx]*self._db[lbIndx]*_DEGTORAD**2.)
+        pixarea= numpy.array(pixarea)
+        extinction= numpy.array(extinction)
+        return (pixarea,extinction)
+               
+    def dmax(self,l,b):
+        """
+        NAME:
+           dmax
+        PURPOSE:
+           return the maximum distance for which to trust the Sale et al. (2014) data
+        INPUT:
+           l- Galactic longitude (deg)
+           b- Galactic latitude (deg)
+        OUTPUT:
+           maximum distance in kpc
+        HISTORY:
+           2015-03-08 - Written - Bovy (IAS)
+        """
+        lbIndx= self._lbIndx(l,b)
+        return self._saledata['trust'][lbIndx]/1000.
+
+    def lbData(self,l,b):
+        """
+        NAME:
+           lbData
+        PURPOSE:
+           return the Sale et al. (2014) data corresponding to a given
+           line of sight
+        INPUT:
+           l- Galactic longitude (deg)
+           b- Galactic latitude (deg)
+        OUTPUT:
+        HISTORY:
+           2015-03-08 - Written - Bovy (IAS)
+        """
+        #Find correct entry
+        lbIndx= self._lbIndx(l,b)
+        #Build output array
+        out= numpy.recarray((self._ndistbin,),
+                            dtype=[('a0', 'f8'),
+                                   ('e_a0','f8')])
+        for ii in range(self._ndistbin):
+            out[ii]['a0']= self._saledata[lbIndx]['meanA%i' % (ii+1)]
+            out[ii]['e_a0']= self._saledata[lbIndx]['meanA%i' % (ii+1)]
+        return out
+
+    def _lbIndx(self,l,b):
+        """Return the index in the _saledata array corresponding to this (l,b)"""
+        if l <= self._lmin or l >= self._lmax \
+                or b <= self._bmin or b >= self._bmax:
+            raise IndexError("Given (l,b) pair not within the region covered by the Sale et al. (2014) dust map")
+        return numpy.argmin((l-self._saledata['GLON'])**2./self._dl**2.\
+                                +(b-self._saledata['GLAT'])**2./self._db**2.)
+
+    @classmethod
+    def download(cls, test=False):
+        sale_folder_path = os.path.join(dust_dir, "sale14")
+        sale_path = os.path.join(sale_folder_path, "Amap.tar.gz")
+        if not os.path.exists(sale_path[:-6] + "dat"):
+            if not os.path.exists(sale_folder_path):
+                os.mkdir(sale_folder_path)
+            _SALE_URL= "http://www.iphas.org/data/extinction/Amap.tar.gz"
+            downloader(_SALE_URL, sale_path, cls.__name__, test=test)
+            if not test:
+                sale_file = tarfile.open(sale_path)
+                sale_file.extractall(sale_folder_path)
+                sale_file.close()
+                os.remove(sale_path)
+                # Fix one line in the dust map
+                with open(os.path.join(sale_folder_path, "tmp.dat"), "w") as fout:
+                    with open(os.path.join(sale_folder_path, "Amap.dat"), "r") as fin:
+                        for line in fin:
+                            if "15960.40000" in line: # bad line
+                                newline= ''
+                                for ii,word in enumerate(line.split(' ')):
+                                    if ii > 0: newline+= ' '
+                                    if ii > 6 and len(word) > 9:
+                                        word= "747.91400"
+                                    newline+= word
+                                fout.write(newline+'\n')
+                            else:
+                                fout.write(line)
+                shutil.move(os.path.join(sale_folder_path, "tmp.dat"), os.path.join(sale_folder_path, "Amap.dat"))
+        return None

mwdust/Zero.py

@@ -0,0 +1,63 @@
+###############################################################################
+#
+#   Zero: model with zero extinction
+#
+###############################################################################
+import numpy
+from mwdust.DustMap3D import DustMap3D
+_DEGTORAD= numpy.pi/180.
+class Zero(DustMap3D):
+    """model with zero extinction"""
+    def __init__(self,filter=None,sf10=True):
+        """
+        NAME:
+           __init__
+        PURPOSE:
+           Initialize the zero extinction model
+        INPUT:
+           filter= filter to return the extinction in
+           sf10= (True) if True, use the Schlafly & Finkbeiner calibrations
+        OUTPUT:
+           Instance
+        HISTORY:
+           2015-03-07 - Written - Bovy (IAS)
+        """
+        DustMap3D.__init__(self,filter=filter)
+        self._sf10= sf10
+        return None
+
+    def _evaluate(self,l,b,d):
+        """
+        NAME:
+           _evaluate
+        PURPOSE:
+           evaluate the dust-map
+        INPUT:
+           l- Galactic longitude (deg)
+           b- Galactic latitude (deg)
+           d- distance (kpc) can be array
+        OUTPUT:
+           extinction E(B-V)
+        HISTORY:
+           2015-03-07 - Written - Bovy (IAS)
+        """
+        return numpy.zeros_like(d)
+
+    def dust_vals_disk(self,lcen,bcen,dist,radius):
+        """
+        NAME:
+           dust_vals_disk
+        PURPOSE:
+           return the distribution of extinction within a small disk as samples
+        INPUT:
+           lcen, bcen - Galactic longitude and latitude of the center of the disk (deg)
+           dist - distance in kpc
+           radius - radius of the disk (deg)
+        OUTPUT:
+           (pixarea,extinction) - arrays of pixel-area in sq rad and extinction value
+        HISTORY:
+           2015-03-06 - Written - Bovy (IAS)
+        """
+        pixarea= (1.-numpy.cos(radius*_DEGTORAD))*2.*numpy.pi
+        return (numpy.array([pixarea]),numpy.zeros((1,len(dist))))
+

mwdust/Zucker25.py

@@ -0,0 +1,107 @@
+###############################################################################
+#
+#   DECaPS25: extinction model from Zucker et al. (2025)
+#
+###############################################################################
+import os, os.path
+import numpy
+import h5py
+from mwdust.util.download import dust_dir, downloader
+from mwdust.HierarchicalHealpixMap import HierarchicalHealpixMap
+
+_DEGTORAD = numpy.pi/180.
+_decapsdir = os.path.join(dust_dir, 'zucker25')
+
+class Zucker25(HierarchicalHealpixMap):
+    """DECaPS 3D dust-reddening map (Zucker et al. 2025)"""
+    def __init__(self, filter=None, sf10=True, load_samples=False, interpk=1):
+        """
+        NAME:
+           __init__
+        PURPOSE:
+           Initialize the DECaPS (2025) dust map
+        INPUT:
+           filter= filter to return the extinction in
+           sf10= (True) if True, use the Schlafly & Finkbeiner calibrations
+           load_samples= (False) if True, also load the samples
+           interpk= (1) interpolation order
+        OUTPUT:
+           object
+        HISTORY:
+           2025-10-01 - Adopted
+        """
+        HierarchicalHealpixMap.__init__(self, filter=filter, sf10=sf10, samples=load_samples)
+        if not os.path.isdir(_decapsdir):
+            os.mkdir(_decapsdir)
+        fname = 'decaps_mean_and_samples.h5' if load_samples else 'decaps_mean.h5'
+        fpath = os.path.join(_decapsdir, fname)
+        if not os.path.exists(fpath):
+            self.download(samples=load_samples)
+        self._f = h5py.File(fpath, 'r')
+        self._best_fit = self._f['/mean'][:, 0, :]
+        p = self._f['/pixel_info']
+        hpx = p['healpix_index'][:]
+        nside_attr = int(p.attrs['nside'])
+        pix_dtype = numpy.dtype([('healpix_index', hpx.dtype), ('nside', numpy.int64)])
+        self._pix_info = numpy.empty(hpx.shape[0], dtype=pix_dtype)
+        self._pix_info['healpix_index'] = hpx
+        self._pix_info['nside'] = nside_attr
+        dm = numpy.array(p.attrs['DM_bin_edges'], dtype=numpy.float64)
+        if dm.shape[0] != self._best_fit.shape[1]:
+            raise RuntimeError("DM_bin_edges length does not match radial dimension")
+        self._distmods = dm
+        if load_samples:
+            if 'samples' not in self._f:
+                raise RuntimeError("Requested load_samples=True, but 'samples' dataset not found.")
+            self._samples_dset = self._f['/samples']
+        else:
+            self._samples_dset = None
+        self._minnside = numpy.amin(self._pix_info['nside'])
+        self._maxnside = numpy.amax(self._pix_info['nside'])
+        nlevels = int(numpy.log2(self._maxnside // self._minnside)) + 1
+        self._nsides = [self._maxnside // 2**ii for ii in range(nlevels)]
+        self._indexArray = numpy.arange(len(self._pix_info['healpix_index']))
+        self._intps = numpy.zeros(len(self._pix_info['healpix_index']), dtype='object')
+        self._interpk = interpk
+        return None
+
+    def substitute_sample(self, samplenum):
+        """
+        NAME:
+           substitute_sample
+        PURPOSE:
+           substitute a sample for the best fit to get the extinction from a sample with the same tools; need to have setup the instance with load_samples=True
+        INPUT:
+           samplenum - sample's index to load
+        OUTPUT:
+           (none; just resets the instance to use the sample rather than the best fit)
+        HISTORY:
+           2025-10-01 - Adopted
+        """
+        if self._samples_dset is None:
+            raise RuntimeError('No samples present in DECaPS file')
+        self._best_fit = self._samples_dset[:, samplenum, :]
+        self._intps = numpy.zeros(len(self._pix_info['healpix_index']), dtype='object')
+        return None
+
+    @classmethod
+    def download(cls, samples=False, test=False):
+        subdir = os.path.join(dust_dir, "decaps25")
+        if not os.path.exists(subdir):
+            os.mkdir(subdir)
+        if samples:
+            target = os.path.join(subdir, "decaps_mean_and_samples.h5")
+            url = "https://dataverse.harvard.edu/api/access/datafile/11840498"
+        else:
+            target = os.path.join(subdir, "decaps_mean.h5")
+            url = "https://dataverse.harvard.edu/api/access/datafile/11838924"
+        if not os.path.exists(target):
+            downloader(url, target, cls.__name__, test=test)
+        return None
+
+    def __del__(self):
+        try:
+            if hasattr(self, "_f") and self._f:
+                self._f.close()
+        except Exception:
+            pass

mwdust/__init__.py

@@ -0,0 +1,27 @@
+from mwdust.SFD import SFD
+from mwdust.Marshall06 import Marshall06
+from mwdust.Drimmel03 import Drimmel03
+from mwdust.Sale14 import Sale14
+from mwdust.Green15 import Green15
+from mwdust.Green17 import Green17
+from mwdust.Green19 import Green19
+from mwdust.Combined15 import Combined15
+from mwdust.Combined19 import Combined19
+from mwdust.Zucker25 import Zucker25
+from mwdust.Zero import Zero
+
+__version__ = "1.6"
+
+
+def download_all(test=False):
+    SFD.download(test=test)
+    Marshall06.download(test=test)
+    Drimmel03.download(test=test)
+    if not test:
+        Sale14.download(test=test)
+    Green15.download(test=test)
+    Green17.download(test=test)
+    Green19.download(test=test)
+    Combined15.download(test=test)
+    Combined19.download(test=test)
+    Zucker25.download(test=test)

mwdust/util/combine_dustmaps19.py

@@ -0,0 +1,202 @@
+# combine_dustmaps.py: make a composite extinction map (Marshall,Green,Drimmel)
+# Using legacy code and only runs in python 2
+# Creates a composite map. Run from command line:
+# python combine_dustmaps19.py combine19.sav (runs the combine_dustmap function)
+# python combine_dustmaps19.py combine19.sav save (runs the store_h5 function)
+# python combine_dustmaps19.py combine19.sav fixnans (runs the fix_nans function)
+# The first call takes a few hours, can be tested via _DRYISHRUN = False first
+###############################################################################
+import sys
+import os, os.path
+import pickle
+import numpy
+import h5py
+import healpy
+import mwdust
+from mwdust.util.download import dust_dir
+
+from galpy.util import save_pickles
+
+
+def distmod2dist(distmod):
+    """distance modulus to distance in kpc"""
+    return 10.**(distmod/5.-2.)
+
+_greendir= os.path.join(dust_dir,'green19')
+#_GREEN15DISTMODS= numpy.linspace(4.,19.,31) # distance spacing for green15 and green17
+_GREEN19DISTMODS= numpy.linspace(4,18.875,120) # distance spacing for green19
+_GREEN19DISTS= distmod2dist(_GREEN19DISTMODS)
+_DEGTORAD= numpy.pi/180.
+_ERASESTR= "                                                                                "
+_DRYISHRUN= False
+def combine_dustmap(picklename):
+    if os.path.exists(picklename): return None
+    ndists= len(_GREEN19DISTS)
+    # First fill in with NSIDE = 512 for Marshall
+    marshallmap= mwdust.Marshall06()
+    nside_mar= 512
+    mar_pix= numpy.arange(healpy.pixelfunc.nside2npix(nside_mar))
+    mar_val= numpy.zeros((len(mar_pix),ndists))+healpy.UNSEEN
+    theta, phi= \
+        healpy.pixelfunc.pix2ang(nside_mar,mar_pix,nest=True)
+    bb= (numpy.pi/2.-theta)/numpy.pi*180.
+    ll= phi/numpy.pi*180.
+    subIndx= (numpy.fabs(bb) < 10.125)\
+        *((ll < 100.125)+(ll > 259.875))
+    mar_pix= mar_pix[subIndx]
+    mar_val= mar_val[subIndx]
+    ll= ll[subIndx]
+    ll[ll > 180.]-= 360.
+    bb= bb[subIndx]
+    for pp,dpix in enumerate(mar_pix):
+        sys.stdout.write('\r'+"Working on pixel %i, %i remaining ...\r" % (pp+1,len(mar_pix)-pp))
+        sys.stdout.flush()
+        if _DRYISHRUN and pp > 100: break
+        mar_val[pp]= marshallmap(ll[pp],bb[pp],_GREEN19DISTS)
+    sys.stdout.write('\r'+_ERASESTR+'\r')
+    sys.stdout.flush()
+    # Now load Green19 and remove those pixels that fall within the Marshall map
+    with h5py.File(os.path.join(_greendir,'bayestar2019.h5'),'r') as greendata:
+        pix_info= greendata['/pixel_info'][:]
+        best_fit= greendata['/best_fit'][:]
+    # Check whether any of these fall within the Marshall map
+    theta, phi= healpy.pixelfunc.pix2ang(pix_info['nside'].astype('int32'),
+                                         pix_info['healpix_index'].astype('int64'),
+                                         nest=True)
+    inMar= ((phi < 100.125*_DEGTORAD)+(phi > 259.875*_DEGTORAD))\
+        *(numpy.fabs(numpy.pi/2.-theta) < 10.125*_DEGTORAD)
+    best_fit[inMar]= healpy.UNSEEN
+    nside_min= numpy.min(pix_info['nside'])
+    nside_max= numpy.max(pix_info['nside'])
+    # Fill in remaining gaps with Drimmel at NSIDE=256
+    pix_drim= []
+    pix_drim_nside= []
+    val_drim= []
+    drimmelmap= mwdust.Drimmel03()   
+    for nside_drim in 2**numpy.arange(8,int(numpy.log2(nside_max))+1,1):
+        tpix= numpy.arange(healpy.pixelfunc.nside2npix(nside_drim))
+        rmIndx= numpy.zeros(len(tpix),dtype='bool')
+        # Remove pixels that already have values at this or a higher level
+        for nside in 2**numpy.arange(8,
+                                     int(numpy.log2(nside_max))+1,1):
+            mult_factor = (nside/nside_drim)**2
+            tgpix= pix_info['healpix_index'][pix_info['nside'] == nside]
+            for offset in numpy.arange(mult_factor):
+                rmIndx[numpy.in1d(tpix*mult_factor+offset,tgpix,
+                                  assume_unique=True)]= True
+        # Remove pixels that already have values at a lower level
+        for nside in 2**numpy.arange(int(numpy.log2(nside_min)),
+                                         int(numpy.log2(nside_drim)),1):
+            mult_factor = (nside_drim/nside)**2
+            # in Green 19
+            tgpix= pix_info['healpix_index'][pix_info['nside'] == nside]
+            rmIndx[numpy.in1d(tpix//mult_factor,tgpix,assume_unique=False)]= True
+            # In the current Drimmel
+            tdpix= numpy.array(pix_drim)[numpy.array(pix_drim_nside) == nside]
+            rmIndx[numpy.in1d(tpix//mult_factor,tdpix,assume_unique=False)]= True
+        # Also remove pixels that lie within the Marshall area
+        theta, phi= healpy.pixelfunc.pix2ang(nside_drim,tpix,nest=True)
+        inMar= ((phi < 100.125*_DEGTORAD)+(phi > 259.875*_DEGTORAD))\
+            *(numpy.fabs(numpy.pi/2.-theta) < 10.125*_DEGTORAD)
+        rmIndx[inMar]= True
+        tpix= tpix[True-rmIndx]
+        pix_drim.extend(tpix)
+        pix_drim_nside.extend(nside_drim*numpy.ones(len(tpix)))
+        ll= phi[True-rmIndx]/_DEGTORAD
+        bb= (numpy.pi/2.-theta[True-rmIndx])/_DEGTORAD
+        for pp in range(len(tpix)):
+            sys.stdout.write('\r'+"Working on level %i, pixel %i, %i remaining ...\r" % (nside_drim,pp+1,len(tpix)-pp))
+            sys.stdout.flush()
+            val_drim.append(drimmelmap(ll[pp],bb[pp],_GREEN19DISTS))
+            if _DRYISHRUN and pp > 1000: break
+    sys.stdout.write('\r'+_ERASESTR+'\r')
+    sys.stdout.flush()
+    # Save
+    g19Indx= best_fit[:,0] != healpy.UNSEEN
+    save_pickles(picklename,mar_pix,mar_val,
+                 pix_info['nside'][g19Indx],pix_info['healpix_index'][g19Indx],
+                 best_fit[g19Indx],
+                 pix_drim,pix_drim_nside,val_drim)
+    return None
+
+def store_h5(picklename):
+    # Restore pickle
+    if not os.path.exists(picklename):
+        print("file %s does not exist!" % picklename)
+        return None
+    with open(picklename,'rb') as picklefile:
+        mar_pix= pickle.load(picklefile)
+        mar_val= pickle.load(picklefile)
+        pix_info_nside= pickle.load(picklefile)
+        pix_info_healpix= pickle.load(picklefile)
+        best_fit= pickle.load(picklefile)
+        pix_drim= pickle.load(picklefile)
+        pix_drim_nside= pickle.load(picklefile)
+        val_drim= pickle.load(picklefile)
+    # Combine
+    nout= len(mar_pix)+len(pix_info_nside)+len(val_drim)
+    out_nside= numpy.empty(nout,dtype='uint32')
+    out_healpix= numpy.empty(nout,dtype='uint64')
+    out_bf= numpy.empty((nout,len(_GREEN19DISTS)),dtype='float64')
+    # Load Marshall
+    out_nside[:len(mar_pix)]= 512
+    out_healpix[:len(mar_pix)]= mar_pix
+    out_bf[:len(mar_pix)]= mar_val
+    # Load Green 19
+    out_nside[len(mar_pix):len(mar_pix)+len(pix_info_nside)]= pix_info_nside
+    out_healpix[len(mar_pix):len(mar_pix)+len(pix_info_nside)]= pix_info_healpix
+    out_bf[len(mar_pix):len(mar_pix)+len(pix_info_nside)]= best_fit
+    # Load Drimmel
+    if _DRYISHRUN:
+        out_nside[len(mar_pix)+len(pix_info_nside):]= pix_drim_nside[:len(val_drim)]
+        out_healpix[len(mar_pix)+len(pix_info_nside):]= pix_drim[:len(val_drim)]
+    else:
+        out_nside[len(mar_pix)+len(pix_info_nside):]= pix_drim_nside
+        out_healpix[len(mar_pix)+len(pix_info_nside):]= pix_drim
+    out_bf[len(mar_pix)+len(pix_info_nside):]= val_drim
+    # Save to h5 file
+    outfile= h5py.File(picklename.replace('.sav','.h5'),"w")
+    pixinfo= numpy.recarray(len(out_nside),
+                            dtype=[('nside','uint32'),
+                                   ('healpix_index','uint64')])
+    pixinfo['nside']= out_nside
+    pixinfo['healpix_index']= out_healpix
+    outfile.create_dataset("pixel_info", data=pixinfo)
+    outfile.create_dataset("best_fit",data=out_bf)
+    outfile.close()
+    return None    
+
+def fix_nans(picklename):
+    """Marshall has a few NaNs, replace these with Drimmel"""
+    with h5py.File(picklename.replace('.sav','.h5'),'r') as combdata:
+        pix_info= combdata['/pixel_info'][:]
+        best_fit= combdata['/best_fit'][:]
+    nanIndx= numpy.isnan(best_fit[:,0])
+    print("Found %i NaNs ..." % numpy.sum(nanIndx))
+    theta, phi= healpy.pixelfunc.pix2ang(pix_info['nside'].astype('int32'),
+                                         pix_info['healpix_index'].astype('int64'),
+                                         nest=True)
+    bb= (numpy.pi/2.-theta)/_DEGTORAD
+    ll= phi/_DEGTORAD
+    indices= numpy.arange(len(pix_info['nside']))[nanIndx]
+    drimmelmap= mwdust.Drimmel03()   
+    for ii in range(numpy.sum(nanIndx)):
+        best_fit[indices[ii]]= drimmelmap(ll[ii],bb[ii],_GREEN19DISTS)
+    # Now save
+    nanIndx= numpy.isnan(best_fit[:,0])
+    print("Found %i NaNs ..." % numpy.sum(nanIndx))
+    # Save to h5 file
+    outfile= h5py.File(picklename.replace('.sav','.h5'),"w")
+    outfile.create_dataset("pixel_info", data=pix_info)
+    outfile.create_dataset("best_fit",data=best_fit)
+    outfile.close()
+    return None
+
+if __name__ == '__main__':
+    if len(sys.argv) > 2 and sys.argv[2] == 'fixnans':
+        fix_nans(sys.argv[1])
+    elif len(sys.argv) > 2:
+        store_h5(sys.argv[1])
+    else:
+        print(sys.argv[1])        
+        combine_dustmap(sys.argv[1])

mwdust/util/download.py

@@ -0,0 +1,32 @@
+import os
+import pathlib
+import shutil
+import requests
+import tqdm
+
+dust_dir = os.environ.get("DUST_DIR")
+if dust_dir is None:
+    dust_dir = os.path.expanduser(os.path.join("~", ".mwdust"))
+if not os.path.exists(dust_dir):
+    os.mkdir(dust_dir)
+
+def downloader(url, fullfilename, name, test=False):
+    """
+    url: URL of data file
+    fullfilename: full local path
+    name: name of the task
+    """
+    user_agent = "Mozilla/5.0"
+    r = requests.get(url, stream=True, allow_redirects=True, verify=True, headers={"User-Agent": user_agent})
+    if r.status_code == 404:
+        raise ConnectionError(f"Cannot find {name} data file at {url}")
+    r.raise_for_status()  # Will only raise for 4xx codes
+    if not test:
+        file_size = int(r.headers.get('Content-Length', 0))
+        path = pathlib.Path(fullfilename).expanduser().resolve()
+        path.parent.mkdir(parents=True, exist_ok=True)
+
+        # r.raw.read
+        with tqdm.tqdm.wrapattr(r.raw, "read", total=file_size, desc=f"Download {name}") as r_raw:
+            with path.open("wb") as f:
+                shutil.copyfileobj(r_raw, f)