stdpipe 0.4.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
stdpipe/templates.py ADDED
@@ -0,0 +1,929 @@
1
+ import numpy as np
2
+
3
+ import os
4
+ import json
5
+ import tempfile
6
+ import shlex
7
+ import time
8
+ import shutil
9
+
10
+ from urllib.parse import urlencode
11
+
12
+ from astropy.wcs import WCS
13
+ from astropy.io import fits
14
+ from astropy.stats import mad_std
15
+ from astropy.table import Table
16
+ from astropy.coordinates import SkyCoord
17
+
18
+ from scipy.ndimage import binary_dilation
19
+ from astropy.convolution import Tophat2DKernel
20
+
21
+ from . import utils
22
+ from . import photometry
23
+ from . import pipeline
24
+ from . import astrometry
25
+ from . import cutouts
26
+ from .reproject import reproject_swarp, reproject_lanczos # backward compat
27
+
28
+ # HiPS images
29
+
30
+
31
+ def get_hips_image(
32
+ hips,
33
+ ra=None,
34
+ dec=None,
35
+ width=None,
36
+ height=None,
37
+ fov=None,
38
+ wcs=None,
39
+ shape=None,
40
+ header=None,
41
+ asinh=None,
42
+ normalize=True,
43
+ upscale=False,
44
+ get_header=True,
45
+ verbose=False,
46
+ ):
47
+ """Load an image from any HiPS survey using the CDS hips2fits service.
48
+
49
+ The pixel grid may be specified by center coordinates and field of view, or
50
+ by passing a WCS solution or FITS header directly.
51
+
52
+ Parameters
53
+ ----------
54
+ hips : str
55
+ HiPS survey identifier. See https://aladin.u-strasbg.fr/hips/list for
56
+ the full list.
57
+ ra : float, optional
58
+ Image center Right Ascension in degrees.
59
+ dec : float, optional
60
+ Image center Declination in degrees.
61
+ width : int, optional
62
+ Image width in pixels.
63
+ height : int, optional
64
+ Image height in pixels.
65
+ fov : float, optional
66
+ Field of view (angular size) in degrees.
67
+ wcs : astropy.wcs.WCS, optional
68
+ WCS defining the pixel grid; supersedes ``ra``, ``dec``, ``fov``.
69
+ shape : tuple of int, optional
70
+ Image shape ``(height, width)``; alternative to ``width`` / ``height``.
71
+ header : astropy.io.fits.Header, optional
72
+ Header containing image dimensions and WCS; alternative to providing
73
+ them explicitly.
74
+ asinh : bool or None, optional
75
+ Whether the survey uses non-linear asinh flux scaling. Auto-detected
76
+ for Pan-STARRS surveys; set to ``False`` to override.
77
+ normalize : bool, optional
78
+ If True, pseudo-normalize the image so that background mean ≈ 100 and
79
+ background RMS ≈ 10.
80
+ upscale : bool or int, optional
81
+ If set, request an upscaled image then downscale before returning.
82
+ Useful for Pan-STARRS to reduce asinh photometric errors at coarse
83
+ pixel scales.
84
+ get_header : bool, optional
85
+ If True, also return the FITS header alongside the image.
86
+ verbose : bool or callable, optional
87
+ Whether to show verbose messages. May be boolean or a ``print``-like callable.
88
+
89
+ Returns
90
+ -------
91
+ ndarray or tuple or None
92
+ Image projected onto the requested pixel grid, or ``(image, header)``
93
+ if ``get_header=True``, or None on failure.
94
+ """
95
+
96
+ # Simple wrapper around print for logging in verbose mode only
97
+ log = (verbose if callable(verbose) else print) if verbose else lambda *args, **kwargs: None
98
+
99
+ wcs_orig = None # Original grid WCS, set if the request is upscaled
100
+
101
+ if wcs is None and header is not None:
102
+ wcs = WCS(header)
103
+
104
+ if width is None or height is None:
105
+ if shape is not None:
106
+ height, width = shape
107
+ elif header is not None:
108
+ width = header['NAXIS1']
109
+ height = header['NAXIS2']
110
+ elif wcs is not None and wcs.pixel_shape is not None:
111
+ width, height = wcs.pixel_shape
112
+
113
+ if width is None or height is None:
114
+ log('Frame size is not provided')
115
+ return (None, None) if get_header else None
116
+
117
+ params = {
118
+ 'hips': hips,
119
+ 'width': int(width),
120
+ 'height': int(height),
121
+ 'coordsys': 'icrs',
122
+ 'format': 'fits',
123
+ }
124
+
125
+ if wcs is not None and wcs.is_celestial:
126
+ # Whether we should upscale the template or not
127
+ if upscale is True:
128
+ # Let's guess the upscaling that will drive pixel scale beyond 1 arcsec/pix
129
+ pixscale = astrometry.get_pixscale(wcs=wcs) * 3600
130
+ upscale = int(np.ceil(pixscale / 1))
131
+
132
+ if upscale > 1:
133
+ log(
134
+ 'Will upscale the image from %.2f to %.2f arcsec/pix'
135
+ % (pixscale, pixscale / upscale)
136
+ )
137
+
138
+ if upscale and upscale > 1:
139
+ log('Upscaling the image %dx' % upscale)
140
+ wcs_orig = wcs
141
+ wcs = astrometry.upscale_wcs(wcs, upscale, will_rebin=True)
142
+ params['width'] *= upscale
143
+ params['height'] *= upscale
144
+
145
+ whdr = wcs.to_header(relax=True)
146
+ if whdr['CTYPE1'] == 'RA---TPV' and 'PV1_0' not in whdr.keys():
147
+ # hips2fits does not like such headers, let's fix it
148
+ whdr['CTYPE1'] = 'RA---TAN'
149
+ whdr['CTYPE2'] = 'DEC--TAN'
150
+
151
+ params['wcs'] = json.dumps(dict(whdr))
152
+ elif ra is not None and dec is not None and fov is not None:
153
+ params['ra'] = ra
154
+ params['dec'] = dec
155
+ params['fov'] = fov
156
+ else:
157
+ log('Sky position and size are not provided')
158
+ return (None, None) if get_header else None
159
+
160
+ for baseurl in [
161
+ 'https://alasky.u-strasbg.fr/hips-image-services/hips2fits',
162
+ 'https://alaskybis.u-strasbg.fr/hips-image-services/hips2fits',
163
+ ]:
164
+ url = baseurl + '?' + urlencode(params)
165
+
166
+ try:
167
+ t0 = time.time()
168
+ hdu = fits.open(url)
169
+ t1 = time.time()
170
+ log('Downloaded HiPS image in %.2f s' % (t1 - t0))
171
+ break
172
+ except Exception:
173
+ log('Failed downloading HiPS image from', url)
174
+ hdu = None
175
+
176
+ if hdu is None:
177
+ log('Cannot download HiPS image!')
178
+ if get_header:
179
+ return None, None
180
+ else:
181
+ return None
182
+
183
+ image = hdu[0].data.astype(np.double)
184
+ header = hdu[0].header
185
+
186
+ # FIXME: hips2fits does not properly return the header for SIP WCS, so we just copy the original WCS over it
187
+ if wcs is not None and wcs.sip is not None:
188
+ wheader = wcs.to_header(relax=True)
189
+ header.update(wheader)
190
+
191
+ hdu.close()
192
+
193
+ if asinh is None:
194
+ # All PanSTARRS bands except g are stored in asinh scaling, as of May 2023
195
+ if 'PanSTARRS' in hips and hips != 'PanSTARRS/DR1/g':
196
+ asinh = True
197
+
198
+ if asinh:
199
+ # Fix asinh flux scaling
200
+ # image = np.sinh(image*np.log(10)/2.5)
201
+ x = image * 0.4 * np.log(10)
202
+ image = np.exp(x) - np.exp(-x)
203
+
204
+ if wcs_orig is not None:
205
+ # We should do downscaling after conversion of the image back to linear flux scaling
206
+ log('Downscaling the image %dx' % upscale)
207
+ image = utils.rebin_image(image, upscale)
208
+ # The header from hips2fits describes the upscaled grid - restore the original WCS
209
+ header.update(wcs_orig.to_header(relax=True))
210
+ header['NAXIS1'] = image.shape[1]
211
+ header['NAXIS2'] = image.shape[0]
212
+
213
+ if normalize:
214
+ # Normalize the image to have median=100 and std=10, corresponding to GAIN=1 assuming Poissonian background
215
+ image -= np.nanmedian(image)
216
+ mad = mad_std(image, ignore_nan=True)
217
+ if mad > 0:
218
+ # We have non-zero background noise, let's normalize it to quasi-Poissonian level
219
+ image *= 10 / mad
220
+ image += 100
221
+
222
+ if get_header:
223
+ return image, header
224
+ else:
225
+ return image
226
+
227
+
228
+ def dilate_mask(mask, dilate=5):
229
+ """
230
+ Dilate binary mask with a circular kernel of a given radius
231
+ """
232
+
233
+ kernel = Tophat2DKernel(dilate).array > 0
234
+ return binary_dilation(mask, kernel)
235
+
236
+
237
+ def mask_template(
238
+ tmpl,
239
+ cat=None,
240
+ cat_saturation_mag=None,
241
+ cat_col_mag='rmag',
242
+ cat_col_mag_err='e_rmag',
243
+ cat_col_ra='RAJ2000',
244
+ cat_col_dec='DEJ2000',
245
+ cat_sr=1 / 3600,
246
+ mask_nans=True,
247
+ mask_masked=True,
248
+ mask_photometric=False,
249
+ aper=2,
250
+ sn=5,
251
+ wcs=None,
252
+ dilate=5,
253
+ verbose=False,
254
+ _tmpdir=None,
255
+ ):
256
+ """Apply masking heuristics to a template image.
257
+
258
+ The following masking steps are applied (where enabled):
259
+
260
+ 1. NaN pixels are masked if ``mask_nans=True``.
261
+ 2. If ``cat`` is provided and ``wcs`` is set:
262
+
263
+ - Stars brighter than ``cat_saturation_mag`` have their central pixel
264
+ masked.
265
+ - Stars with masked or NaN magnitudes are masked if ``mask_masked=True``.
266
+ - Photometrically saturated stars are masked if
267
+ ``mask_photometric=True``.
268
+
269
+ 3. The mask is optionally dilated by ``dilate`` pixels.
270
+
271
+ Parameters
272
+ ----------
273
+ tmpl : ndarray
274
+ Template image array.
275
+ cat : astropy.table.Table, optional
276
+ Reference catalogue for catalogue-based masking.
277
+ cat_saturation_mag : float, optional
278
+ Stars brighter than this magnitude have their central pixel masked.
279
+ cat_col_mag : str, optional
280
+ Catalogue column name for magnitude.
281
+ cat_col_mag_err : str, optional
282
+ Catalogue column name for magnitude error.
283
+ cat_col_ra : str, optional
284
+ Catalogue column name for Right Ascension.
285
+ cat_col_dec : str, optional
286
+ Catalogue column name for Declination.
287
+ cat_sr : float, optional
288
+ Catalogue matching radius in degrees.
289
+ mask_nans : bool, optional
290
+ If True, mask NaN pixels in the template.
291
+ mask_masked : bool, optional
292
+ If True, mask catalogue stars whose magnitudes are masked or NaN.
293
+ mask_photometric : bool, optional
294
+ If True, apply photometric-based masking to identify saturated stars.
295
+ aper : float, optional
296
+ Aperture radius (pixels) for photometry in photometric masking.
297
+ sn : float, optional
298
+ Minimum S/N for star detection in photometric masking.
299
+ wcs : astropy.wcs.WCS, optional
300
+ WCS of the template image.
301
+ dilate : int, optional
302
+ Dilation kernel size in pixels for expanding masked regions.
303
+ verbose : bool or callable, optional
304
+ Whether to show verbose messages. May be boolean or a ``print``-like callable.
305
+ _tmpdir : str, optional
306
+ Directory for temporary files created during photometric masking.
307
+
308
+ Returns
309
+ -------
310
+ ndarray of bool
311
+ Boolean mask for the template image (``True`` = masked).
312
+ """
313
+
314
+ # Simple wrapper around print for logging in verbose mode only
315
+ log = (verbose if callable(verbose) else print) if verbose else lambda *args, **kwargs: None
316
+
317
+ if mask_nans:
318
+ tmask = ~np.isfinite(tmpl)
319
+ log(np.sum(tmask), 'template pixels masked after NaN checking')
320
+ else:
321
+ tmask = np.zeros_like(tmpl, dtype=bool)
322
+
323
+ if cat is not None and wcs is None:
324
+ log('Catalogue provided but no WCS, skipping catalogue-based masking')
325
+
326
+ if cat is not None and wcs is not None:
327
+ # Mask the central pixels of saturated stars
328
+ cx, cy = wcs.all_world2pix(cat[cat_col_ra], cat[cat_col_dec], 0, quiet=True)
329
+ # Objects that failed to project will be rejected by the bounds check below
330
+ cx = np.where(np.isfinite(cx), np.round(cx), -1).astype(int)
331
+ cy = np.where(np.isfinite(cy), np.round(cy), -1).astype(int)
332
+
333
+ tidx = np.zeros(len(cat), dtype=bool)
334
+
335
+ # First, we select all catalogue objects with masked measurements
336
+ if mask_masked:
337
+ mag_mask = getattr(cat[cat_col_mag], 'mask', np.zeros(len(cat), dtype=bool))
338
+ tidx = np.asarray(mag_mask, dtype=bool)
339
+ tidx |= ~np.isfinite(cat[cat_col_mag])
340
+ if cat_col_mag_err:
341
+ err_mask = getattr(cat[cat_col_mag_err], 'mask', np.zeros(len(cat), dtype=bool))
342
+ tidx |= np.asarray(err_mask, dtype=bool)
343
+ tidx |= ~np.isfinite(cat[cat_col_mag_err])
344
+
345
+ # Next, we also add the ones corresponding to saturation limit
346
+ if cat_saturation_mag is not None:
347
+ tidx |= cat[cat_col_mag] < cat_saturation_mag
348
+
349
+ # Also, we may mask photometrically saturated stars
350
+ if mask_photometric:
351
+ # Detect the stars on the template and match with the catalogue
352
+ tobj = photometry.get_objects_sextractor(
353
+ tmpl, mask=tmask, sn=sn, aper=aper, wcs=wcs, _tmpdir=_tmpdir
354
+ )
355
+ # tobj = photometry.measure_objects(tobj, tmpl, mask=tmask, fwhm=np.median(tobj['fwhm'][tobj['flags'] == 0]), aper=1)
356
+
357
+ tm = pipeline.calibrate_photometry(
358
+ tobj,
359
+ cat,
360
+ sr=cat_sr,
361
+ cat_col_mag=cat_col_mag,
362
+ cat_col_mag_err=cat_col_mag_err,
363
+ cat_col_ra=cat_col_ra,
364
+ cat_col_dec=cat_col_dec,
365
+ order=0,
366
+ accept_flags=0x02,
367
+ robust=True,
368
+ scale_noise=True,
369
+ )
370
+
371
+ if tm is not None:
372
+ idx = ~tm['idx'] & (tm['zero'] - tm['zero_model'] < -0.1)
373
+ tidx[tm['cidx'][idx]] = True
374
+ else:
375
+ log('Photometric calibration failed, skipping photometric masking')
376
+
377
+ # ..and keep only the ones inside the image
378
+ tidx &= (cx >= 0) & (cx <= tmpl.shape[1] - 1)
379
+ tidx &= (cy >= 0) & (cy <= tmpl.shape[0] - 1)
380
+
381
+ tmask[cy[tidx], cx[tidx]] = True
382
+
383
+ if cat_saturation_mag is not None:
384
+ log(
385
+ np.sum(tmask),
386
+ 'template pixels masked after checking saturated (%s < %.1f) stars'
387
+ % (cat_col_mag, cat_saturation_mag),
388
+ )
389
+ else:
390
+ log(np.sum(tmask), 'template pixels masked after catalogue checking')
391
+
392
+ if dilate and dilate > 0:
393
+ log('Dilating the mask with circular kernel of radius %d' % dilate)
394
+ tmask = dilate_mask(tmask, dilate)
395
+ log(np.sum(tmask), 'template pixels masked after dilation')
396
+
397
+ return tmask
398
+
399
+
400
+ # Pan-STARRS images
401
+ __ps1_skycells = None
402
+
403
+
404
+ def point_in_ps1(ra, dec):
405
+ """
406
+ Whether the sky point is covered by Pan-STARRS, rough estimation
407
+ """
408
+ return dec > -30
409
+
410
+
411
+ # Legacy Survey images
412
+ __ls_skycells = None
413
+
414
+
415
+ def point_in_ls(ra, dec):
416
+ """
417
+ Whether the sky point is covered by Legacy Survey, rough estimation
418
+ """
419
+ sc = SkyCoord(ra, dec, unit='deg')
420
+ res = np.abs(sc.galactic.b.deg) > 20
421
+
422
+ return res
423
+
424
+
425
+ def _filter_cells_by_footprint(cell_ra, cell_dec, cell_radius, wcs, width, height):
426
+ """Return boolean mask of cells that overlap the image footprint.
427
+
428
+ After the coarse spherical-match preselection, this keeps only the
429
+ cells whose centre lies within ``cell_radius`` (degrees) of the image
430
+ footprint.
431
+
432
+ The test is performed in the *forward* WCS direction: the image pixel
433
+ grid is projected onto the sky with :meth:`~astropy.wcs.WCS.all_pix2world`
434
+ and every cell centre is compared against those samples. This avoids
435
+ inverting the WCS (``all_world2pix``), whose distortion de-projection may
436
+ fail to converge for points lying well outside the image - as happens for
437
+ high-order TPV/SIP solutions, where it previously raised
438
+ ``InvalidCoordinateError`` and aborted template retrieval.
439
+ """
440
+ from astropy.wcs.utils import proj_plane_pixel_scales
441
+
442
+ pixscale = np.mean(proj_plane_pixel_scales(wcs)) # deg/pixel
443
+
444
+ # Sample the image on a grid fine enough that neighbouring samples stay
445
+ # well below one cell radius apart, so no overlapping cell is missed.
446
+ step = max(1.0, 0.5 * cell_radius / pixscale) # pixels
447
+ nx = int(min(200, max(2, np.ceil(width / step) + 1)))
448
+ ny = int(min(200, max(2, np.ceil(height / step) + 1)))
449
+ xx, yy = np.meshgrid(np.linspace(0, width, nx), np.linspace(0, height, ny))
450
+
451
+ # Forward projection only - never needs distortion inversion
452
+ sample_ra, sample_dec = wcs.all_pix2world(xx.ravel(), yy.ravel(), 0)
453
+ samples = SkyCoord(sample_ra, sample_dec, unit='deg')
454
+
455
+ cells = SkyCoord(
456
+ np.asarray(cell_ra, dtype=float), np.asarray(cell_dec, dtype=float), unit='deg'
457
+ )
458
+
459
+ # Angular distance from each cell centre to the nearest image sample
460
+ _, sep, _ = cells.match_to_catalog_sky(samples)
461
+
462
+ return sep.deg < cell_radius
463
+
464
+
465
+ def find_skycells(
466
+ ra, dec, sr, band='r', ext='image', survey='ps1', wcs=None, width=None, height=None
467
+ ):
468
+ """Find survey skycell URLs covering a sky region.
469
+
470
+ Parameters
471
+ ----------
472
+ ra, dec, sr : float
473
+ Centre and radius (degrees) of the search region.
474
+ band : str
475
+ Photometric band.
476
+ ext : str
477
+ ``'image'`` or ``'mask'``.
478
+ survey : str
479
+ ``'ps1'`` or ``'ls'``.
480
+ wcs : `~astropy.wcs.WCS`, optional
481
+ If provided together with *width*/*height*, used to reject
482
+ cells that fall outside the actual rectangular image footprint
483
+ (the default circular search can over-select for elongated
484
+ or rotated fields).
485
+ width, height : int, optional
486
+ Image dimensions in pixels (required when *wcs* is given).
487
+ """
488
+ results = []
489
+
490
+ if survey == 'ps1':
491
+ global __ps1_skycells
492
+
493
+ if __ps1_skycells is None:
494
+ # Load skycells information and store to global variable
495
+ __ps1_skycells = Table.read(utils.get_data_path('ps1skycells.txt'), format='ascii')
496
+
497
+ cell_radius = 0.3
498
+
499
+ # Coarse spherical preselection
500
+ _, idx, _ = astrometry.spherical_match(
501
+ ra, dec, __ps1_skycells['ra0'], __ps1_skycells['dec0'], sr + cell_radius
502
+ )
503
+
504
+ candidates = __ps1_skycells[idx]
505
+
506
+ # Refine against actual rectangular footprint when WCS is available
507
+ if wcs is not None and width is not None and height is not None:
508
+ keep = _filter_cells_by_footprint(
509
+ candidates['ra0'],
510
+ candidates['dec0'],
511
+ cell_radius,
512
+ wcs,
513
+ width,
514
+ height,
515
+ )
516
+ candidates = candidates[keep]
517
+
518
+ for cell in candidates:
519
+ url = 'https://ps1images.stsci.edu/'
520
+ url += 'rings.v3.skycell/%04d/%03d/' % (
521
+ cell['projectionID'],
522
+ cell['skyCellID'],
523
+ )
524
+ url += 'rings.v3.skycell.%04d.%03d' % (
525
+ cell['projectionID'],
526
+ cell['skyCellID'],
527
+ )
528
+ url += '.stk.%s.unconv%s.fits' % (band, '.' + ext if ext != 'image' else '')
529
+
530
+ results.append(url)
531
+
532
+ elif survey == 'ls':
533
+ global __ls_skycells
534
+
535
+ if __ls_skycells is None:
536
+ # Load skycells information and store to global variable
537
+ __ls_skycells = Table.read(
538
+ utils.get_data_path('legacysurvey_bricks.fits.gz'), format='fits'
539
+ )
540
+
541
+ cell_radius = 0.186
542
+
543
+ # Coarse spherical preselection
544
+ _, idx, _ = astrometry.spherical_match(
545
+ ra, dec, __ls_skycells['ra'], __ls_skycells['dec'], sr + cell_radius
546
+ )
547
+
548
+ candidates = __ls_skycells[idx]
549
+
550
+ # Refine against actual rectangular footprint when WCS is available
551
+ if wcs is not None and width is not None and height is not None:
552
+ keep = _filter_cells_by_footprint(
553
+ candidates['ra'],
554
+ candidates['dec'],
555
+ cell_radius,
556
+ wcs,
557
+ width,
558
+ height,
559
+ )
560
+ candidates = candidates[keep]
561
+
562
+ for cell in candidates:
563
+ url = 'https://portal.nersc.gov/cfs/cosmo/data/legacysurvey/'
564
+ url += 'dr10/south/' if (cell['survey'] == 'S') else 'dr9/north/'
565
+ url += 'coadd/%s/%s/' % (cell['brickname'][:3], cell['brickname'])
566
+ if ext == 'mask':
567
+ url += (
568
+ 'legacysurvey-%s-maskbits' % (cell['brickname'])
569
+ ) # Legacy Survey uses single mask for all bands
570
+ else:
571
+ url += 'legacysurvey-%s-image-%s' % (cell['brickname'], band)
572
+ url += '.fits.fz'
573
+
574
+ results.append(url)
575
+
576
+ else:
577
+ raise RuntimeError('Unsupported survey %s' % survey)
578
+
579
+ return results
580
+
581
+
582
+ def fits_open_remote(url, **kwargs):
583
+ """
584
+ Simple wrapper around astropy.io.fits.open() that handles cache corrution errors
585
+ and downloads the image directly if necessary
586
+ """
587
+ try:
588
+ # First try opening using the cache
589
+ hdu = fits.open(url, **kwargs)
590
+ except OSError:
591
+ # If that fails, cache may be corrupted - let's skip it
592
+ kwargs.update({'cache': False})
593
+
594
+ try:
595
+ hdu = fits.open(url, **kwargs)
596
+ except Exception:
597
+ import traceback
598
+
599
+ traceback.print_exc()
600
+
601
+ return None
602
+
603
+ return hdu
604
+
605
+
606
+ def get_skycells(
607
+ ra0,
608
+ dec0,
609
+ sr0,
610
+ band='r',
611
+ ext='image',
612
+ survey='ps1',
613
+ normalize=True,
614
+ overwrite=False,
615
+ wcs=None,
616
+ width=None,
617
+ height=None,
618
+ _cachedir=None,
619
+ _cache_downscale=1,
620
+ _tmpdir=None,
621
+ verbose=False,
622
+ ):
623
+ """
624
+ Get the list of filenames corresponding to skycells in the user-specified sky region.
625
+ The cells are downloaded and stored to the specified cache location.
626
+
627
+ To get the masks, use `ext='mask'`.
628
+
629
+ Downloaded skycells are (optionally) normalized according to the parameters from their FITS headers.
630
+ """
631
+
632
+ # Simple wrapper around print for logging in verbose mode only
633
+ log = (verbose if callable(verbose) else print) if verbose else lambda *args, **kwargs: None
634
+
635
+ # Normalize _cachedir
636
+ if _cachedir is not None:
637
+ _cachedir = os.path.expanduser(_cachedir)
638
+ log('Cache location is', _cachedir)
639
+ else:
640
+ if _tmpdir is None:
641
+ _tmpdir = tempfile.gettempdir()
642
+ _cachedir = os.path.join(_tmpdir, survey)
643
+ log('Cache location not specified, falling back to %s' % _cachedir)
644
+
645
+ # Ensure the cache dir exists
646
+ os.makedirs(_cachedir, exist_ok=True)
647
+
648
+ filenames = []
649
+
650
+ cells = find_skycells(
651
+ ra0, dec0, sr0, band=band, ext=ext, survey=survey, wcs=wcs, width=width, height=height
652
+ )
653
+
654
+ for cell in cells:
655
+ cellname = os.path.basename(cell)
656
+ filename = os.path.join(_cachedir, cellname)
657
+
658
+ if survey == 'ls' and filename.endswith('.fz'):
659
+ filename = os.path.splitext(filename)[0]
660
+
661
+ if _cache_downscale > 1:
662
+ filename, fext = os.path.splitext(filename)
663
+ filename = filename + ('.x%d' % _cache_downscale) + fext
664
+
665
+ if os.path.exists(filename) and not overwrite:
666
+ log('%s already downloaded' % os.path.split(filename)[-1])
667
+ else:
668
+ log('Downloading %s' % cellname)
669
+
670
+ hdu = fits_open_remote(cell)
671
+ if hdu is not None:
672
+ image, header = hdu[1].data, hdu[1].header
673
+
674
+ if normalize:
675
+ if survey == 'ps1':
676
+ image, header = normalize_ps1_skycell(image, header)
677
+
678
+ if survey == 'ls' and ext == 'image':
679
+ try:
680
+ # Get invvar file to mask not covered regions
681
+ ihdu = fits_open_remote(cell.replace('-image-', '-invvar-'))
682
+ if ihdu is not None:
683
+ invvar = ihdu[1].data
684
+ image[invvar == 0] = np.nan
685
+ ihdu.close()
686
+ except Exception:
687
+ pass
688
+
689
+ if _cache_downscale > 1:
690
+ image, header = cutouts.downscale_image(
691
+ image,
692
+ header=header,
693
+ scale=_cache_downscale,
694
+ mode='or' if ext == 'mask' else 'sum',
695
+ )
696
+
697
+ log(
698
+ "Downscaling the image and storing it as",
699
+ os.path.split(filename)[-1],
700
+ )
701
+
702
+ # Write atomically so that an interrupted write does not leave
703
+ # a truncated file in the cache
704
+ fits.writeto(filename + '.tmp', image, header, overwrite=True)
705
+ os.replace(filename + '.tmp', filename)
706
+
707
+ hdu.close()
708
+
709
+ if os.path.exists(filename):
710
+ filenames.append(filename)
711
+
712
+ return filenames
713
+
714
+
715
+ def normalize_ps1_skycell(image, header, verbose=False):
716
+ """
717
+ Normalize PanSTARRS skycell file according to its FITS header
718
+ """
719
+
720
+ # Simple wrapper around print for logging in verbose mode only
721
+ log = (verbose if callable(verbose) else print) if verbose else lambda *args, **kwargs: None
722
+
723
+ header = header.copy()
724
+
725
+ if 'RADESYS' not in header and 'PC001001' in header:
726
+ # Normalize WCS in the header
727
+ log('Normalizing WCS keywords')
728
+
729
+ header['RADESYS'] = 'FK5'
730
+ header.rename_keyword('PC001001', 'PC1_1')
731
+ header.rename_keyword('PC001002', 'PC1_2')
732
+ header.rename_keyword('PC002001', 'PC2_1')
733
+ header.rename_keyword('PC002002', 'PC2_2')
734
+
735
+ if 'BSOFTEN' in header and 'BOFFSET' in header:
736
+ # Linearize ASINH scaling
737
+ log('Normalizing ASINH scaling')
738
+
739
+ x = image * 0.4 * np.log(10)
740
+ image = header['BOFFSET'] + header['BSOFTEN'] * (np.exp(x) - np.exp(-x))
741
+ if header.get('EXPTIME'):
742
+ image /= header['EXPTIME'] # For common photometric zero-point
743
+
744
+ for _ in ['BSOFTEN', 'BOFFSET', 'BLANK']:
745
+ header.remove(_, ignore_missing=True)
746
+
747
+ return image, header
748
+
749
+
750
+ # Higher level retrieval function
751
+ def get_survey_image(
752
+ band='r',
753
+ ext='image',
754
+ survey='ps1',
755
+ wcs=None,
756
+ shape=None,
757
+ width=None,
758
+ height=None,
759
+ header=None,
760
+ reproject='lanczos',
761
+ extra=None,
762
+ _cachedir=None,
763
+ _cache_downscale=1,
764
+ _tmpdir=None,
765
+ _workdir=None,
766
+ verbose=False,
767
+ **kwargs,
768
+ ):
769
+ """Download and reproject a Pan-STARRS or Legacy Survey image onto a WCS grid.
770
+
771
+ Pan-STARRS images are converted from asinh scaling to linear flux.
772
+ Pan-STARRS mask bits: https://outerspace.stsci.edu/display/PANSTARRS/PS1+Pixel+flags+in+Image+Table+Data
773
+ Legacy Survey mask bits: https://www.legacysurvey.org/dr10/bitmasks
774
+
775
+ Parameters
776
+ ----------
777
+ band : str, optional
778
+ Photometric band: ``'g'``, ``'r'``, ``'i'``, ``'z'``, or ``'y'``.
779
+ ext : str, optional
780
+ Data type: ``'image'`` or ``'mask'``.
781
+ survey : str, optional
782
+ Survey name: ``'ps1'`` for Pan-STARRS or ``'ls'`` for Legacy Survey.
783
+ wcs : astropy.wcs.WCS, optional
784
+ Output WCS projection.
785
+ shape : tuple of int, optional
786
+ Output image shape ``(height, width)``; alternative to ``width`` / ``height``.
787
+ width : int, optional
788
+ Output image width in pixels.
789
+ height : int, optional
790
+ Output image height in pixels.
791
+ header : astropy.io.fits.Header, optional
792
+ Header containing WCS and image dimensions; alternative to providing
793
+ them explicitly.
794
+ reproject : str, optional
795
+ Reprojection method: ``'lanczos'`` (default, pure-Python with flux
796
+ conservation) or ``'swarp'`` (external SWarp binary).
797
+ extra : dict, optional
798
+ Extra SWarp parameters (only used when ``reproject='swarp'``).
799
+ _cachedir : str, optional
800
+ Directory for caching downloaded sky cells between calls.
801
+ _cache_downscale : int, optional
802
+ Integer downscale factor for cached images.
803
+ _tmpdir : str, optional
804
+ Directory for temporary files.
805
+ _workdir : str, optional
806
+ If specified, temporary SWarp files are kept for debugging.
807
+ verbose : bool or callable, optional
808
+ Whether to show verbose messages. May be boolean or a ``print``-like callable.
809
+ **kwargs
810
+ Additional keyword arguments passed to the reprojection function.
811
+
812
+ Returns
813
+ -------
814
+ ndarray or None
815
+ Image reprojected onto the requested pixel grid, or None on failure.
816
+ """
817
+
818
+ # Simple wrapper around print for logging in verbose mode only
819
+ log = (verbose if callable(verbose) else print) if verbose else lambda *args, **kwargs: None
820
+
821
+ # Resolve WCS and dimensions early so we can use them for cell filtering
822
+ if wcs is None:
823
+ if header is None:
824
+ raise ValueError('Either wcs or header should be provided')
825
+ wcs = WCS(header)
826
+ if shape is not None:
827
+ height, width = shape
828
+ if (width is None or height is None) and header is None:
829
+ raise ValueError('Either shape, width/height or header should be provided')
830
+ if width is None:
831
+ width = header['NAXIS1']
832
+ if height is None:
833
+ height = header['NAXIS2']
834
+
835
+ ra0, dec0, sr0 = astrometry.get_frame_center(wcs=wcs, width=width, height=height)
836
+
837
+ cellnames = get_skycells(
838
+ ra0,
839
+ dec0,
840
+ sr0,
841
+ band=band,
842
+ ext=ext,
843
+ survey=survey,
844
+ wcs=wcs,
845
+ width=width,
846
+ height=height,
847
+ _cachedir=_cachedir,
848
+ _cache_downscale=_cache_downscale,
849
+ _tmpdir=_tmpdir,
850
+ verbose=verbose,
851
+ )
852
+
853
+ if reproject == 'lanczos':
854
+ coadd = reproject_lanczos(
855
+ cellnames,
856
+ wcs=wcs,
857
+ width=width,
858
+ height=height,
859
+ is_flags=(ext == 'mask'),
860
+ verbose=verbose,
861
+ **kwargs,
862
+ )
863
+ elif reproject == 'swarp':
864
+ coadd = reproject_swarp(
865
+ cellnames,
866
+ wcs=wcs,
867
+ width=width,
868
+ height=height,
869
+ is_flags=(ext == 'mask'),
870
+ extra=extra,
871
+ _tmpdir=_tmpdir,
872
+ _workdir=_workdir,
873
+ verbose=verbose,
874
+ **kwargs,
875
+ )
876
+ else:
877
+ log("Unknown reproject method '%s', use 'lanczos' or 'swarp'" % reproject)
878
+ return None
879
+
880
+ if coadd is None:
881
+ log('No skycells were reprojected onto the requested grid')
882
+ return None
883
+
884
+ if ext == 'mask' and survey == 'ps1':
885
+ coadd &= (
886
+ 0xFFFF - 0x8000
887
+ ) # Remove undocumented PS1 'temporary marked' mask bit that is masking seemingly good pixels
888
+
889
+ return coadd
890
+
891
+
892
+ def get_survey_image_and_mask(band='r', **kwargs):
893
+ """Download both the image and mask from Pan-STARRS or Legacy Survey.
894
+
895
+ Convenience wrapper that calls :func:`get_survey_image` twice.
896
+
897
+ Parameters
898
+ ----------
899
+ band : str, optional
900
+ Photometric band: ``'g'``, ``'r'``, ``'i'``, ``'z'``, or ``'y'``.
901
+ **kwargs
902
+ All other keyword arguments are passed to :func:`get_survey_image`.
903
+
904
+ Returns
905
+ -------
906
+ tuple of ndarray
907
+ ``(image, mask)`` reprojected onto the requested pixel grid.
908
+ """
909
+
910
+ image = get_survey_image(band=band, ext='image', **kwargs)
911
+ mask = get_survey_image(band=band, ext='mask', **kwargs)
912
+
913
+ return image, mask
914
+
915
+
916
+ def get_ps1_image(band='r', **kwargs):
917
+ return get_survey_image(band=band, survey='ps1', **kwargs)
918
+
919
+
920
+ def get_ps1_image_and_mask(band='r', **kwargs):
921
+ return get_survey_image_and_mask(band=band, survey='ps1', **kwargs)
922
+
923
+
924
+ def get_ls_image(band='r', **kwargs):
925
+ return get_survey_image(band=band, survey='ls', **kwargs)
926
+
927
+
928
+ def get_ls_image_and_mask(band='r', **kwargs):
929
+ return get_survey_image_and_mask(band=band, survey='ls', **kwargs)