stdpipe 0.4.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
stdpipe/realbogus.py ADDED
@@ -0,0 +1,971 @@
1
+ """
2
+ Real-Bogus Classifier for Astronomical Object Detection
3
+
4
+ This module provides a CNN-based classifier to distinguish real astronomical sources
5
+ (stars, galaxies) from artifacts (cosmic rays, hot pixels, satellite trails) in
6
+ detected object catalogs.
7
+
8
+ Key Features:
9
+ - FWHM-invariant: Hybrid downscaling to canonical PSF size (no auxiliary FWHM input)
10
+ - Brightness-invariant: Peak normalization generalizes to any flux level
11
+ - Pure morphology: Classification based solely on source shape from 2-channel images
12
+ - 2-channel input: background-subtracted (linear), asinh-scaled (dynamic range compression)
13
+ - Lightweight 5-layer CNN (~100k parameters)
14
+ - Batch processing for efficient inference
15
+ - Optional TensorFlow dependency
16
+
17
+ Usage:
18
+ from stdpipe import photometry, realbogus
19
+
20
+ # Detect objects
21
+ obj = photometry.get_objects_sep(image, thresh=3.0)
22
+
23
+ # Classify and filter
24
+ obj_clean = realbogus.classify_realbogus(obj, image, threshold=0.5)
25
+
26
+ # Or add scores without filtering
27
+ obj = realbogus.classify_realbogus(obj, image, add_score=True, flag_bogus=False)
28
+ print(obj['rb_score'])
29
+
30
+ Author: STDPipe Contributors
31
+ """
32
+
33
+ from __future__ import absolute_import, division, print_function
34
+
35
+ import numpy as np
36
+ import os
37
+ import warnings
38
+ from astropy.table import Column
39
+
40
+ from .realbogus_features import FLAG_BOGUS
41
+
42
+ # Conditional TensorFlow import
43
+ try:
44
+ import tensorflow as tf
45
+ from tensorflow import keras
46
+ from tensorflow.keras import layers, models
47
+
48
+ HAS_TENSORFLOW = True
49
+ except ImportError:
50
+ HAS_TENSORFLOW = False
51
+ warnings.warn(
52
+ "TensorFlow not found. Real-bogus classifier will not be available. "
53
+ "Install with: pip install stdpipe[ml]",
54
+ ImportWarning,
55
+ )
56
+
57
+
58
+ # Model architecture parameters
59
+ DEFAULT_MODEL_DIR = os.path.expanduser('~/.stdpipe/models')
60
+ DEFAULT_MODEL_NAME = 'realbogus_default.h5'
61
+ TARGET_FWHM = 3.0 # Canonical FWHM for downscaling normalization
62
+ DEFAULT_ASINH_SOFTENING_SIGMA = 3.0 # Asinh softening in units of background sigma
63
+
64
+
65
+ def _check_tensorflow():
66
+ """Check if TensorFlow is available, raise error if not."""
67
+ if not HAS_TENSORFLOW:
68
+ raise ImportError(
69
+ "TensorFlow is required for real-bogus classification. "
70
+ "Install with: pip install stdpipe[ml] or pip install tensorflow>=2.10"
71
+ )
72
+
73
+
74
+ def create_realbogus_model(
75
+ input_shape=(31, 31, 2), filters=(32, 64, 128), dense_units=64, dropout_rate=0.5
76
+ ):
77
+ """
78
+ Create CNN architecture for real-bogus classification.
79
+
80
+ Architecture:
81
+ - 3-5 convolutional layers with batch normalization
82
+ - Global average pooling (handles variable input sizes)
83
+ - Dense layer with dropout
84
+ - Sigmoid output (binary classification)
85
+
86
+ Design Philosophy:
87
+ - FWHM-invariant: Images downscaled to canonical FWHM, no auxiliary FWHM input needed
88
+ - Brightness-invariant: Peak normalization allows generalization to any flux level
89
+ - Pure morphology: Classification based solely on source shape
90
+
91
+ Input Channels:
92
+ - Channel 0: Background-subtracted (linear scale), peak-normalized
93
+ - Channel 1: Asinh-scaled background-subtracted, peak-normalized
94
+
95
+ Parameters
96
+ ----------
97
+ input_shape : tuple, optional
98
+ Input shape (height, width, channels). Default: (31, 31, 2)
99
+ Height/width can be None for variable-size inputs.
100
+ filters : tuple, optional
101
+ Number of filters in each conv layer. Default: (32, 64, 128)
102
+ dense_units : int, optional
103
+ Units in dense layer. Default: 64
104
+ dropout_rate : float, optional
105
+ Dropout rate for regularization. Default: 0.5
106
+
107
+ Returns
108
+ -------
109
+ model : keras.Model
110
+ Compiled Keras model ready for training
111
+ """
112
+ _check_tensorflow()
113
+
114
+ # Main image input (2 channels: background-subtracted linear, asinh-scaled)
115
+ image_input = keras.Input(shape=input_shape, name='image_input')
116
+
117
+ # Convolutional feature extraction
118
+ x = image_input
119
+ for i, n_filters in enumerate(filters):
120
+ x = layers.Conv2D(
121
+ n_filters, (3, 3), activation='relu', padding='same', name=f'conv{i + 1}'
122
+ )(x)
123
+ x = layers.BatchNormalization(name=f'bn{i + 1}')(x)
124
+ x = layers.MaxPooling2D((2, 2), name=f'pool{i + 1}')(x)
125
+
126
+ # Global pooling to handle variable input sizes
127
+ x = layers.GlobalAveragePooling2D(name='global_pool')(x)
128
+
129
+ # Dense layers (no auxiliary inputs - pure image-based classification)
130
+ x = layers.Dense(dense_units, activation='relu', name='dense1')(x)
131
+ x = layers.Dropout(dropout_rate, name='dropout')(x)
132
+
133
+ # Output layer (sigmoid for binary classification)
134
+ output = layers.Dense(1, activation='sigmoid', name='output')(x)
135
+
136
+ # Create and compile model
137
+ model = keras.Model(inputs=image_input, outputs=output, name='realbogus_classifier')
138
+
139
+ model.compile(
140
+ optimizer=keras.optimizers.Adam(learning_rate=0.001),
141
+ loss='binary_crossentropy',
142
+ metrics=['accuracy', keras.metrics.AUC(name='auc')],
143
+ )
144
+
145
+ return model
146
+
147
+
148
+ def _downscale_cutout(cutout, scale_factor, mode='mean'):
149
+ """
150
+ Downscale cutout by integer factor using block averaging.
151
+
152
+ Parameters
153
+ ----------
154
+ cutout : ndarray
155
+ 2D image cutout
156
+ scale_factor : int or float
157
+ Downscaling factor (converted to integer, must be >= 1).
158
+ Typically passed as round(fwhm/target_fwhm) for robust hot pixel suppression.
159
+ mode : str, optional
160
+ Downscaling mode: 'mean' or 'median'. Default: 'mean'
161
+
162
+ Returns
163
+ -------
164
+ downscaled : ndarray
165
+ Downscaled cutout
166
+ """
167
+ if scale_factor <= 1:
168
+ return cutout
169
+
170
+ scale_factor = int(scale_factor)
171
+ h, w = cutout.shape
172
+
173
+ # Trim to multiple of scale_factor
174
+ new_h = (h // scale_factor) * scale_factor
175
+ new_w = (w // scale_factor) * scale_factor
176
+ cutout_trimmed = cutout[:new_h, :new_w]
177
+
178
+ # Reshape and aggregate
179
+ reshaped = cutout_trimmed.reshape(
180
+ new_h // scale_factor, scale_factor, new_w // scale_factor, scale_factor
181
+ )
182
+
183
+ if mode == 'median':
184
+ downscaled = np.median(reshaped, axis=(1, 3))
185
+ else: # mean
186
+ downscaled = np.mean(reshaped, axis=(1, 3))
187
+
188
+ return downscaled
189
+
190
+
191
+ def _infer_cutout_radius_from_model(model, default_radius=15, log=None):
192
+ """Infer cutout radius from model input shape."""
193
+ if log is None:
194
+ log = lambda *args, **kwargs: None
195
+
196
+ shape = model.input_shape
197
+ if isinstance(shape, list):
198
+ shape = shape[0] if shape else None
199
+
200
+ if shape is None or len(shape) < 4:
201
+ log(
202
+ f"Model input shape unavailable; using default cutout size {2 * default_radius + 1} "
203
+ f"(radius {default_radius})"
204
+ )
205
+ return default_radius
206
+
207
+ height, width = shape[1], shape[2]
208
+ channels = shape[3]
209
+
210
+ if height is None or width is None:
211
+ log(
212
+ f"Model input shape is dynamic; using default cutout size {2 * default_radius + 1} "
213
+ f"(radius {default_radius})"
214
+ )
215
+ return default_radius
216
+
217
+ if height != width:
218
+ raise ValueError(
219
+ f"Model input shape must be square for cutout extraction (got {height}x{width})."
220
+ )
221
+ if height % 2 == 0:
222
+ raise ValueError(f"Model input size must be odd for symmetric cutouts (got {height}).")
223
+
224
+ radius = int((height - 1) // 2)
225
+ log(f"Using cutout size {height}x{width} (radius {radius}) from model input shape")
226
+
227
+ if channels is not None and channels != 2:
228
+ log(f"Warning: model expects {channels} channels; realbogus generates 2-channel cutouts")
229
+
230
+ return radius
231
+
232
+
233
+ def _upscale_cutout(cutout, scale_factor):
234
+ """
235
+ Upscale cutout by integer factor using pixel replication.
236
+
237
+ Parameters
238
+ ----------
239
+ cutout : ndarray
240
+ 2D image cutout
241
+ scale_factor : int or float
242
+ Upscaling factor (converted to integer, must be >= 1).
243
+ Each pixel is replicated into scale_factor x scale_factor block.
244
+
245
+ Returns
246
+ -------
247
+ upscaled : ndarray
248
+ Upscaled cutout
249
+ """
250
+ if scale_factor <= 1:
251
+ return cutout
252
+
253
+ scale_factor = int(scale_factor)
254
+
255
+ # Use numpy repeat for integer upscaling
256
+ # Repeat along each axis
257
+ upscaled = np.repeat(cutout, scale_factor, axis=0)
258
+ upscaled = np.repeat(upscaled, scale_factor, axis=1)
259
+
260
+ return upscaled
261
+
262
+
263
+ def _pad_to_size(cutout, target_size, mode='edge'):
264
+ """
265
+ Pad cutout to target size (centered).
266
+
267
+ Parameters
268
+ ----------
269
+ cutout : ndarray
270
+ 2D image cutout
271
+ target_size : int
272
+ Target size (square)
273
+ mode : str, optional
274
+ Padding mode for np.pad. Default: 'edge'
275
+
276
+ Returns
277
+ -------
278
+ padded : ndarray
279
+ Padded cutout (target_size x target_size)
280
+ """
281
+ h, w = cutout.shape
282
+
283
+ # Crop dimensions that exceed target
284
+ if h > target_size:
285
+ start_h = (h - target_size) // 2
286
+ cutout = cutout[start_h : start_h + target_size, :]
287
+ h = target_size
288
+ if w > target_size:
289
+ start_w = (w - target_size) // 2
290
+ cutout = cutout[:, start_w : start_w + target_size]
291
+ w = target_size
292
+
293
+ # Pad dimensions that are smaller than target
294
+ pad_h = target_size - h
295
+ pad_w = target_size - w
296
+ if pad_h > 0 or pad_w > 0:
297
+ cutout = np.pad(
298
+ cutout, ((pad_h // 2, pad_h - pad_h // 2), (pad_w // 2, pad_w - pad_w // 2)), mode=mode
299
+ )
300
+
301
+ return cutout
302
+
303
+
304
+ def preprocess_cutout(
305
+ cutout_sci,
306
+ cutout_bg=None,
307
+ cutout_err=None,
308
+ fwhm=None,
309
+ target_fwhm=TARGET_FWHM,
310
+ target_size=31,
311
+ downscale_threshold=1.5,
312
+ normalize=True,
313
+ asinh_softening=None,
314
+ ):
315
+ """
316
+ Preprocess cutout for CNN input.
317
+
318
+ Steps:
319
+ 1. Optional scaling (downscale or upscale) to canonical FWHM
320
+ 2. Create 2-channel input (background-subtracted linear, asinh-scaled)
321
+ 3. Peak normalization (each channel normalized by its own peak value)
322
+ 4. Pad/crop to target size
323
+
324
+ FWHM Scaling Strategy (Symmetric):
325
+ - Downscaling (FWHM > target_fwhm × threshold): Integer block averaging
326
+ - No scaling (target_fwhm / threshold ≤ FWHM ≤ target_fwhm × threshold): Keep as-is
327
+ - Upscaling (FWHM < target_fwhm / threshold): Integer pixel replication
328
+
329
+ Default: target_fwhm=3.0, threshold=1.5
330
+ → Downscale if FWHM > 4.5, upscale if FWHM < 2.0, else unchanged
331
+
332
+ This ensures all PSFs normalized to approximately the same size regardless of
333
+ sharpness, eliminating FWHM as a confounding variable.
334
+
335
+ Channel Design:
336
+ - Channel 0: Background-subtracted (linear scale), peak-normalized
337
+ - Channel 1: Asinh-scaled background-subtracted, peak-normalized
338
+
339
+ Peak normalization makes the representation brightness-invariant: all sources
340
+ (faint to extremely bright) are scaled to [-1, 1] range based on their peak
341
+ value. This allows the CNN to learn pure morphological features that generalize
342
+ to ANY brightness level, including sources far brighter than the training set.
343
+
344
+ The asinh channel complements the linear channel by providing compressed
345
+ dynamic range information useful for distinguishing extended vs. compact sources.
346
+
347
+ Parameters
348
+ ----------
349
+ cutout_sci : ndarray
350
+ Science image cutout (assumed to be background-subtracted if cutout_bg is None)
351
+ cutout_bg : ndarray, optional
352
+ Background cutout (or scalar value). If None, estimated from cutout edges.
353
+ cutout_err : ndarray or float, optional
354
+ Error/noise cutout (or scalar value). Used to estimate the noise level
355
+ (sigma) for asinh softening. Only the median value is used.
356
+ fwhm : float, optional
357
+ Image FWHM in pixels. If provided, cutout will be downscaled to target_fwhm.
358
+ target_fwhm : float, optional
359
+ Target FWHM for downscaling normalization. Default: 3.0
360
+ target_size : int, optional
361
+ Target cutout size (square). Default: 31
362
+ downscale_threshold : float, optional
363
+ Only downscale if fwhm/target_fwhm > threshold. Default: 1.5
364
+ normalize : bool, optional
365
+ Apply peak normalization to each channel (scales to [-1, 1] range).
366
+ Default: True. This makes the representation brightness-invariant.
367
+ asinh_softening : float, optional
368
+ Asinh softening in units of background sigma. If None, uses
369
+ DEFAULT_ASINH_SOFTENING_SIGMA. Actual softening is
370
+ (asinh_softening * sigma), where sigma is estimated from cutout_err.
371
+
372
+ Returns
373
+ -------
374
+ preprocessed : ndarray
375
+ Preprocessed cutout (target_size, target_size, 2)
376
+ scale_factor : float
377
+ Applied scale factor (for diagnostics)
378
+ """
379
+ # Handle background
380
+ if cutout_bg is None:
381
+ # Estimate from edges
382
+ edge_pixels = np.concatenate(
383
+ [cutout_sci[0, :], cutout_sci[-1, :], cutout_sci[:, 0], cutout_sci[:, -1]]
384
+ )
385
+ cutout_bg = np.median(edge_pixels)
386
+
387
+ if np.isscalar(cutout_bg):
388
+ cutout_bg = np.full_like(cutout_sci, cutout_bg)
389
+
390
+ # Estimate noise sigma for asinh softening (only the median is needed,
391
+ # so compute before any scaling to avoid unnecessary array operations)
392
+ if cutout_err is None:
393
+ from astropy.stats import mad_std
394
+
395
+ sigma = float(mad_std(cutout_sci))
396
+ elif np.isscalar(cutout_err):
397
+ sigma = float(cutout_err)
398
+ else:
399
+ sigma = float(np.nanmedian(cutout_err))
400
+ if not np.isfinite(sigma) or sigma <= 0:
401
+ sigma = 1.0
402
+
403
+ # Scaling to canonical FWHM (both downscale and upscale)
404
+ scale_factor = 1.0
405
+ if fwhm is not None and fwhm > 0:
406
+ scale_factor = fwhm / target_fwhm
407
+
408
+ # Downscale if PSF too large
409
+ if scale_factor > downscale_threshold:
410
+ factor = round(scale_factor)
411
+ cutout_sci = _downscale_cutout(cutout_sci, factor)
412
+ cutout_bg = _downscale_cutout(cutout_bg, factor)
413
+
414
+ # Upscale if PSF too sharp (symmetric with downscaling)
415
+ elif scale_factor < 1.0 / downscale_threshold:
416
+ factor = round(1.0 / scale_factor)
417
+ cutout_sci = _upscale_cutout(cutout_sci, factor)
418
+ cutout_bg = _upscale_cutout(cutout_bg, factor)
419
+
420
+ # Create background-subtracted channel
421
+ cutout_bgsub = cutout_sci - cutout_bg
422
+
423
+ # Create asinh-scaled channel for dynamic range compression
424
+ if asinh_softening is None:
425
+ asinh_softening = DEFAULT_ASINH_SOFTENING_SIGMA
426
+
427
+ softening = float(asinh_softening) * sigma
428
+ if not np.isfinite(softening) or softening <= 0:
429
+ softening = sigma
430
+
431
+ # Asinh scaling: compresses high values, ~linear for low values
432
+ cutout_asinh = np.arcsinh(cutout_bgsub / softening)
433
+
434
+ # Stack channels: [background-subtracted linear, asinh-scaled]
435
+ channels = [cutout_bgsub, cutout_asinh]
436
+
437
+ # Peak normalization for brightness invariance
438
+ if normalize:
439
+ for i, ch in enumerate(channels):
440
+ peak = np.max(np.abs(ch))
441
+ if peak > 1e-10:
442
+ channels[i] = ch / peak
443
+
444
+ # Pad/crop to target size
445
+ channels = [_pad_to_size(ch, target_size) for ch in channels]
446
+
447
+ # Stack into 2-channel image
448
+ preprocessed = np.stack(channels, axis=-1).astype(np.float32)
449
+
450
+ return preprocessed, scale_factor
451
+
452
+
453
+ def extract_cutouts(
454
+ obj,
455
+ image,
456
+ bg=None,
457
+ err=None,
458
+ mask=None,
459
+ radius=15,
460
+ fwhm=None,
461
+ target_fwhm=TARGET_FWHM,
462
+ asinh_softening=None,
463
+ verbose=False,
464
+ ):
465
+ """
466
+ Extract and preprocess cutouts for all objects.
467
+
468
+ Parameters
469
+ ----------
470
+ obj : astropy.table.Table
471
+ Object catalog with 'x' and 'y' columns
472
+ image : ndarray
473
+ Science image
474
+ bg : ndarray or float, optional
475
+ Background map or scalar value
476
+ err : ndarray or float, optional
477
+ Error/noise map or scalar value
478
+ mask : ndarray, optional
479
+ Boolean mask (True = masked)
480
+ radius : int, optional
481
+ Cutout radius in pixels. Default: 15 (31x31 cutouts)
482
+ fwhm : float, optional
483
+ Image FWHM. If None, estimated from object catalog.
484
+ target_fwhm : float, optional
485
+ Target FWHM for downscaling. Default: 3.0
486
+ asinh_softening : float, optional
487
+ Asinh softening in units of background sigma. If None, uses
488
+ DEFAULT_ASINH_SOFTENING_SIGMA.
489
+ verbose : bool, optional
490
+ Print progress. Default: False
491
+
492
+ Returns
493
+ -------
494
+ cutouts : ndarray
495
+ Array of preprocessed cutouts (N, 2*radius+1, 2*radius+1, 2)
496
+ valid_indices : ndarray
497
+ Indices of successfully extracted cutouts
498
+ """
499
+ log = print if verbose else lambda *args, **kwargs: None
500
+
501
+ # Estimate FWHM if not provided
502
+ if fwhm is None:
503
+ if 'fwhm' in obj.colnames:
504
+ fwhm = np.median(obj['fwhm'])
505
+ elif 'a' in obj.colnames:
506
+ # Approximate from semi-major axis
507
+ fwhm = np.median(obj['a']) * 2.35
508
+ else:
509
+ fwhm = 3.0 # Fallback
510
+ log(f"Estimated FWHM: {fwhm:.2f} pixels")
511
+
512
+ cutout_size = 2 * radius + 1
513
+ cutouts = []
514
+ valid_indices = []
515
+
516
+ h, w = image.shape
517
+
518
+ for i, row in enumerate(obj):
519
+ x, y = row['x'], row['y']
520
+
521
+ if not (np.isfinite(x) and np.isfinite(y)):
522
+ log(f"Object {i} has non-finite position, skipping")
523
+ continue
524
+
525
+ # Center the cutout on the nearest pixel, consistent with
526
+ # realbogus_features._extract_cutout
527
+ xc, yc = int(round(x)), int(round(y))
528
+ x_min = xc - radius
529
+ x_max = xc + radius + 1
530
+ y_min = yc - radius
531
+ y_max = yc + radius + 1
532
+
533
+ if x_min < 0 or x_max > w or y_min < 0 or y_max > h:
534
+ log(f"Object {i} too close to edge, skipping")
535
+ continue
536
+
537
+ # Extract cutout
538
+ cutout_sci = image[y_min:y_max, x_min:x_max].copy()
539
+
540
+ # Extract background cutout
541
+ if bg is not None:
542
+ if np.isscalar(bg):
543
+ cutout_bg = bg
544
+ else:
545
+ cutout_bg = bg[y_min:y_max, x_min:x_max].copy()
546
+ else:
547
+ cutout_bg = None
548
+
549
+ # Extract error cutout
550
+ if err is not None:
551
+ if np.isscalar(err):
552
+ cutout_err = err
553
+ else:
554
+ cutout_err = err[y_min:y_max, x_min:x_max].copy()
555
+ else:
556
+ cutout_err = None
557
+
558
+ # Apply mask if provided
559
+ if mask is not None:
560
+ cutout_mask = mask[y_min:y_max, x_min:x_max]
561
+ if np.sum(~cutout_mask) < 0.5 * cutout_sci.size:
562
+ log(f"Object {i} too masked, skipping")
563
+ continue
564
+ # Set masked pixels to median
565
+ if np.any(cutout_mask):
566
+ median_val = np.median(cutout_sci[~cutout_mask])
567
+ cutout_sci[cutout_mask] = median_val
568
+
569
+ # Preprocess
570
+ try:
571
+ preprocessed, scale_factor = preprocess_cutout(
572
+ cutout_sci,
573
+ cutout_bg=cutout_bg,
574
+ cutout_err=cutout_err,
575
+ fwhm=fwhm,
576
+ target_fwhm=target_fwhm,
577
+ target_size=cutout_size,
578
+ asinh_softening=asinh_softening,
579
+ )
580
+
581
+ cutouts.append(preprocessed)
582
+ valid_indices.append(i)
583
+
584
+ except Exception as e:
585
+ log(f"Failed to preprocess object {i}: {e}")
586
+ continue
587
+
588
+ if len(cutouts) == 0:
589
+ raise ValueError("No valid cutouts extracted")
590
+
591
+ cutouts = np.array(cutouts, dtype=np.float32)
592
+ valid_indices = np.array(valid_indices, dtype=int)
593
+
594
+ log(f"Extracted {len(cutouts)} valid cutouts from {len(obj)} objects")
595
+
596
+ return cutouts, valid_indices
597
+
598
+
599
+ def load_realbogus_model(model_file=None, verbose=False):
600
+ """
601
+ Load pre-trained real-bogus model.
602
+
603
+ Parameters
604
+ ----------
605
+ model_file : str, optional
606
+ Path to model file (.h5 or SavedModel directory).
607
+ If None, loads default model from ~/.stdpipe/models/
608
+ verbose : bool, optional
609
+ Print loading information. Default: False
610
+
611
+ Returns
612
+ -------
613
+ model : keras.Model
614
+ Loaded Keras model
615
+ """
616
+ _check_tensorflow()
617
+
618
+ log = print if verbose else lambda *args, **kwargs: None
619
+
620
+ # Default model path
621
+ if model_file is None:
622
+ model_file = os.path.join(DEFAULT_MODEL_DIR, DEFAULT_MODEL_NAME)
623
+
624
+ if not os.path.exists(model_file):
625
+ raise FileNotFoundError(
626
+ f"Model file not found: {model_file}\n"
627
+ "Train a model using train_realbogus_classifier() or specify model_file."
628
+ )
629
+
630
+ log(f"Loading model from {model_file}")
631
+ model = keras.models.load_model(model_file)
632
+ log(f"Model loaded successfully ({model.count_params()} parameters)")
633
+
634
+ return model
635
+
636
+
637
+ def save_realbogus_model(model, model_file=None, verbose=False):
638
+ """
639
+ Save trained real-bogus model.
640
+
641
+ Parameters
642
+ ----------
643
+ model : keras.Model
644
+ Trained model
645
+ model_file : str, optional
646
+ Output path. If None, saves to ~/.stdpipe/models/realbogus_default.h5
647
+ verbose : bool, optional
648
+ Print saving information. Default: False
649
+ """
650
+ _check_tensorflow()
651
+
652
+ log = print if verbose else lambda *args, **kwargs: None
653
+
654
+ # Default model path
655
+ if model_file is None:
656
+ os.makedirs(DEFAULT_MODEL_DIR, exist_ok=True)
657
+ model_file = os.path.join(DEFAULT_MODEL_DIR, DEFAULT_MODEL_NAME)
658
+
659
+ # Ensure directory exists
660
+ model_dir = os.path.dirname(model_file)
661
+ if model_dir:
662
+ os.makedirs(model_dir, exist_ok=True)
663
+
664
+ log(f"Saving model to {model_file}")
665
+ model.save(model_file)
666
+ log("Model saved successfully")
667
+
668
+
669
+ def classify_realbogus(
670
+ obj,
671
+ image,
672
+ model=None,
673
+ model_file=None,
674
+ bg=None,
675
+ err=None,
676
+ mask=None,
677
+ fwhm=None,
678
+ asinh_softening=None,
679
+ threshold=0.5,
680
+ add_score=True,
681
+ flag_bogus=True,
682
+ batch_size=128,
683
+ verbose=False,
684
+ ):
685
+ """
686
+ Classify detected objects as real or bogus using CNN.
687
+
688
+ This is the main entry point for real-bogus classification.
689
+
690
+ Parameters
691
+ ----------
692
+ obj : astropy.table.Table
693
+ Object catalog with 'x' and 'y' columns (from photometry.get_objects_*)
694
+ image : ndarray
695
+ Science image
696
+ model : keras.Model, optional
697
+ Pre-loaded model. If None, loads from model_file.
698
+ model_file : str, optional
699
+ Path to model file. If None, uses default model.
700
+ bg : ndarray or float, optional
701
+ Background map or scalar value
702
+ err : ndarray or float, optional
703
+ Error/noise map or scalar value
704
+ mask : ndarray, optional
705
+ Boolean mask (True = masked pixels)
706
+ cutout size : derived
707
+ Cutout size is inferred from the model input shape. If the model has
708
+ dynamic spatial dimensions, defaults to 31x31 (radius 15).
709
+ fwhm : float, optional
710
+ Image FWHM. If None, estimated from catalog.
711
+ asinh_softening : float, optional
712
+ Asinh softening in units of background sigma. If None, uses
713
+ DEFAULT_ASINH_SOFTENING_SIGMA.
714
+ threshold : float, optional
715
+ Classification threshold (0-1). Objects with score > threshold are real.
716
+ Default: 0.5
717
+ add_score : bool, optional
718
+ Add 'rb_score' column to output catalog. Default: True
719
+ flag_bogus : bool, optional
720
+ Set FLAG_BOGUS (0x4000) for bogus objects and filter them out.
721
+ Default: True
722
+ batch_size : int, optional
723
+ Batch size for inference. Default: 128
724
+ verbose : bool or callable, optional
725
+ Print progress. Can be callable for custom logging. Default: False
726
+
727
+ Returns
728
+ -------
729
+ obj_filtered : astropy.table.Table
730
+ Filtered catalog with real sources only (if flag_bogus=True)
731
+ or full catalog with 'rb_score' column (if flag_bogus=False).
732
+ The input catalog is not modified; a copy is returned.
733
+
734
+ Examples
735
+ --------
736
+ >>> from stdpipe import photometry, realbogus
737
+ >>> obj = photometry.get_objects_sep(image, thresh=3.0)
738
+ >>> obj_clean = realbogus.classify_realbogus(obj, image)
739
+ >>> print(f"Kept {len(obj_clean)}/{len(obj)} objects")
740
+ """
741
+ _check_tensorflow()
742
+
743
+ # Handle verbose as callable
744
+ log = (verbose if callable(verbose) else print) if verbose else lambda *args, **kwargs: None
745
+
746
+ # Load model if not provided
747
+ if model is None:
748
+ model = load_realbogus_model(model_file=model_file, verbose=verbose)
749
+
750
+ # Make a copy to avoid modifying the input catalog
751
+ obj = obj.copy()
752
+
753
+ cutout_radius = _infer_cutout_radius_from_model(model, default_radius=15, log=log)
754
+
755
+ log(f"Classifying {len(obj)} objects (threshold={threshold:.2f})")
756
+
757
+ # Extract cutouts
758
+ cutouts, valid_indices = extract_cutouts(
759
+ obj,
760
+ image,
761
+ bg=bg,
762
+ err=err,
763
+ mask=mask,
764
+ radius=cutout_radius,
765
+ fwhm=fwhm,
766
+ asinh_softening=asinh_softening,
767
+ verbose=verbose,
768
+ )
769
+
770
+ # Batch inference
771
+ log(f"Running inference on {len(cutouts)} cutouts (batch_size={batch_size})")
772
+ predictions = model.predict(cutouts, batch_size=batch_size, verbose=1 if verbose else 0)
773
+
774
+ # Flatten predictions
775
+ scores = predictions.flatten()
776
+
777
+ # Create full score array (NaN for invalid objects)
778
+ full_scores = np.full(len(obj), np.nan, dtype=float)
779
+ full_scores[valid_indices] = scores
780
+
781
+ # Add score column if requested
782
+ if add_score:
783
+ if 'rb_score' in obj.colnames:
784
+ obj['rb_score'] = full_scores
785
+ else:
786
+ obj.add_column(Column(full_scores, name='rb_score'))
787
+
788
+ # Flag and filter bogus objects
789
+ if flag_bogus:
790
+ # Ensure flags column exists
791
+ if 'flags' not in obj.colnames:
792
+ obj.add_column(Column(np.zeros(len(obj), dtype=int), name='flags'))
793
+
794
+ # Set bogus flag
795
+ is_bogus = full_scores < threshold
796
+ is_bogus[np.isnan(full_scores)] = True # Flag invalid objects as bogus
797
+ obj['flags'][is_bogus] |= FLAG_BOGUS
798
+
799
+ # Filter
800
+ obj_filtered = obj[~is_bogus]
801
+ log(
802
+ f"Filtered: {len(obj_filtered)}/{len(obj)} objects retained "
803
+ f"({100 * len(obj_filtered) / len(obj):.1f}%)"
804
+ )
805
+
806
+ return obj_filtered
807
+ else:
808
+ return obj
809
+
810
+
811
+ def train_realbogus_classifier(
812
+ training_data=None,
813
+ n_simulated=1000,
814
+ image_size=(2048, 2048),
815
+ fwhm_range=(1.5, 8.0),
816
+ real_source_types=['star'],
817
+ validation_split=0.15,
818
+ model=None,
819
+ model_file=None,
820
+ epochs=50,
821
+ batch_size=64,
822
+ class_weight='balanced',
823
+ callbacks=None,
824
+ verbose=True,
825
+ ):
826
+ """
827
+ Train real-bogus classifier on simulated or real data.
828
+
829
+ Parameters
830
+ ----------
831
+ training_data : tuple or dict, optional
832
+ Pre-generated training data: (X, y, fwhm_features) tuple or dict with 'X', 'y', 'fwhm' keys.
833
+ If None, generates simulated data using simulation.generate_realbogus_training_data().
834
+ n_simulated : int, optional
835
+ Number of simulated images to generate (if training_data=None). Default: 1000
836
+ image_size : tuple, optional
837
+ Size of simulated images (width, height). Default: (2048, 2048)
838
+ fwhm_range : tuple, optional
839
+ Range of FWHM values for simulated images. Default: (1.5, 8.0)
840
+ real_source_types : list, optional
841
+ List of source types to consider 'real' (if training_data=None).
842
+ Default: ['star'] treats only stars as real and galaxies as bogus.
843
+ Use ['star', 'galaxy'] to train a classifier that treats both as real.
844
+ validation_split : float, optional
845
+ Fraction of data for validation. Default: 0.15
846
+ model : keras.Model, optional
847
+ Model to train. If None, creates new model.
848
+ model_file : str, optional
849
+ Path to save trained model. Default: ~/.stdpipe/models/realbogus_default.h5
850
+ epochs : int, optional
851
+ Training epochs. Default: 50
852
+ batch_size : int, optional
853
+ Batch size. Default: 64
854
+ class_weight : str or dict, optional
855
+ Class weights for imbalanced data. 'balanced' or dict {0: w0, 1: w1}.
856
+ Default: 'balanced'
857
+ callbacks : list, optional
858
+ Keras callbacks (e.g., early stopping, checkpoints)
859
+ verbose : bool, optional
860
+ Print training progress. Default: True
861
+
862
+ Returns
863
+ -------
864
+ model : keras.Model
865
+ Trained model
866
+ history : keras.callbacks.History
867
+ Training history
868
+
869
+ Examples
870
+ --------
871
+ >>> from stdpipe import realbogus
872
+ >>> # Train on simulated data (stars and galaxies as real)
873
+ >>> model, history = realbogus.train_realbogus_classifier(
874
+ ... n_simulated=500,
875
+ ... epochs=30,
876
+ ... verbose=True
877
+ ... )
878
+ >>> # Train stars-only classifier (galaxies as bogus)
879
+ >>> model, history = realbogus.train_realbogus_classifier(
880
+ ... n_simulated=500,
881
+ ... real_source_types=['star'],
882
+ ... epochs=30,
883
+ ... verbose=True
884
+ ... )
885
+ >>> # Or use pre-generated data
886
+ >>> data = realbogus.generate_training_data(...)
887
+ >>> model, history = realbogus.train_realbogus_classifier(
888
+ ... training_data=(data['X'], data['y']),
889
+ ... epochs=30
890
+ ... )
891
+ """
892
+ _check_tensorflow()
893
+
894
+ log = print if verbose else lambda *args, **kwargs: None
895
+
896
+ # Generate or load training data
897
+ if training_data is None:
898
+ log(f"Generating training data from {n_simulated} simulated images...")
899
+ from . import simulation
900
+
901
+ data = simulation.generate_realbogus_training_data(
902
+ n_images=n_simulated,
903
+ image_size=image_size,
904
+ fwhm_range=fwhm_range,
905
+ real_source_types=real_source_types,
906
+ augment=True,
907
+ verbose=verbose,
908
+ )
909
+ X = data['X']
910
+ y = data['y']
911
+ else:
912
+ # Handle both tuple and dict formats
913
+ if isinstance(training_data, dict):
914
+ X = training_data['X']
915
+ y = training_data['y']
916
+ elif len(training_data) == 3:
917
+ X, y, _fwhm_features = training_data # Legacy 3-tuple format
918
+ else:
919
+ X, y = training_data
920
+
921
+ log(f"Training data: {len(X)} samples, {np.sum(y)} real, {len(y) - np.sum(y)} bogus")
922
+
923
+ # Create model if not provided
924
+ if model is None:
925
+ input_shape = X.shape[1:] # (height, width, channels)
926
+ model = create_realbogus_model(input_shape=input_shape)
927
+
928
+ model.summary(print_fn=log)
929
+
930
+ # Calculate class weights
931
+ if class_weight == 'balanced':
932
+ n_real = np.sum(y)
933
+ n_bogus = len(y) - n_real
934
+ if n_real == 0 or n_bogus == 0:
935
+ log("Warning: Only one class present in training data; disabling class weighting")
936
+ class_weight = None
937
+ else:
938
+ class_weight = {0: len(y) / (2 * n_bogus), 1: len(y) / (2 * n_real)}
939
+ log(f"Class weights: {class_weight}")
940
+
941
+ # Default callbacks
942
+ if callbacks is None:
943
+ callbacks = [
944
+ keras.callbacks.EarlyStopping(
945
+ monitor='val_loss', patience=10, restore_best_weights=True
946
+ ),
947
+ keras.callbacks.ReduceLROnPlateau(
948
+ monitor='val_loss', factor=0.5, patience=5, min_lr=1e-6
949
+ ),
950
+ ]
951
+
952
+ # Train model (pure image-based, no FWHM auxiliary input)
953
+ log("Starting training...")
954
+ history = model.fit(
955
+ X,
956
+ y,
957
+ validation_split=validation_split,
958
+ epochs=epochs,
959
+ batch_size=batch_size,
960
+ class_weight=class_weight,
961
+ callbacks=callbacks,
962
+ verbose=1 if verbose else 0,
963
+ )
964
+
965
+ log("Training complete")
966
+
967
+ # Save model only when explicitly requested
968
+ if model_file is not None:
969
+ save_realbogus_model(model, model_file=model_file, verbose=verbose)
970
+
971
+ return model, history