stdpipe 0.4.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- stdpipe/__init__.py +9 -0
- stdpipe/artefacts.py +370 -0
- stdpipe/astrometry.py +2001 -0
- stdpipe/astrometry_quad.py +1643 -0
- stdpipe/astrometry_wcs.py +1141 -0
- stdpipe/catalogs.py +942 -0
- stdpipe/cutouts.py +570 -0
- stdpipe/data/legacysurvey_bricks.fits.gz +0 -0
- stdpipe/data/ps1skycells.txt +199708 -0
- stdpipe/db.py +130 -0
- stdpipe/fringe_removal.py +1086 -0
- stdpipe/lcs.py +440 -0
- stdpipe/photometry.py +2340 -0
- stdpipe/photometry_background.py +998 -0
- stdpipe/photometry_iraf.py +913 -0
- stdpipe/photometry_measure.py +2268 -0
- stdpipe/photometry_model.py +931 -0
- stdpipe/photometry_psf.py +1124 -0
- stdpipe/photometry_quality.py +185 -0
- stdpipe/pipeline.py +1303 -0
- stdpipe/plots.py +1582 -0
- stdpipe/psf.py +1542 -0
- stdpipe/realbogus.py +971 -0
- stdpipe/realbogus_features.py +1657 -0
- stdpipe/reproject.py +943 -0
- stdpipe/resolve.py +267 -0
- stdpipe/sfft.py +1077 -0
- stdpipe/simulation.py +2131 -0
- stdpipe/smoothing.py +488 -0
- stdpipe/subtraction.py +1324 -0
- stdpipe/templates.py +929 -0
- stdpipe/utils.py +365 -0
- stdpipe-0.4.0.dist-info/METADATA +171 -0
- stdpipe-0.4.0.dist-info/RECORD +37 -0
- stdpipe-0.4.0.dist-info/WHEEL +5 -0
- stdpipe-0.4.0.dist-info/licenses/LICENSE.md +21 -0
- stdpipe-0.4.0.dist-info/top_level.txt +1 -0
stdpipe/realbogus.py
ADDED
|
@@ -0,0 +1,971 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Real-Bogus Classifier for Astronomical Object Detection
|
|
3
|
+
|
|
4
|
+
This module provides a CNN-based classifier to distinguish real astronomical sources
|
|
5
|
+
(stars, galaxies) from artifacts (cosmic rays, hot pixels, satellite trails) in
|
|
6
|
+
detected object catalogs.
|
|
7
|
+
|
|
8
|
+
Key Features:
|
|
9
|
+
- FWHM-invariant: Hybrid downscaling to canonical PSF size (no auxiliary FWHM input)
|
|
10
|
+
- Brightness-invariant: Peak normalization generalizes to any flux level
|
|
11
|
+
- Pure morphology: Classification based solely on source shape from 2-channel images
|
|
12
|
+
- 2-channel input: background-subtracted (linear), asinh-scaled (dynamic range compression)
|
|
13
|
+
- Lightweight 5-layer CNN (~100k parameters)
|
|
14
|
+
- Batch processing for efficient inference
|
|
15
|
+
- Optional TensorFlow dependency
|
|
16
|
+
|
|
17
|
+
Usage:
|
|
18
|
+
from stdpipe import photometry, realbogus
|
|
19
|
+
|
|
20
|
+
# Detect objects
|
|
21
|
+
obj = photometry.get_objects_sep(image, thresh=3.0)
|
|
22
|
+
|
|
23
|
+
# Classify and filter
|
|
24
|
+
obj_clean = realbogus.classify_realbogus(obj, image, threshold=0.5)
|
|
25
|
+
|
|
26
|
+
# Or add scores without filtering
|
|
27
|
+
obj = realbogus.classify_realbogus(obj, image, add_score=True, flag_bogus=False)
|
|
28
|
+
print(obj['rb_score'])
|
|
29
|
+
|
|
30
|
+
Author: STDPipe Contributors
|
|
31
|
+
"""
|
|
32
|
+
|
|
33
|
+
from __future__ import absolute_import, division, print_function
|
|
34
|
+
|
|
35
|
+
import numpy as np
|
|
36
|
+
import os
|
|
37
|
+
import warnings
|
|
38
|
+
from astropy.table import Column
|
|
39
|
+
|
|
40
|
+
from .realbogus_features import FLAG_BOGUS
|
|
41
|
+
|
|
42
|
+
# Conditional TensorFlow import
|
|
43
|
+
try:
|
|
44
|
+
import tensorflow as tf
|
|
45
|
+
from tensorflow import keras
|
|
46
|
+
from tensorflow.keras import layers, models
|
|
47
|
+
|
|
48
|
+
HAS_TENSORFLOW = True
|
|
49
|
+
except ImportError:
|
|
50
|
+
HAS_TENSORFLOW = False
|
|
51
|
+
warnings.warn(
|
|
52
|
+
"TensorFlow not found. Real-bogus classifier will not be available. "
|
|
53
|
+
"Install with: pip install stdpipe[ml]",
|
|
54
|
+
ImportWarning,
|
|
55
|
+
)
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
# Model architecture parameters
|
|
59
|
+
DEFAULT_MODEL_DIR = os.path.expanduser('~/.stdpipe/models')
|
|
60
|
+
DEFAULT_MODEL_NAME = 'realbogus_default.h5'
|
|
61
|
+
TARGET_FWHM = 3.0 # Canonical FWHM for downscaling normalization
|
|
62
|
+
DEFAULT_ASINH_SOFTENING_SIGMA = 3.0 # Asinh softening in units of background sigma
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
def _check_tensorflow():
|
|
66
|
+
"""Check if TensorFlow is available, raise error if not."""
|
|
67
|
+
if not HAS_TENSORFLOW:
|
|
68
|
+
raise ImportError(
|
|
69
|
+
"TensorFlow is required for real-bogus classification. "
|
|
70
|
+
"Install with: pip install stdpipe[ml] or pip install tensorflow>=2.10"
|
|
71
|
+
)
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
def create_realbogus_model(
|
|
75
|
+
input_shape=(31, 31, 2), filters=(32, 64, 128), dense_units=64, dropout_rate=0.5
|
|
76
|
+
):
|
|
77
|
+
"""
|
|
78
|
+
Create CNN architecture for real-bogus classification.
|
|
79
|
+
|
|
80
|
+
Architecture:
|
|
81
|
+
- 3-5 convolutional layers with batch normalization
|
|
82
|
+
- Global average pooling (handles variable input sizes)
|
|
83
|
+
- Dense layer with dropout
|
|
84
|
+
- Sigmoid output (binary classification)
|
|
85
|
+
|
|
86
|
+
Design Philosophy:
|
|
87
|
+
- FWHM-invariant: Images downscaled to canonical FWHM, no auxiliary FWHM input needed
|
|
88
|
+
- Brightness-invariant: Peak normalization allows generalization to any flux level
|
|
89
|
+
- Pure morphology: Classification based solely on source shape
|
|
90
|
+
|
|
91
|
+
Input Channels:
|
|
92
|
+
- Channel 0: Background-subtracted (linear scale), peak-normalized
|
|
93
|
+
- Channel 1: Asinh-scaled background-subtracted, peak-normalized
|
|
94
|
+
|
|
95
|
+
Parameters
|
|
96
|
+
----------
|
|
97
|
+
input_shape : tuple, optional
|
|
98
|
+
Input shape (height, width, channels). Default: (31, 31, 2)
|
|
99
|
+
Height/width can be None for variable-size inputs.
|
|
100
|
+
filters : tuple, optional
|
|
101
|
+
Number of filters in each conv layer. Default: (32, 64, 128)
|
|
102
|
+
dense_units : int, optional
|
|
103
|
+
Units in dense layer. Default: 64
|
|
104
|
+
dropout_rate : float, optional
|
|
105
|
+
Dropout rate for regularization. Default: 0.5
|
|
106
|
+
|
|
107
|
+
Returns
|
|
108
|
+
-------
|
|
109
|
+
model : keras.Model
|
|
110
|
+
Compiled Keras model ready for training
|
|
111
|
+
"""
|
|
112
|
+
_check_tensorflow()
|
|
113
|
+
|
|
114
|
+
# Main image input (2 channels: background-subtracted linear, asinh-scaled)
|
|
115
|
+
image_input = keras.Input(shape=input_shape, name='image_input')
|
|
116
|
+
|
|
117
|
+
# Convolutional feature extraction
|
|
118
|
+
x = image_input
|
|
119
|
+
for i, n_filters in enumerate(filters):
|
|
120
|
+
x = layers.Conv2D(
|
|
121
|
+
n_filters, (3, 3), activation='relu', padding='same', name=f'conv{i + 1}'
|
|
122
|
+
)(x)
|
|
123
|
+
x = layers.BatchNormalization(name=f'bn{i + 1}')(x)
|
|
124
|
+
x = layers.MaxPooling2D((2, 2), name=f'pool{i + 1}')(x)
|
|
125
|
+
|
|
126
|
+
# Global pooling to handle variable input sizes
|
|
127
|
+
x = layers.GlobalAveragePooling2D(name='global_pool')(x)
|
|
128
|
+
|
|
129
|
+
# Dense layers (no auxiliary inputs - pure image-based classification)
|
|
130
|
+
x = layers.Dense(dense_units, activation='relu', name='dense1')(x)
|
|
131
|
+
x = layers.Dropout(dropout_rate, name='dropout')(x)
|
|
132
|
+
|
|
133
|
+
# Output layer (sigmoid for binary classification)
|
|
134
|
+
output = layers.Dense(1, activation='sigmoid', name='output')(x)
|
|
135
|
+
|
|
136
|
+
# Create and compile model
|
|
137
|
+
model = keras.Model(inputs=image_input, outputs=output, name='realbogus_classifier')
|
|
138
|
+
|
|
139
|
+
model.compile(
|
|
140
|
+
optimizer=keras.optimizers.Adam(learning_rate=0.001),
|
|
141
|
+
loss='binary_crossentropy',
|
|
142
|
+
metrics=['accuracy', keras.metrics.AUC(name='auc')],
|
|
143
|
+
)
|
|
144
|
+
|
|
145
|
+
return model
|
|
146
|
+
|
|
147
|
+
|
|
148
|
+
def _downscale_cutout(cutout, scale_factor, mode='mean'):
|
|
149
|
+
"""
|
|
150
|
+
Downscale cutout by integer factor using block averaging.
|
|
151
|
+
|
|
152
|
+
Parameters
|
|
153
|
+
----------
|
|
154
|
+
cutout : ndarray
|
|
155
|
+
2D image cutout
|
|
156
|
+
scale_factor : int or float
|
|
157
|
+
Downscaling factor (converted to integer, must be >= 1).
|
|
158
|
+
Typically passed as round(fwhm/target_fwhm) for robust hot pixel suppression.
|
|
159
|
+
mode : str, optional
|
|
160
|
+
Downscaling mode: 'mean' or 'median'. Default: 'mean'
|
|
161
|
+
|
|
162
|
+
Returns
|
|
163
|
+
-------
|
|
164
|
+
downscaled : ndarray
|
|
165
|
+
Downscaled cutout
|
|
166
|
+
"""
|
|
167
|
+
if scale_factor <= 1:
|
|
168
|
+
return cutout
|
|
169
|
+
|
|
170
|
+
scale_factor = int(scale_factor)
|
|
171
|
+
h, w = cutout.shape
|
|
172
|
+
|
|
173
|
+
# Trim to multiple of scale_factor
|
|
174
|
+
new_h = (h // scale_factor) * scale_factor
|
|
175
|
+
new_w = (w // scale_factor) * scale_factor
|
|
176
|
+
cutout_trimmed = cutout[:new_h, :new_w]
|
|
177
|
+
|
|
178
|
+
# Reshape and aggregate
|
|
179
|
+
reshaped = cutout_trimmed.reshape(
|
|
180
|
+
new_h // scale_factor, scale_factor, new_w // scale_factor, scale_factor
|
|
181
|
+
)
|
|
182
|
+
|
|
183
|
+
if mode == 'median':
|
|
184
|
+
downscaled = np.median(reshaped, axis=(1, 3))
|
|
185
|
+
else: # mean
|
|
186
|
+
downscaled = np.mean(reshaped, axis=(1, 3))
|
|
187
|
+
|
|
188
|
+
return downscaled
|
|
189
|
+
|
|
190
|
+
|
|
191
|
+
def _infer_cutout_radius_from_model(model, default_radius=15, log=None):
|
|
192
|
+
"""Infer cutout radius from model input shape."""
|
|
193
|
+
if log is None:
|
|
194
|
+
log = lambda *args, **kwargs: None
|
|
195
|
+
|
|
196
|
+
shape = model.input_shape
|
|
197
|
+
if isinstance(shape, list):
|
|
198
|
+
shape = shape[0] if shape else None
|
|
199
|
+
|
|
200
|
+
if shape is None or len(shape) < 4:
|
|
201
|
+
log(
|
|
202
|
+
f"Model input shape unavailable; using default cutout size {2 * default_radius + 1} "
|
|
203
|
+
f"(radius {default_radius})"
|
|
204
|
+
)
|
|
205
|
+
return default_radius
|
|
206
|
+
|
|
207
|
+
height, width = shape[1], shape[2]
|
|
208
|
+
channels = shape[3]
|
|
209
|
+
|
|
210
|
+
if height is None or width is None:
|
|
211
|
+
log(
|
|
212
|
+
f"Model input shape is dynamic; using default cutout size {2 * default_radius + 1} "
|
|
213
|
+
f"(radius {default_radius})"
|
|
214
|
+
)
|
|
215
|
+
return default_radius
|
|
216
|
+
|
|
217
|
+
if height != width:
|
|
218
|
+
raise ValueError(
|
|
219
|
+
f"Model input shape must be square for cutout extraction (got {height}x{width})."
|
|
220
|
+
)
|
|
221
|
+
if height % 2 == 0:
|
|
222
|
+
raise ValueError(f"Model input size must be odd for symmetric cutouts (got {height}).")
|
|
223
|
+
|
|
224
|
+
radius = int((height - 1) // 2)
|
|
225
|
+
log(f"Using cutout size {height}x{width} (radius {radius}) from model input shape")
|
|
226
|
+
|
|
227
|
+
if channels is not None and channels != 2:
|
|
228
|
+
log(f"Warning: model expects {channels} channels; realbogus generates 2-channel cutouts")
|
|
229
|
+
|
|
230
|
+
return radius
|
|
231
|
+
|
|
232
|
+
|
|
233
|
+
def _upscale_cutout(cutout, scale_factor):
|
|
234
|
+
"""
|
|
235
|
+
Upscale cutout by integer factor using pixel replication.
|
|
236
|
+
|
|
237
|
+
Parameters
|
|
238
|
+
----------
|
|
239
|
+
cutout : ndarray
|
|
240
|
+
2D image cutout
|
|
241
|
+
scale_factor : int or float
|
|
242
|
+
Upscaling factor (converted to integer, must be >= 1).
|
|
243
|
+
Each pixel is replicated into scale_factor x scale_factor block.
|
|
244
|
+
|
|
245
|
+
Returns
|
|
246
|
+
-------
|
|
247
|
+
upscaled : ndarray
|
|
248
|
+
Upscaled cutout
|
|
249
|
+
"""
|
|
250
|
+
if scale_factor <= 1:
|
|
251
|
+
return cutout
|
|
252
|
+
|
|
253
|
+
scale_factor = int(scale_factor)
|
|
254
|
+
|
|
255
|
+
# Use numpy repeat for integer upscaling
|
|
256
|
+
# Repeat along each axis
|
|
257
|
+
upscaled = np.repeat(cutout, scale_factor, axis=0)
|
|
258
|
+
upscaled = np.repeat(upscaled, scale_factor, axis=1)
|
|
259
|
+
|
|
260
|
+
return upscaled
|
|
261
|
+
|
|
262
|
+
|
|
263
|
+
def _pad_to_size(cutout, target_size, mode='edge'):
|
|
264
|
+
"""
|
|
265
|
+
Pad cutout to target size (centered).
|
|
266
|
+
|
|
267
|
+
Parameters
|
|
268
|
+
----------
|
|
269
|
+
cutout : ndarray
|
|
270
|
+
2D image cutout
|
|
271
|
+
target_size : int
|
|
272
|
+
Target size (square)
|
|
273
|
+
mode : str, optional
|
|
274
|
+
Padding mode for np.pad. Default: 'edge'
|
|
275
|
+
|
|
276
|
+
Returns
|
|
277
|
+
-------
|
|
278
|
+
padded : ndarray
|
|
279
|
+
Padded cutout (target_size x target_size)
|
|
280
|
+
"""
|
|
281
|
+
h, w = cutout.shape
|
|
282
|
+
|
|
283
|
+
# Crop dimensions that exceed target
|
|
284
|
+
if h > target_size:
|
|
285
|
+
start_h = (h - target_size) // 2
|
|
286
|
+
cutout = cutout[start_h : start_h + target_size, :]
|
|
287
|
+
h = target_size
|
|
288
|
+
if w > target_size:
|
|
289
|
+
start_w = (w - target_size) // 2
|
|
290
|
+
cutout = cutout[:, start_w : start_w + target_size]
|
|
291
|
+
w = target_size
|
|
292
|
+
|
|
293
|
+
# Pad dimensions that are smaller than target
|
|
294
|
+
pad_h = target_size - h
|
|
295
|
+
pad_w = target_size - w
|
|
296
|
+
if pad_h > 0 or pad_w > 0:
|
|
297
|
+
cutout = np.pad(
|
|
298
|
+
cutout, ((pad_h // 2, pad_h - pad_h // 2), (pad_w // 2, pad_w - pad_w // 2)), mode=mode
|
|
299
|
+
)
|
|
300
|
+
|
|
301
|
+
return cutout
|
|
302
|
+
|
|
303
|
+
|
|
304
|
+
def preprocess_cutout(
|
|
305
|
+
cutout_sci,
|
|
306
|
+
cutout_bg=None,
|
|
307
|
+
cutout_err=None,
|
|
308
|
+
fwhm=None,
|
|
309
|
+
target_fwhm=TARGET_FWHM,
|
|
310
|
+
target_size=31,
|
|
311
|
+
downscale_threshold=1.5,
|
|
312
|
+
normalize=True,
|
|
313
|
+
asinh_softening=None,
|
|
314
|
+
):
|
|
315
|
+
"""
|
|
316
|
+
Preprocess cutout for CNN input.
|
|
317
|
+
|
|
318
|
+
Steps:
|
|
319
|
+
1. Optional scaling (downscale or upscale) to canonical FWHM
|
|
320
|
+
2. Create 2-channel input (background-subtracted linear, asinh-scaled)
|
|
321
|
+
3. Peak normalization (each channel normalized by its own peak value)
|
|
322
|
+
4. Pad/crop to target size
|
|
323
|
+
|
|
324
|
+
FWHM Scaling Strategy (Symmetric):
|
|
325
|
+
- Downscaling (FWHM > target_fwhm × threshold): Integer block averaging
|
|
326
|
+
- No scaling (target_fwhm / threshold ≤ FWHM ≤ target_fwhm × threshold): Keep as-is
|
|
327
|
+
- Upscaling (FWHM < target_fwhm / threshold): Integer pixel replication
|
|
328
|
+
|
|
329
|
+
Default: target_fwhm=3.0, threshold=1.5
|
|
330
|
+
→ Downscale if FWHM > 4.5, upscale if FWHM < 2.0, else unchanged
|
|
331
|
+
|
|
332
|
+
This ensures all PSFs normalized to approximately the same size regardless of
|
|
333
|
+
sharpness, eliminating FWHM as a confounding variable.
|
|
334
|
+
|
|
335
|
+
Channel Design:
|
|
336
|
+
- Channel 0: Background-subtracted (linear scale), peak-normalized
|
|
337
|
+
- Channel 1: Asinh-scaled background-subtracted, peak-normalized
|
|
338
|
+
|
|
339
|
+
Peak normalization makes the representation brightness-invariant: all sources
|
|
340
|
+
(faint to extremely bright) are scaled to [-1, 1] range based on their peak
|
|
341
|
+
value. This allows the CNN to learn pure morphological features that generalize
|
|
342
|
+
to ANY brightness level, including sources far brighter than the training set.
|
|
343
|
+
|
|
344
|
+
The asinh channel complements the linear channel by providing compressed
|
|
345
|
+
dynamic range information useful for distinguishing extended vs. compact sources.
|
|
346
|
+
|
|
347
|
+
Parameters
|
|
348
|
+
----------
|
|
349
|
+
cutout_sci : ndarray
|
|
350
|
+
Science image cutout (assumed to be background-subtracted if cutout_bg is None)
|
|
351
|
+
cutout_bg : ndarray, optional
|
|
352
|
+
Background cutout (or scalar value). If None, estimated from cutout edges.
|
|
353
|
+
cutout_err : ndarray or float, optional
|
|
354
|
+
Error/noise cutout (or scalar value). Used to estimate the noise level
|
|
355
|
+
(sigma) for asinh softening. Only the median value is used.
|
|
356
|
+
fwhm : float, optional
|
|
357
|
+
Image FWHM in pixels. If provided, cutout will be downscaled to target_fwhm.
|
|
358
|
+
target_fwhm : float, optional
|
|
359
|
+
Target FWHM for downscaling normalization. Default: 3.0
|
|
360
|
+
target_size : int, optional
|
|
361
|
+
Target cutout size (square). Default: 31
|
|
362
|
+
downscale_threshold : float, optional
|
|
363
|
+
Only downscale if fwhm/target_fwhm > threshold. Default: 1.5
|
|
364
|
+
normalize : bool, optional
|
|
365
|
+
Apply peak normalization to each channel (scales to [-1, 1] range).
|
|
366
|
+
Default: True. This makes the representation brightness-invariant.
|
|
367
|
+
asinh_softening : float, optional
|
|
368
|
+
Asinh softening in units of background sigma. If None, uses
|
|
369
|
+
DEFAULT_ASINH_SOFTENING_SIGMA. Actual softening is
|
|
370
|
+
(asinh_softening * sigma), where sigma is estimated from cutout_err.
|
|
371
|
+
|
|
372
|
+
Returns
|
|
373
|
+
-------
|
|
374
|
+
preprocessed : ndarray
|
|
375
|
+
Preprocessed cutout (target_size, target_size, 2)
|
|
376
|
+
scale_factor : float
|
|
377
|
+
Applied scale factor (for diagnostics)
|
|
378
|
+
"""
|
|
379
|
+
# Handle background
|
|
380
|
+
if cutout_bg is None:
|
|
381
|
+
# Estimate from edges
|
|
382
|
+
edge_pixels = np.concatenate(
|
|
383
|
+
[cutout_sci[0, :], cutout_sci[-1, :], cutout_sci[:, 0], cutout_sci[:, -1]]
|
|
384
|
+
)
|
|
385
|
+
cutout_bg = np.median(edge_pixels)
|
|
386
|
+
|
|
387
|
+
if np.isscalar(cutout_bg):
|
|
388
|
+
cutout_bg = np.full_like(cutout_sci, cutout_bg)
|
|
389
|
+
|
|
390
|
+
# Estimate noise sigma for asinh softening (only the median is needed,
|
|
391
|
+
# so compute before any scaling to avoid unnecessary array operations)
|
|
392
|
+
if cutout_err is None:
|
|
393
|
+
from astropy.stats import mad_std
|
|
394
|
+
|
|
395
|
+
sigma = float(mad_std(cutout_sci))
|
|
396
|
+
elif np.isscalar(cutout_err):
|
|
397
|
+
sigma = float(cutout_err)
|
|
398
|
+
else:
|
|
399
|
+
sigma = float(np.nanmedian(cutout_err))
|
|
400
|
+
if not np.isfinite(sigma) or sigma <= 0:
|
|
401
|
+
sigma = 1.0
|
|
402
|
+
|
|
403
|
+
# Scaling to canonical FWHM (both downscale and upscale)
|
|
404
|
+
scale_factor = 1.0
|
|
405
|
+
if fwhm is not None and fwhm > 0:
|
|
406
|
+
scale_factor = fwhm / target_fwhm
|
|
407
|
+
|
|
408
|
+
# Downscale if PSF too large
|
|
409
|
+
if scale_factor > downscale_threshold:
|
|
410
|
+
factor = round(scale_factor)
|
|
411
|
+
cutout_sci = _downscale_cutout(cutout_sci, factor)
|
|
412
|
+
cutout_bg = _downscale_cutout(cutout_bg, factor)
|
|
413
|
+
|
|
414
|
+
# Upscale if PSF too sharp (symmetric with downscaling)
|
|
415
|
+
elif scale_factor < 1.0 / downscale_threshold:
|
|
416
|
+
factor = round(1.0 / scale_factor)
|
|
417
|
+
cutout_sci = _upscale_cutout(cutout_sci, factor)
|
|
418
|
+
cutout_bg = _upscale_cutout(cutout_bg, factor)
|
|
419
|
+
|
|
420
|
+
# Create background-subtracted channel
|
|
421
|
+
cutout_bgsub = cutout_sci - cutout_bg
|
|
422
|
+
|
|
423
|
+
# Create asinh-scaled channel for dynamic range compression
|
|
424
|
+
if asinh_softening is None:
|
|
425
|
+
asinh_softening = DEFAULT_ASINH_SOFTENING_SIGMA
|
|
426
|
+
|
|
427
|
+
softening = float(asinh_softening) * sigma
|
|
428
|
+
if not np.isfinite(softening) or softening <= 0:
|
|
429
|
+
softening = sigma
|
|
430
|
+
|
|
431
|
+
# Asinh scaling: compresses high values, ~linear for low values
|
|
432
|
+
cutout_asinh = np.arcsinh(cutout_bgsub / softening)
|
|
433
|
+
|
|
434
|
+
# Stack channels: [background-subtracted linear, asinh-scaled]
|
|
435
|
+
channels = [cutout_bgsub, cutout_asinh]
|
|
436
|
+
|
|
437
|
+
# Peak normalization for brightness invariance
|
|
438
|
+
if normalize:
|
|
439
|
+
for i, ch in enumerate(channels):
|
|
440
|
+
peak = np.max(np.abs(ch))
|
|
441
|
+
if peak > 1e-10:
|
|
442
|
+
channels[i] = ch / peak
|
|
443
|
+
|
|
444
|
+
# Pad/crop to target size
|
|
445
|
+
channels = [_pad_to_size(ch, target_size) for ch in channels]
|
|
446
|
+
|
|
447
|
+
# Stack into 2-channel image
|
|
448
|
+
preprocessed = np.stack(channels, axis=-1).astype(np.float32)
|
|
449
|
+
|
|
450
|
+
return preprocessed, scale_factor
|
|
451
|
+
|
|
452
|
+
|
|
453
|
+
def extract_cutouts(
|
|
454
|
+
obj,
|
|
455
|
+
image,
|
|
456
|
+
bg=None,
|
|
457
|
+
err=None,
|
|
458
|
+
mask=None,
|
|
459
|
+
radius=15,
|
|
460
|
+
fwhm=None,
|
|
461
|
+
target_fwhm=TARGET_FWHM,
|
|
462
|
+
asinh_softening=None,
|
|
463
|
+
verbose=False,
|
|
464
|
+
):
|
|
465
|
+
"""
|
|
466
|
+
Extract and preprocess cutouts for all objects.
|
|
467
|
+
|
|
468
|
+
Parameters
|
|
469
|
+
----------
|
|
470
|
+
obj : astropy.table.Table
|
|
471
|
+
Object catalog with 'x' and 'y' columns
|
|
472
|
+
image : ndarray
|
|
473
|
+
Science image
|
|
474
|
+
bg : ndarray or float, optional
|
|
475
|
+
Background map or scalar value
|
|
476
|
+
err : ndarray or float, optional
|
|
477
|
+
Error/noise map or scalar value
|
|
478
|
+
mask : ndarray, optional
|
|
479
|
+
Boolean mask (True = masked)
|
|
480
|
+
radius : int, optional
|
|
481
|
+
Cutout radius in pixels. Default: 15 (31x31 cutouts)
|
|
482
|
+
fwhm : float, optional
|
|
483
|
+
Image FWHM. If None, estimated from object catalog.
|
|
484
|
+
target_fwhm : float, optional
|
|
485
|
+
Target FWHM for downscaling. Default: 3.0
|
|
486
|
+
asinh_softening : float, optional
|
|
487
|
+
Asinh softening in units of background sigma. If None, uses
|
|
488
|
+
DEFAULT_ASINH_SOFTENING_SIGMA.
|
|
489
|
+
verbose : bool, optional
|
|
490
|
+
Print progress. Default: False
|
|
491
|
+
|
|
492
|
+
Returns
|
|
493
|
+
-------
|
|
494
|
+
cutouts : ndarray
|
|
495
|
+
Array of preprocessed cutouts (N, 2*radius+1, 2*radius+1, 2)
|
|
496
|
+
valid_indices : ndarray
|
|
497
|
+
Indices of successfully extracted cutouts
|
|
498
|
+
"""
|
|
499
|
+
log = print if verbose else lambda *args, **kwargs: None
|
|
500
|
+
|
|
501
|
+
# Estimate FWHM if not provided
|
|
502
|
+
if fwhm is None:
|
|
503
|
+
if 'fwhm' in obj.colnames:
|
|
504
|
+
fwhm = np.median(obj['fwhm'])
|
|
505
|
+
elif 'a' in obj.colnames:
|
|
506
|
+
# Approximate from semi-major axis
|
|
507
|
+
fwhm = np.median(obj['a']) * 2.35
|
|
508
|
+
else:
|
|
509
|
+
fwhm = 3.0 # Fallback
|
|
510
|
+
log(f"Estimated FWHM: {fwhm:.2f} pixels")
|
|
511
|
+
|
|
512
|
+
cutout_size = 2 * radius + 1
|
|
513
|
+
cutouts = []
|
|
514
|
+
valid_indices = []
|
|
515
|
+
|
|
516
|
+
h, w = image.shape
|
|
517
|
+
|
|
518
|
+
for i, row in enumerate(obj):
|
|
519
|
+
x, y = row['x'], row['y']
|
|
520
|
+
|
|
521
|
+
if not (np.isfinite(x) and np.isfinite(y)):
|
|
522
|
+
log(f"Object {i} has non-finite position, skipping")
|
|
523
|
+
continue
|
|
524
|
+
|
|
525
|
+
# Center the cutout on the nearest pixel, consistent with
|
|
526
|
+
# realbogus_features._extract_cutout
|
|
527
|
+
xc, yc = int(round(x)), int(round(y))
|
|
528
|
+
x_min = xc - radius
|
|
529
|
+
x_max = xc + radius + 1
|
|
530
|
+
y_min = yc - radius
|
|
531
|
+
y_max = yc + radius + 1
|
|
532
|
+
|
|
533
|
+
if x_min < 0 or x_max > w or y_min < 0 or y_max > h:
|
|
534
|
+
log(f"Object {i} too close to edge, skipping")
|
|
535
|
+
continue
|
|
536
|
+
|
|
537
|
+
# Extract cutout
|
|
538
|
+
cutout_sci = image[y_min:y_max, x_min:x_max].copy()
|
|
539
|
+
|
|
540
|
+
# Extract background cutout
|
|
541
|
+
if bg is not None:
|
|
542
|
+
if np.isscalar(bg):
|
|
543
|
+
cutout_bg = bg
|
|
544
|
+
else:
|
|
545
|
+
cutout_bg = bg[y_min:y_max, x_min:x_max].copy()
|
|
546
|
+
else:
|
|
547
|
+
cutout_bg = None
|
|
548
|
+
|
|
549
|
+
# Extract error cutout
|
|
550
|
+
if err is not None:
|
|
551
|
+
if np.isscalar(err):
|
|
552
|
+
cutout_err = err
|
|
553
|
+
else:
|
|
554
|
+
cutout_err = err[y_min:y_max, x_min:x_max].copy()
|
|
555
|
+
else:
|
|
556
|
+
cutout_err = None
|
|
557
|
+
|
|
558
|
+
# Apply mask if provided
|
|
559
|
+
if mask is not None:
|
|
560
|
+
cutout_mask = mask[y_min:y_max, x_min:x_max]
|
|
561
|
+
if np.sum(~cutout_mask) < 0.5 * cutout_sci.size:
|
|
562
|
+
log(f"Object {i} too masked, skipping")
|
|
563
|
+
continue
|
|
564
|
+
# Set masked pixels to median
|
|
565
|
+
if np.any(cutout_mask):
|
|
566
|
+
median_val = np.median(cutout_sci[~cutout_mask])
|
|
567
|
+
cutout_sci[cutout_mask] = median_val
|
|
568
|
+
|
|
569
|
+
# Preprocess
|
|
570
|
+
try:
|
|
571
|
+
preprocessed, scale_factor = preprocess_cutout(
|
|
572
|
+
cutout_sci,
|
|
573
|
+
cutout_bg=cutout_bg,
|
|
574
|
+
cutout_err=cutout_err,
|
|
575
|
+
fwhm=fwhm,
|
|
576
|
+
target_fwhm=target_fwhm,
|
|
577
|
+
target_size=cutout_size,
|
|
578
|
+
asinh_softening=asinh_softening,
|
|
579
|
+
)
|
|
580
|
+
|
|
581
|
+
cutouts.append(preprocessed)
|
|
582
|
+
valid_indices.append(i)
|
|
583
|
+
|
|
584
|
+
except Exception as e:
|
|
585
|
+
log(f"Failed to preprocess object {i}: {e}")
|
|
586
|
+
continue
|
|
587
|
+
|
|
588
|
+
if len(cutouts) == 0:
|
|
589
|
+
raise ValueError("No valid cutouts extracted")
|
|
590
|
+
|
|
591
|
+
cutouts = np.array(cutouts, dtype=np.float32)
|
|
592
|
+
valid_indices = np.array(valid_indices, dtype=int)
|
|
593
|
+
|
|
594
|
+
log(f"Extracted {len(cutouts)} valid cutouts from {len(obj)} objects")
|
|
595
|
+
|
|
596
|
+
return cutouts, valid_indices
|
|
597
|
+
|
|
598
|
+
|
|
599
|
+
def load_realbogus_model(model_file=None, verbose=False):
|
|
600
|
+
"""
|
|
601
|
+
Load pre-trained real-bogus model.
|
|
602
|
+
|
|
603
|
+
Parameters
|
|
604
|
+
----------
|
|
605
|
+
model_file : str, optional
|
|
606
|
+
Path to model file (.h5 or SavedModel directory).
|
|
607
|
+
If None, loads default model from ~/.stdpipe/models/
|
|
608
|
+
verbose : bool, optional
|
|
609
|
+
Print loading information. Default: False
|
|
610
|
+
|
|
611
|
+
Returns
|
|
612
|
+
-------
|
|
613
|
+
model : keras.Model
|
|
614
|
+
Loaded Keras model
|
|
615
|
+
"""
|
|
616
|
+
_check_tensorflow()
|
|
617
|
+
|
|
618
|
+
log = print if verbose else lambda *args, **kwargs: None
|
|
619
|
+
|
|
620
|
+
# Default model path
|
|
621
|
+
if model_file is None:
|
|
622
|
+
model_file = os.path.join(DEFAULT_MODEL_DIR, DEFAULT_MODEL_NAME)
|
|
623
|
+
|
|
624
|
+
if not os.path.exists(model_file):
|
|
625
|
+
raise FileNotFoundError(
|
|
626
|
+
f"Model file not found: {model_file}\n"
|
|
627
|
+
"Train a model using train_realbogus_classifier() or specify model_file."
|
|
628
|
+
)
|
|
629
|
+
|
|
630
|
+
log(f"Loading model from {model_file}")
|
|
631
|
+
model = keras.models.load_model(model_file)
|
|
632
|
+
log(f"Model loaded successfully ({model.count_params()} parameters)")
|
|
633
|
+
|
|
634
|
+
return model
|
|
635
|
+
|
|
636
|
+
|
|
637
|
+
def save_realbogus_model(model, model_file=None, verbose=False):
|
|
638
|
+
"""
|
|
639
|
+
Save trained real-bogus model.
|
|
640
|
+
|
|
641
|
+
Parameters
|
|
642
|
+
----------
|
|
643
|
+
model : keras.Model
|
|
644
|
+
Trained model
|
|
645
|
+
model_file : str, optional
|
|
646
|
+
Output path. If None, saves to ~/.stdpipe/models/realbogus_default.h5
|
|
647
|
+
verbose : bool, optional
|
|
648
|
+
Print saving information. Default: False
|
|
649
|
+
"""
|
|
650
|
+
_check_tensorflow()
|
|
651
|
+
|
|
652
|
+
log = print if verbose else lambda *args, **kwargs: None
|
|
653
|
+
|
|
654
|
+
# Default model path
|
|
655
|
+
if model_file is None:
|
|
656
|
+
os.makedirs(DEFAULT_MODEL_DIR, exist_ok=True)
|
|
657
|
+
model_file = os.path.join(DEFAULT_MODEL_DIR, DEFAULT_MODEL_NAME)
|
|
658
|
+
|
|
659
|
+
# Ensure directory exists
|
|
660
|
+
model_dir = os.path.dirname(model_file)
|
|
661
|
+
if model_dir:
|
|
662
|
+
os.makedirs(model_dir, exist_ok=True)
|
|
663
|
+
|
|
664
|
+
log(f"Saving model to {model_file}")
|
|
665
|
+
model.save(model_file)
|
|
666
|
+
log("Model saved successfully")
|
|
667
|
+
|
|
668
|
+
|
|
669
|
+
def classify_realbogus(
|
|
670
|
+
obj,
|
|
671
|
+
image,
|
|
672
|
+
model=None,
|
|
673
|
+
model_file=None,
|
|
674
|
+
bg=None,
|
|
675
|
+
err=None,
|
|
676
|
+
mask=None,
|
|
677
|
+
fwhm=None,
|
|
678
|
+
asinh_softening=None,
|
|
679
|
+
threshold=0.5,
|
|
680
|
+
add_score=True,
|
|
681
|
+
flag_bogus=True,
|
|
682
|
+
batch_size=128,
|
|
683
|
+
verbose=False,
|
|
684
|
+
):
|
|
685
|
+
"""
|
|
686
|
+
Classify detected objects as real or bogus using CNN.
|
|
687
|
+
|
|
688
|
+
This is the main entry point for real-bogus classification.
|
|
689
|
+
|
|
690
|
+
Parameters
|
|
691
|
+
----------
|
|
692
|
+
obj : astropy.table.Table
|
|
693
|
+
Object catalog with 'x' and 'y' columns (from photometry.get_objects_*)
|
|
694
|
+
image : ndarray
|
|
695
|
+
Science image
|
|
696
|
+
model : keras.Model, optional
|
|
697
|
+
Pre-loaded model. If None, loads from model_file.
|
|
698
|
+
model_file : str, optional
|
|
699
|
+
Path to model file. If None, uses default model.
|
|
700
|
+
bg : ndarray or float, optional
|
|
701
|
+
Background map or scalar value
|
|
702
|
+
err : ndarray or float, optional
|
|
703
|
+
Error/noise map or scalar value
|
|
704
|
+
mask : ndarray, optional
|
|
705
|
+
Boolean mask (True = masked pixels)
|
|
706
|
+
cutout size : derived
|
|
707
|
+
Cutout size is inferred from the model input shape. If the model has
|
|
708
|
+
dynamic spatial dimensions, defaults to 31x31 (radius 15).
|
|
709
|
+
fwhm : float, optional
|
|
710
|
+
Image FWHM. If None, estimated from catalog.
|
|
711
|
+
asinh_softening : float, optional
|
|
712
|
+
Asinh softening in units of background sigma. If None, uses
|
|
713
|
+
DEFAULT_ASINH_SOFTENING_SIGMA.
|
|
714
|
+
threshold : float, optional
|
|
715
|
+
Classification threshold (0-1). Objects with score > threshold are real.
|
|
716
|
+
Default: 0.5
|
|
717
|
+
add_score : bool, optional
|
|
718
|
+
Add 'rb_score' column to output catalog. Default: True
|
|
719
|
+
flag_bogus : bool, optional
|
|
720
|
+
Set FLAG_BOGUS (0x4000) for bogus objects and filter them out.
|
|
721
|
+
Default: True
|
|
722
|
+
batch_size : int, optional
|
|
723
|
+
Batch size for inference. Default: 128
|
|
724
|
+
verbose : bool or callable, optional
|
|
725
|
+
Print progress. Can be callable for custom logging. Default: False
|
|
726
|
+
|
|
727
|
+
Returns
|
|
728
|
+
-------
|
|
729
|
+
obj_filtered : astropy.table.Table
|
|
730
|
+
Filtered catalog with real sources only (if flag_bogus=True)
|
|
731
|
+
or full catalog with 'rb_score' column (if flag_bogus=False).
|
|
732
|
+
The input catalog is not modified; a copy is returned.
|
|
733
|
+
|
|
734
|
+
Examples
|
|
735
|
+
--------
|
|
736
|
+
>>> from stdpipe import photometry, realbogus
|
|
737
|
+
>>> obj = photometry.get_objects_sep(image, thresh=3.0)
|
|
738
|
+
>>> obj_clean = realbogus.classify_realbogus(obj, image)
|
|
739
|
+
>>> print(f"Kept {len(obj_clean)}/{len(obj)} objects")
|
|
740
|
+
"""
|
|
741
|
+
_check_tensorflow()
|
|
742
|
+
|
|
743
|
+
# Handle verbose as callable
|
|
744
|
+
log = (verbose if callable(verbose) else print) if verbose else lambda *args, **kwargs: None
|
|
745
|
+
|
|
746
|
+
# Load model if not provided
|
|
747
|
+
if model is None:
|
|
748
|
+
model = load_realbogus_model(model_file=model_file, verbose=verbose)
|
|
749
|
+
|
|
750
|
+
# Make a copy to avoid modifying the input catalog
|
|
751
|
+
obj = obj.copy()
|
|
752
|
+
|
|
753
|
+
cutout_radius = _infer_cutout_radius_from_model(model, default_radius=15, log=log)
|
|
754
|
+
|
|
755
|
+
log(f"Classifying {len(obj)} objects (threshold={threshold:.2f})")
|
|
756
|
+
|
|
757
|
+
# Extract cutouts
|
|
758
|
+
cutouts, valid_indices = extract_cutouts(
|
|
759
|
+
obj,
|
|
760
|
+
image,
|
|
761
|
+
bg=bg,
|
|
762
|
+
err=err,
|
|
763
|
+
mask=mask,
|
|
764
|
+
radius=cutout_radius,
|
|
765
|
+
fwhm=fwhm,
|
|
766
|
+
asinh_softening=asinh_softening,
|
|
767
|
+
verbose=verbose,
|
|
768
|
+
)
|
|
769
|
+
|
|
770
|
+
# Batch inference
|
|
771
|
+
log(f"Running inference on {len(cutouts)} cutouts (batch_size={batch_size})")
|
|
772
|
+
predictions = model.predict(cutouts, batch_size=batch_size, verbose=1 if verbose else 0)
|
|
773
|
+
|
|
774
|
+
# Flatten predictions
|
|
775
|
+
scores = predictions.flatten()
|
|
776
|
+
|
|
777
|
+
# Create full score array (NaN for invalid objects)
|
|
778
|
+
full_scores = np.full(len(obj), np.nan, dtype=float)
|
|
779
|
+
full_scores[valid_indices] = scores
|
|
780
|
+
|
|
781
|
+
# Add score column if requested
|
|
782
|
+
if add_score:
|
|
783
|
+
if 'rb_score' in obj.colnames:
|
|
784
|
+
obj['rb_score'] = full_scores
|
|
785
|
+
else:
|
|
786
|
+
obj.add_column(Column(full_scores, name='rb_score'))
|
|
787
|
+
|
|
788
|
+
# Flag and filter bogus objects
|
|
789
|
+
if flag_bogus:
|
|
790
|
+
# Ensure flags column exists
|
|
791
|
+
if 'flags' not in obj.colnames:
|
|
792
|
+
obj.add_column(Column(np.zeros(len(obj), dtype=int), name='flags'))
|
|
793
|
+
|
|
794
|
+
# Set bogus flag
|
|
795
|
+
is_bogus = full_scores < threshold
|
|
796
|
+
is_bogus[np.isnan(full_scores)] = True # Flag invalid objects as bogus
|
|
797
|
+
obj['flags'][is_bogus] |= FLAG_BOGUS
|
|
798
|
+
|
|
799
|
+
# Filter
|
|
800
|
+
obj_filtered = obj[~is_bogus]
|
|
801
|
+
log(
|
|
802
|
+
f"Filtered: {len(obj_filtered)}/{len(obj)} objects retained "
|
|
803
|
+
f"({100 * len(obj_filtered) / len(obj):.1f}%)"
|
|
804
|
+
)
|
|
805
|
+
|
|
806
|
+
return obj_filtered
|
|
807
|
+
else:
|
|
808
|
+
return obj
|
|
809
|
+
|
|
810
|
+
|
|
811
|
+
def train_realbogus_classifier(
|
|
812
|
+
training_data=None,
|
|
813
|
+
n_simulated=1000,
|
|
814
|
+
image_size=(2048, 2048),
|
|
815
|
+
fwhm_range=(1.5, 8.0),
|
|
816
|
+
real_source_types=['star'],
|
|
817
|
+
validation_split=0.15,
|
|
818
|
+
model=None,
|
|
819
|
+
model_file=None,
|
|
820
|
+
epochs=50,
|
|
821
|
+
batch_size=64,
|
|
822
|
+
class_weight='balanced',
|
|
823
|
+
callbacks=None,
|
|
824
|
+
verbose=True,
|
|
825
|
+
):
|
|
826
|
+
"""
|
|
827
|
+
Train real-bogus classifier on simulated or real data.
|
|
828
|
+
|
|
829
|
+
Parameters
|
|
830
|
+
----------
|
|
831
|
+
training_data : tuple or dict, optional
|
|
832
|
+
Pre-generated training data: (X, y, fwhm_features) tuple or dict with 'X', 'y', 'fwhm' keys.
|
|
833
|
+
If None, generates simulated data using simulation.generate_realbogus_training_data().
|
|
834
|
+
n_simulated : int, optional
|
|
835
|
+
Number of simulated images to generate (if training_data=None). Default: 1000
|
|
836
|
+
image_size : tuple, optional
|
|
837
|
+
Size of simulated images (width, height). Default: (2048, 2048)
|
|
838
|
+
fwhm_range : tuple, optional
|
|
839
|
+
Range of FWHM values for simulated images. Default: (1.5, 8.0)
|
|
840
|
+
real_source_types : list, optional
|
|
841
|
+
List of source types to consider 'real' (if training_data=None).
|
|
842
|
+
Default: ['star'] treats only stars as real and galaxies as bogus.
|
|
843
|
+
Use ['star', 'galaxy'] to train a classifier that treats both as real.
|
|
844
|
+
validation_split : float, optional
|
|
845
|
+
Fraction of data for validation. Default: 0.15
|
|
846
|
+
model : keras.Model, optional
|
|
847
|
+
Model to train. If None, creates new model.
|
|
848
|
+
model_file : str, optional
|
|
849
|
+
Path to save trained model. Default: ~/.stdpipe/models/realbogus_default.h5
|
|
850
|
+
epochs : int, optional
|
|
851
|
+
Training epochs. Default: 50
|
|
852
|
+
batch_size : int, optional
|
|
853
|
+
Batch size. Default: 64
|
|
854
|
+
class_weight : str or dict, optional
|
|
855
|
+
Class weights for imbalanced data. 'balanced' or dict {0: w0, 1: w1}.
|
|
856
|
+
Default: 'balanced'
|
|
857
|
+
callbacks : list, optional
|
|
858
|
+
Keras callbacks (e.g., early stopping, checkpoints)
|
|
859
|
+
verbose : bool, optional
|
|
860
|
+
Print training progress. Default: True
|
|
861
|
+
|
|
862
|
+
Returns
|
|
863
|
+
-------
|
|
864
|
+
model : keras.Model
|
|
865
|
+
Trained model
|
|
866
|
+
history : keras.callbacks.History
|
|
867
|
+
Training history
|
|
868
|
+
|
|
869
|
+
Examples
|
|
870
|
+
--------
|
|
871
|
+
>>> from stdpipe import realbogus
|
|
872
|
+
>>> # Train on simulated data (stars and galaxies as real)
|
|
873
|
+
>>> model, history = realbogus.train_realbogus_classifier(
|
|
874
|
+
... n_simulated=500,
|
|
875
|
+
... epochs=30,
|
|
876
|
+
... verbose=True
|
|
877
|
+
... )
|
|
878
|
+
>>> # Train stars-only classifier (galaxies as bogus)
|
|
879
|
+
>>> model, history = realbogus.train_realbogus_classifier(
|
|
880
|
+
... n_simulated=500,
|
|
881
|
+
... real_source_types=['star'],
|
|
882
|
+
... epochs=30,
|
|
883
|
+
... verbose=True
|
|
884
|
+
... )
|
|
885
|
+
>>> # Or use pre-generated data
|
|
886
|
+
>>> data = realbogus.generate_training_data(...)
|
|
887
|
+
>>> model, history = realbogus.train_realbogus_classifier(
|
|
888
|
+
... training_data=(data['X'], data['y']),
|
|
889
|
+
... epochs=30
|
|
890
|
+
... )
|
|
891
|
+
"""
|
|
892
|
+
_check_tensorflow()
|
|
893
|
+
|
|
894
|
+
log = print if verbose else lambda *args, **kwargs: None
|
|
895
|
+
|
|
896
|
+
# Generate or load training data
|
|
897
|
+
if training_data is None:
|
|
898
|
+
log(f"Generating training data from {n_simulated} simulated images...")
|
|
899
|
+
from . import simulation
|
|
900
|
+
|
|
901
|
+
data = simulation.generate_realbogus_training_data(
|
|
902
|
+
n_images=n_simulated,
|
|
903
|
+
image_size=image_size,
|
|
904
|
+
fwhm_range=fwhm_range,
|
|
905
|
+
real_source_types=real_source_types,
|
|
906
|
+
augment=True,
|
|
907
|
+
verbose=verbose,
|
|
908
|
+
)
|
|
909
|
+
X = data['X']
|
|
910
|
+
y = data['y']
|
|
911
|
+
else:
|
|
912
|
+
# Handle both tuple and dict formats
|
|
913
|
+
if isinstance(training_data, dict):
|
|
914
|
+
X = training_data['X']
|
|
915
|
+
y = training_data['y']
|
|
916
|
+
elif len(training_data) == 3:
|
|
917
|
+
X, y, _fwhm_features = training_data # Legacy 3-tuple format
|
|
918
|
+
else:
|
|
919
|
+
X, y = training_data
|
|
920
|
+
|
|
921
|
+
log(f"Training data: {len(X)} samples, {np.sum(y)} real, {len(y) - np.sum(y)} bogus")
|
|
922
|
+
|
|
923
|
+
# Create model if not provided
|
|
924
|
+
if model is None:
|
|
925
|
+
input_shape = X.shape[1:] # (height, width, channels)
|
|
926
|
+
model = create_realbogus_model(input_shape=input_shape)
|
|
927
|
+
|
|
928
|
+
model.summary(print_fn=log)
|
|
929
|
+
|
|
930
|
+
# Calculate class weights
|
|
931
|
+
if class_weight == 'balanced':
|
|
932
|
+
n_real = np.sum(y)
|
|
933
|
+
n_bogus = len(y) - n_real
|
|
934
|
+
if n_real == 0 or n_bogus == 0:
|
|
935
|
+
log("Warning: Only one class present in training data; disabling class weighting")
|
|
936
|
+
class_weight = None
|
|
937
|
+
else:
|
|
938
|
+
class_weight = {0: len(y) / (2 * n_bogus), 1: len(y) / (2 * n_real)}
|
|
939
|
+
log(f"Class weights: {class_weight}")
|
|
940
|
+
|
|
941
|
+
# Default callbacks
|
|
942
|
+
if callbacks is None:
|
|
943
|
+
callbacks = [
|
|
944
|
+
keras.callbacks.EarlyStopping(
|
|
945
|
+
monitor='val_loss', patience=10, restore_best_weights=True
|
|
946
|
+
),
|
|
947
|
+
keras.callbacks.ReduceLROnPlateau(
|
|
948
|
+
monitor='val_loss', factor=0.5, patience=5, min_lr=1e-6
|
|
949
|
+
),
|
|
950
|
+
]
|
|
951
|
+
|
|
952
|
+
# Train model (pure image-based, no FWHM auxiliary input)
|
|
953
|
+
log("Starting training...")
|
|
954
|
+
history = model.fit(
|
|
955
|
+
X,
|
|
956
|
+
y,
|
|
957
|
+
validation_split=validation_split,
|
|
958
|
+
epochs=epochs,
|
|
959
|
+
batch_size=batch_size,
|
|
960
|
+
class_weight=class_weight,
|
|
961
|
+
callbacks=callbacks,
|
|
962
|
+
verbose=1 if verbose else 0,
|
|
963
|
+
)
|
|
964
|
+
|
|
965
|
+
log("Training complete")
|
|
966
|
+
|
|
967
|
+
# Save model only when explicitly requested
|
|
968
|
+
if model_file is not None:
|
|
969
|
+
save_realbogus_model(model, model_file=model_file, verbose=verbose)
|
|
970
|
+
|
|
971
|
+
return model, history
|