RACS-tools 4.3.1__tar.gz → 4.3.2__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {racs_tools-4.3.1 → racs_tools-4.3.2}/PKG-INFO +1 -1
- {racs_tools-4.3.1 → racs_tools-4.3.2}/pyproject.toml +1 -1
- {racs_tools-4.3.1 → racs_tools-4.3.2}/racs_tools/beamcon_2D.py +73 -74
- {racs_tools-4.3.1 → racs_tools-4.3.2}/racs_tools/beamcon_3D.py +106 -106
- {racs_tools-4.3.1 → racs_tools-4.3.2}/racs_tools/logging.py +47 -2
- {racs_tools-4.3.1 → racs_tools-4.3.2}/LICENSE +0 -0
- {racs_tools-4.3.1 → racs_tools-4.3.2}/README.md +0 -0
- {racs_tools-4.3.1 → racs_tools-4.3.2}/racs_tools/__init__.py +0 -0
- {racs_tools-4.3.1 → racs_tools-4.3.2}/racs_tools/au2.py +0 -0
- {racs_tools-4.3.1 → racs_tools-4.3.2}/racs_tools/convolve_uv.py +0 -0
- {racs_tools-4.3.1 → racs_tools-4.3.2}/racs_tools/gaussft.py +0 -0
- {racs_tools-4.3.1 → racs_tools-4.3.2}/racs_tools/getnoise_list.py +0 -0
- {racs_tools-4.3.1 → racs_tools-4.3.2}/racs_tools/parallel.py +0 -0
|
@@ -33,9 +33,9 @@ from racs_tools.convolve_uv import (
|
|
|
33
33
|
)
|
|
34
34
|
from racs_tools.logging import (
|
|
35
35
|
init_worker,
|
|
36
|
-
log_listener,
|
|
37
36
|
log_queue,
|
|
38
37
|
logger,
|
|
38
|
+
running_log_listener,
|
|
39
39
|
set_verbosity,
|
|
40
40
|
)
|
|
41
41
|
from racs_tools.parallel import get_executor
|
|
@@ -482,84 +482,83 @@ def smooth_fits_files(
|
|
|
482
482
|
Beam: Common beam used.
|
|
483
483
|
"""
|
|
484
484
|
# Required for multiprocessing logging
|
|
485
|
-
|
|
486
|
-
|
|
487
|
-
|
|
488
|
-
|
|
489
|
-
|
|
490
|
-
|
|
491
|
-
|
|
492
|
-
|
|
493
|
-
|
|
494
|
-
|
|
495
|
-
|
|
496
|
-
|
|
497
|
-
|
|
498
|
-
|
|
499
|
-
|
|
500
|
-
|
|
501
|
-
|
|
502
|
-
|
|
503
|
-
|
|
504
|
-
|
|
505
|
-
|
|
506
|
-
|
|
507
|
-
|
|
508
|
-
|
|
509
|
-
|
|
510
|
-
|
|
511
|
-
|
|
512
|
-
|
|
513
|
-
|
|
514
|
-
|
|
515
|
-
|
|
516
|
-
|
|
517
|
-
|
|
518
|
-
|
|
519
|
-
|
|
520
|
-
|
|
521
|
-
|
|
485
|
+
with running_log_listener():
|
|
486
|
+
if dryrun:
|
|
487
|
+
logger.info("Doing a dry run -- no files will be saved")
|
|
488
|
+
|
|
489
|
+
# Check early as can fail
|
|
490
|
+
Executor = get_executor(executor_type)
|
|
491
|
+
|
|
492
|
+
# Get file list
|
|
493
|
+
if listfile:
|
|
494
|
+
assert len(infile_list) == 1, "Only one list file can be provided!"
|
|
495
|
+
with open(infile_list[0]) as f:
|
|
496
|
+
infile_list = [Path(line) for line in f.read().splitlines()]
|
|
497
|
+
files = sorted(infile_list)
|
|
498
|
+
if len(files) == 0:
|
|
499
|
+
raise FileNotFoundError("No files found!")
|
|
500
|
+
|
|
501
|
+
conv_mode = parse_conv_mode(conv_mode)
|
|
502
|
+
|
|
503
|
+
nonetest = [param is None for param in (bmaj, bmin, bpa)]
|
|
504
|
+
if all(nonetest):
|
|
505
|
+
target_beam = None
|
|
506
|
+
elif any(nonetest):
|
|
507
|
+
raise ValueError("Please specify all target beam params!")
|
|
508
|
+
else:
|
|
509
|
+
target_beam = Beam(bmaj * u.arcsec, bmin * u.arcsec, bpa * u.deg)
|
|
510
|
+
logger.info(f"Target beam is {target_beam!r}")
|
|
511
|
+
|
|
512
|
+
# Find smallest common beam
|
|
513
|
+
common_beam, all_beams = get_common_beam(
|
|
514
|
+
files,
|
|
515
|
+
conv_mode=conv_mode,
|
|
516
|
+
target_beam=target_beam,
|
|
517
|
+
cutoff=cutoff,
|
|
518
|
+
tolerance=tolerance,
|
|
519
|
+
nsamps=nsamps,
|
|
520
|
+
epsilon=epsilon,
|
|
521
|
+
)
|
|
522
522
|
|
|
523
|
-
|
|
524
|
-
|
|
525
|
-
|
|
523
|
+
if target_beam is not None:
|
|
524
|
+
if not check_target_beam(target_beam, all_beams, files, cutoff):
|
|
525
|
+
raise BeamError("Please choose a larger target beam!")
|
|
526
526
|
|
|
527
|
-
|
|
527
|
+
common_beam = target_beam
|
|
528
528
|
|
|
529
|
-
|
|
530
|
-
|
|
531
|
-
|
|
532
|
-
|
|
533
|
-
|
|
534
|
-
|
|
535
|
-
)
|
|
536
|
-
|
|
537
|
-
logger.info(f"Final beam is {common_beam!r}")
|
|
538
|
-
with Executor(
|
|
539
|
-
max_workers=ncores, initializer=init_worker, initargs=(log_queue, verbosity)
|
|
540
|
-
) as executor:
|
|
541
|
-
futures = []
|
|
542
|
-
for file in files:
|
|
543
|
-
future = executor.submit(
|
|
544
|
-
beamcon_2d_on_fits,
|
|
545
|
-
file=file,
|
|
546
|
-
outdir=outdir,
|
|
547
|
-
new_beam=common_beam,
|
|
548
|
-
conv_mode=conv_mode,
|
|
549
|
-
suffix=suffix,
|
|
550
|
-
prefix=prefix,
|
|
551
|
-
cutoff=cutoff,
|
|
552
|
-
dryrun=dryrun,
|
|
529
|
+
if circularise:
|
|
530
|
+
logger.info("Circular beam requested, setting BMIN=BMAJ and BPA=0")
|
|
531
|
+
common_beam = Beam(
|
|
532
|
+
major=common_beam.major,
|
|
533
|
+
minor=common_beam.major,
|
|
534
|
+
pa=0 * u.deg,
|
|
553
535
|
)
|
|
554
|
-
futures.append(future)
|
|
555
|
-
|
|
556
|
-
beam_log_list = [future.result() for future in futures]
|
|
557
|
-
if log is not None:
|
|
558
|
-
writelog(beam_log_list, log)
|
|
559
536
|
|
|
560
|
-
|
|
561
|
-
|
|
562
|
-
|
|
537
|
+
logger.info(f"Final beam is {common_beam!r}")
|
|
538
|
+
with Executor(
|
|
539
|
+
max_workers=ncores, initializer=init_worker, initargs=(log_queue, verbosity)
|
|
540
|
+
) as executor:
|
|
541
|
+
futures = []
|
|
542
|
+
for file in files:
|
|
543
|
+
future = executor.submit(
|
|
544
|
+
beamcon_2d_on_fits,
|
|
545
|
+
file=file,
|
|
546
|
+
outdir=outdir,
|
|
547
|
+
new_beam=common_beam,
|
|
548
|
+
conv_mode=conv_mode,
|
|
549
|
+
suffix=suffix,
|
|
550
|
+
prefix=prefix,
|
|
551
|
+
cutoff=cutoff,
|
|
552
|
+
dryrun=dryrun,
|
|
553
|
+
)
|
|
554
|
+
futures.append(future)
|
|
555
|
+
|
|
556
|
+
beam_log_list = [future.result() for future in futures]
|
|
557
|
+
if log is not None:
|
|
558
|
+
writelog(beam_log_list, log)
|
|
559
|
+
|
|
560
|
+
logger.info("Done!")
|
|
561
|
+
return common_beam
|
|
563
562
|
|
|
564
563
|
|
|
565
564
|
def cli():
|
|
@@ -30,9 +30,9 @@ from racs_tools.beamcon_2D import my_ceil, round_up
|
|
|
30
30
|
from racs_tools.convolve_uv import NAN_BEAM, ZERO_BEAM, parse_conv_mode, smooth
|
|
31
31
|
from racs_tools.logging import (
|
|
32
32
|
init_worker,
|
|
33
|
-
log_listener,
|
|
34
33
|
log_queue,
|
|
35
34
|
logger,
|
|
35
|
+
running_log_listener,
|
|
36
36
|
set_verbosity,
|
|
37
37
|
)
|
|
38
38
|
from racs_tools.parallel import get_executor
|
|
@@ -1076,131 +1076,131 @@ def smooth_fits_cube(
|
|
|
1076
1076
|
tuple[list[CubeData], list[CommonBeamData], list[Path]]: Cube data and common beam data, and output filenames.
|
|
1077
1077
|
"""
|
|
1078
1078
|
# Required for multiprocessing logging
|
|
1079
|
-
|
|
1080
|
-
|
|
1081
|
-
|
|
1082
|
-
|
|
1083
|
-
|
|
1084
|
-
|
|
1085
|
-
|
|
1086
|
-
|
|
1087
|
-
|
|
1088
|
-
|
|
1089
|
-
|
|
1090
|
-
|
|
1091
|
-
|
|
1092
|
-
|
|
1093
|
-
|
|
1079
|
+
with running_log_listener():
|
|
1080
|
+
if dryrun:
|
|
1081
|
+
logger.info("Doing a dry run -- no files will be saved")
|
|
1082
|
+
|
|
1083
|
+
# Check early as can fail
|
|
1084
|
+
Executor = get_executor(executor_type)
|
|
1085
|
+
|
|
1086
|
+
# Check mode
|
|
1087
|
+
logger.info(f"Mode is {mode}")
|
|
1088
|
+
if mode == "natural":
|
|
1089
|
+
logger.info("Smoothing each channel to a common resolution")
|
|
1090
|
+
elif mode == "total":
|
|
1091
|
+
logger.info("Smoothing all channels to a common resolution")
|
|
1092
|
+
else:
|
|
1093
|
+
raise ValueError(f"Mode must be 'natural' or 'total', not '{mode}'")
|
|
1094
1094
|
|
|
1095
|
-
|
|
1096
|
-
|
|
1097
|
-
|
|
1098
|
-
|
|
1095
|
+
# Check cutoff
|
|
1096
|
+
if cutoff is not None:
|
|
1097
|
+
cutoff *= u.arcsec
|
|
1098
|
+
logger.info(f"Cutoff is: {cutoff}")
|
|
1099
1099
|
|
|
1100
|
-
|
|
1101
|
-
|
|
1100
|
+
# Check target
|
|
1101
|
+
conv_mode = parse_conv_mode(conv_mode)
|
|
1102
1102
|
|
|
1103
|
-
|
|
1103
|
+
target_beam = _get_target_beam(bmaj=bmaj, bmin=bmin, bpa=bpa)
|
|
1104
1104
|
|
|
1105
|
-
|
|
1106
|
-
|
|
1107
|
-
|
|
1105
|
+
files = sorted(infiles_list)
|
|
1106
|
+
if len(files) == 0:
|
|
1107
|
+
raise FileNotFoundError("No files found!")
|
|
1108
1108
|
|
|
1109
|
-
|
|
1110
|
-
|
|
1111
|
-
|
|
1109
|
+
outdir_list: list[Path] = (
|
|
1110
|
+
[outdir] * len(files) if outdir is not None else [f.parent for f in files]
|
|
1111
|
+
)
|
|
1112
1112
|
|
|
1113
|
-
|
|
1113
|
+
cube_data_list = make_data(files, outdir_list)
|
|
1114
1114
|
|
|
1115
|
-
|
|
1116
|
-
|
|
1117
|
-
|
|
1118
|
-
|
|
1115
|
+
# Sanity check channel counts
|
|
1116
|
+
nchans = np.array([cube_data.nchan for cube_data in cube_data_list])
|
|
1117
|
+
if not all(nchans == nchans[0]):
|
|
1118
|
+
raise ValueError(f"Unequal number of spectral channels! Got {nchans}")
|
|
1119
1119
|
|
|
1120
|
-
|
|
1121
|
-
|
|
1122
|
-
|
|
1123
|
-
|
|
1120
|
+
if isinstance(target_beam, Beams) and any(nchans != len(target_beam)):
|
|
1121
|
+
raise ValueError(
|
|
1122
|
+
f"Unequal length of target beams ({len(target_beam)}) to channels ({nchans})"
|
|
1123
|
+
)
|
|
1124
1124
|
|
|
1125
|
-
|
|
1125
|
+
nchans = nchans[0]
|
|
1126
1126
|
|
|
1127
|
-
|
|
1128
|
-
|
|
1129
|
-
|
|
1127
|
+
# Check suffix
|
|
1128
|
+
if suffix is None:
|
|
1129
|
+
suffix = mode
|
|
1130
1130
|
|
|
1131
|
-
|
|
1132
|
-
|
|
1131
|
+
# Apply some masking
|
|
1132
|
+
cube_data_list = masking(cube_data_list, cutoff=cutoff)
|
|
1133
1133
|
|
|
1134
|
-
|
|
1135
|
-
|
|
1136
|
-
|
|
1137
|
-
|
|
1138
|
-
|
|
1139
|
-
|
|
1140
|
-
mode=mode,
|
|
1141
|
-
suffix=suffix,
|
|
1142
|
-
circularise=circularise,
|
|
1143
|
-
tolerance=tolerance,
|
|
1144
|
-
nsamps=nsamps,
|
|
1145
|
-
epsilon=epsilon,
|
|
1146
|
-
)
|
|
1147
|
-
else:
|
|
1148
|
-
logger.info("Reading from convolve beamlog files")
|
|
1149
|
-
common_beam_data_list: list[CommonBeamData] = []
|
|
1150
|
-
for cube_data in cube_data_list:
|
|
1151
|
-
commonbeam_log = cube_data.beamlog.with_suffix(f".{suffix}.txt")
|
|
1152
|
-
common_beam_data = readlogs(commonbeam_log)
|
|
1153
|
-
common_beam_data_list.append(common_beam_data)
|
|
1154
|
-
|
|
1155
|
-
if dryrun:
|
|
1156
|
-
logger.info("Doing a dryrun so all done!")
|
|
1157
|
-
return cube_data_list, common_beam_data_list
|
|
1158
|
-
|
|
1159
|
-
# Init the files in parallel
|
|
1160
|
-
logger.info("Initialising output files")
|
|
1161
|
-
# Init output files and retrieve file names
|
|
1162
|
-
with Executor(
|
|
1163
|
-
max_workers=ncores, initializer=init_worker, initargs=(log_queue, verbosity)
|
|
1164
|
-
) as executor:
|
|
1165
|
-
futures = []
|
|
1166
|
-
for cube_data, common_beam_data in zip(cube_data_list, common_beam_data_list):
|
|
1167
|
-
future = executor.submit(
|
|
1168
|
-
initfiles,
|
|
1169
|
-
filename=cube_data.filename,
|
|
1170
|
-
commonbeams=common_beam_data.commonbeams,
|
|
1171
|
-
outdir=cube_data.outdir,
|
|
1134
|
+
if not uselogs:
|
|
1135
|
+
common_beam_data_list = commonbeamer(
|
|
1136
|
+
cube_data_list=cube_data_list,
|
|
1137
|
+
nchans=nchans,
|
|
1138
|
+
conv_mode=conv_mode,
|
|
1139
|
+
target_beam=target_beam,
|
|
1172
1140
|
mode=mode,
|
|
1173
1141
|
suffix=suffix,
|
|
1174
|
-
|
|
1175
|
-
|
|
1142
|
+
circularise=circularise,
|
|
1143
|
+
tolerance=tolerance,
|
|
1144
|
+
nsamps=nsamps,
|
|
1145
|
+
epsilon=epsilon,
|
|
1176
1146
|
)
|
|
1177
|
-
|
|
1178
|
-
|
|
1179
|
-
|
|
1180
|
-
|
|
1181
|
-
|
|
1182
|
-
|
|
1183
|
-
|
|
1184
|
-
|
|
1185
|
-
|
|
1186
|
-
|
|
1187
|
-
|
|
1147
|
+
else:
|
|
1148
|
+
logger.info("Reading from convolve beamlog files")
|
|
1149
|
+
common_beam_data_list: list[CommonBeamData] = []
|
|
1150
|
+
for cube_data in cube_data_list:
|
|
1151
|
+
commonbeam_log = cube_data.beamlog.with_suffix(f".{suffix}.txt")
|
|
1152
|
+
common_beam_data = readlogs(commonbeam_log)
|
|
1153
|
+
common_beam_data_list.append(common_beam_data)
|
|
1154
|
+
|
|
1155
|
+
if dryrun:
|
|
1156
|
+
logger.info("Doing a dryrun so all done!")
|
|
1157
|
+
return cube_data_list, common_beam_data_list
|
|
1158
|
+
|
|
1159
|
+
# Init the files in parallel
|
|
1160
|
+
logger.info("Initialising output files")
|
|
1161
|
+
# Init output files and retrieve file names
|
|
1162
|
+
with Executor(
|
|
1163
|
+
max_workers=ncores, initializer=init_worker, initargs=(log_queue, verbosity)
|
|
1164
|
+
) as executor:
|
|
1165
|
+
futures = []
|
|
1166
|
+
for cube_data, common_beam_data in zip(
|
|
1167
|
+
cube_data_list, common_beam_data_list
|
|
1168
|
+
):
|
|
1188
1169
|
future = executor.submit(
|
|
1189
|
-
|
|
1190
|
-
|
|
1191
|
-
|
|
1192
|
-
|
|
1193
|
-
|
|
1194
|
-
|
|
1170
|
+
initfiles,
|
|
1171
|
+
filename=cube_data.filename,
|
|
1172
|
+
commonbeams=common_beam_data.commonbeams,
|
|
1173
|
+
outdir=cube_data.outdir,
|
|
1174
|
+
mode=mode,
|
|
1175
|
+
suffix=suffix,
|
|
1176
|
+
prefix=prefix,
|
|
1177
|
+
ref_chan=ref_chan,
|
|
1195
1178
|
)
|
|
1196
1179
|
futures.append(future)
|
|
1197
|
-
|
|
1198
|
-
|
|
1199
|
-
|
|
1180
|
+
outfiles: list[Path] = [future.result() for future in futures]
|
|
1181
|
+
|
|
1182
|
+
with Executor(
|
|
1183
|
+
max_workers=ncores, initializer=init_worker, initargs=(log_queue, verbosity)
|
|
1184
|
+
) as executor:
|
|
1185
|
+
futures = []
|
|
1186
|
+
for cube_data, common_beam_data, outfile in zip(
|
|
1187
|
+
cube_data_list, common_beam_data_list, outfiles
|
|
1188
|
+
):
|
|
1189
|
+
for chan in range(nchans):
|
|
1190
|
+
future = executor.submit(
|
|
1191
|
+
smooth_and_write_plane,
|
|
1192
|
+
chan=chan,
|
|
1193
|
+
cube_data=cube_data,
|
|
1194
|
+
common_beam_data=common_beam_data,
|
|
1195
|
+
outfile=outfile,
|
|
1196
|
+
conv_mode=conv_mode,
|
|
1197
|
+
)
|
|
1198
|
+
futures.append(future)
|
|
1199
|
+
_ = [future.result() for future in futures]
|
|
1200
1200
|
|
|
1201
|
-
|
|
1201
|
+
logger.info("Done!")
|
|
1202
1202
|
|
|
1203
|
-
|
|
1203
|
+
return cube_data_list, common_beam_data_list, outfiles
|
|
1204
1204
|
|
|
1205
1205
|
|
|
1206
1206
|
def cli():
|
|
@@ -2,6 +2,9 @@
|
|
|
2
2
|
|
|
3
3
|
import logging
|
|
4
4
|
import multiprocessing as mp
|
|
5
|
+
import threading
|
|
6
|
+
from collections.abc import Iterator
|
|
7
|
+
from contextlib import contextmanager
|
|
5
8
|
from logging.handlers import QueueHandler, QueueListener
|
|
6
9
|
|
|
7
10
|
logging.captureWarnings(True)
|
|
@@ -71,8 +74,50 @@ def init_worker(log_queue: mp.Queue, verbosity: int = 0) -> None:
|
|
|
71
74
|
|
|
72
75
|
set_verbosity(logger, verbosity)
|
|
73
76
|
|
|
74
|
-
handler
|
|
75
|
-
logger
|
|
77
|
+
# Keep exactly one queue handler per process. With a thread executor every
|
|
78
|
+
# worker shares this logger, so adding unconditionally fans each record out
|
|
79
|
+
# once per worker and leaves the handlers attached for later calls - where
|
|
80
|
+
# they keep filling the queue after the listener has stopped consuming it.
|
|
81
|
+
for stale in [h for h in logger.handlers if isinstance(h, QueueHandler)]:
|
|
82
|
+
logger.removeHandler(stale)
|
|
83
|
+
|
|
84
|
+
logger.addHandler(QueueHandler(log_queue))
|
|
76
85
|
|
|
77
86
|
|
|
78
87
|
logger, log_listener, log_queue = setup_logger()
|
|
88
|
+
|
|
89
|
+
_listener_lock = threading.Lock()
|
|
90
|
+
_listener_users = 0
|
|
91
|
+
"""Number of ``running_log_listener`` blocks currently active."""
|
|
92
|
+
|
|
93
|
+
|
|
94
|
+
@contextmanager
|
|
95
|
+
def running_log_listener() -> Iterator[None]:
|
|
96
|
+
"""Run the module-level log listener for the duration of the block.
|
|
97
|
+
|
|
98
|
+
On Python 3.13, ``QueueListener.start()`` raises ``RuntimeError`` if the
|
|
99
|
+
listener's monitor thread is still set from a previous run. Calling
|
|
100
|
+
``enqueue_sentinel()`` alone (without ``stop()``) leaves the thread
|
|
101
|
+
reference set even after the monitor has exited, so the listener must
|
|
102
|
+
always be stopped -- on every exit path -- before it can be started
|
|
103
|
+
again.
|
|
104
|
+
|
|
105
|
+
The listener is a module-level singleton, so nested or concurrent blocks
|
|
106
|
+
share the one run: it is started by the first block to enter and stopped
|
|
107
|
+
by the last to leave. Without that, an inner ``stop()`` would enqueue a
|
|
108
|
+
sentinel that the outer block's monitor thread consumes, and the
|
|
109
|
+
``join()`` inside ``stop()`` would then never return.
|
|
110
|
+
"""
|
|
111
|
+
global _listener_users
|
|
112
|
+
|
|
113
|
+
with _listener_lock:
|
|
114
|
+
if _listener_users == 0:
|
|
115
|
+
log_listener.start()
|
|
116
|
+
_listener_users += 1
|
|
117
|
+
try:
|
|
118
|
+
yield
|
|
119
|
+
finally:
|
|
120
|
+
with _listener_lock:
|
|
121
|
+
_listener_users -= 1
|
|
122
|
+
if _listener_users == 0:
|
|
123
|
+
log_listener.stop()
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|