RACS-tools 4.3.1__tar.gz → 4.3.2__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: racs-tools
3
- Version: 4.3.1
3
+ Version: 4.3.2
4
4
  Summary: Useful scripts for RACS.
5
5
  License: BSD
6
6
  License-File: LICENSE
@@ -1,6 +1,6 @@
1
1
  [tool.poetry]
2
2
  name = "racs-tools"
3
- version = "4.3.1" # A placeholder
3
+ version = "4.3.2" # A placeholder
4
4
  description = "Useful scripts for RACS."
5
5
  authors = ["Alec Thomson"]
6
6
  license = "BSD"
@@ -33,9 +33,9 @@ from racs_tools.convolve_uv import (
33
33
  )
34
34
  from racs_tools.logging import (
35
35
  init_worker,
36
- log_listener,
37
36
  log_queue,
38
37
  logger,
38
+ running_log_listener,
39
39
  set_verbosity,
40
40
  )
41
41
  from racs_tools.parallel import get_executor
@@ -482,84 +482,83 @@ def smooth_fits_files(
482
482
  Beam: Common beam used.
483
483
  """
484
484
  # Required for multiprocessing logging
485
- log_listener.start()
486
- if dryrun:
487
- logger.info("Doing a dry run -- no files will be saved")
488
-
489
- # Check early as can fail
490
- Executor = get_executor(executor_type)
491
-
492
- # Get file list
493
- if listfile:
494
- assert len(infile_list) == 1, "Only one list file can be provided!"
495
- with open(infile_list[0]) as f:
496
- infile_list = [Path(line) for line in f.read().splitlines()]
497
- files = sorted(infile_list)
498
- if len(files) == 0:
499
- raise FileNotFoundError("No files found!")
500
-
501
- conv_mode = parse_conv_mode(conv_mode)
502
-
503
- nonetest = [param is None for param in (bmaj, bmin, bpa)]
504
- if all(nonetest):
505
- target_beam = None
506
- elif any(nonetest):
507
- raise ValueError("Please specify all target beam params!")
508
- else:
509
- target_beam = Beam(bmaj * u.arcsec, bmin * u.arcsec, bpa * u.deg)
510
- logger.info(f"Target beam is {target_beam!r}")
511
-
512
- # Find smallest common beam
513
- common_beam, all_beams = get_common_beam(
514
- files,
515
- conv_mode=conv_mode,
516
- target_beam=target_beam,
517
- cutoff=cutoff,
518
- tolerance=tolerance,
519
- nsamps=nsamps,
520
- epsilon=epsilon,
521
- )
485
+ with running_log_listener():
486
+ if dryrun:
487
+ logger.info("Doing a dry run -- no files will be saved")
488
+
489
+ # Check early as can fail
490
+ Executor = get_executor(executor_type)
491
+
492
+ # Get file list
493
+ if listfile:
494
+ assert len(infile_list) == 1, "Only one list file can be provided!"
495
+ with open(infile_list[0]) as f:
496
+ infile_list = [Path(line) for line in f.read().splitlines()]
497
+ files = sorted(infile_list)
498
+ if len(files) == 0:
499
+ raise FileNotFoundError("No files found!")
500
+
501
+ conv_mode = parse_conv_mode(conv_mode)
502
+
503
+ nonetest = [param is None for param in (bmaj, bmin, bpa)]
504
+ if all(nonetest):
505
+ target_beam = None
506
+ elif any(nonetest):
507
+ raise ValueError("Please specify all target beam params!")
508
+ else:
509
+ target_beam = Beam(bmaj * u.arcsec, bmin * u.arcsec, bpa * u.deg)
510
+ logger.info(f"Target beam is {target_beam!r}")
511
+
512
+ # Find smallest common beam
513
+ common_beam, all_beams = get_common_beam(
514
+ files,
515
+ conv_mode=conv_mode,
516
+ target_beam=target_beam,
517
+ cutoff=cutoff,
518
+ tolerance=tolerance,
519
+ nsamps=nsamps,
520
+ epsilon=epsilon,
521
+ )
522
522
 
523
- if target_beam is not None:
524
- if not check_target_beam(target_beam, all_beams, files, cutoff):
525
- raise BeamError("Please choose a larger target beam!")
523
+ if target_beam is not None:
524
+ if not check_target_beam(target_beam, all_beams, files, cutoff):
525
+ raise BeamError("Please choose a larger target beam!")
526
526
 
527
- common_beam = target_beam
527
+ common_beam = target_beam
528
528
 
529
- if circularise:
530
- logger.info("Circular beam requested, setting BMIN=BMAJ and BPA=0")
531
- common_beam = Beam(
532
- major=common_beam.major,
533
- minor=common_beam.major,
534
- pa=0 * u.deg,
535
- )
536
-
537
- logger.info(f"Final beam is {common_beam!r}")
538
- with Executor(
539
- max_workers=ncores, initializer=init_worker, initargs=(log_queue, verbosity)
540
- ) as executor:
541
- futures = []
542
- for file in files:
543
- future = executor.submit(
544
- beamcon_2d_on_fits,
545
- file=file,
546
- outdir=outdir,
547
- new_beam=common_beam,
548
- conv_mode=conv_mode,
549
- suffix=suffix,
550
- prefix=prefix,
551
- cutoff=cutoff,
552
- dryrun=dryrun,
529
+ if circularise:
530
+ logger.info("Circular beam requested, setting BMIN=BMAJ and BPA=0")
531
+ common_beam = Beam(
532
+ major=common_beam.major,
533
+ minor=common_beam.major,
534
+ pa=0 * u.deg,
553
535
  )
554
- futures.append(future)
555
-
556
- beam_log_list = [future.result() for future in futures]
557
- if log is not None:
558
- writelog(beam_log_list, log)
559
536
 
560
- logger.info("Done!")
561
- log_listener.enqueue_sentinel()
562
- return common_beam
537
+ logger.info(f"Final beam is {common_beam!r}")
538
+ with Executor(
539
+ max_workers=ncores, initializer=init_worker, initargs=(log_queue, verbosity)
540
+ ) as executor:
541
+ futures = []
542
+ for file in files:
543
+ future = executor.submit(
544
+ beamcon_2d_on_fits,
545
+ file=file,
546
+ outdir=outdir,
547
+ new_beam=common_beam,
548
+ conv_mode=conv_mode,
549
+ suffix=suffix,
550
+ prefix=prefix,
551
+ cutoff=cutoff,
552
+ dryrun=dryrun,
553
+ )
554
+ futures.append(future)
555
+
556
+ beam_log_list = [future.result() for future in futures]
557
+ if log is not None:
558
+ writelog(beam_log_list, log)
559
+
560
+ logger.info("Done!")
561
+ return common_beam
563
562
 
564
563
 
565
564
  def cli():
@@ -30,9 +30,9 @@ from racs_tools.beamcon_2D import my_ceil, round_up
30
30
  from racs_tools.convolve_uv import NAN_BEAM, ZERO_BEAM, parse_conv_mode, smooth
31
31
  from racs_tools.logging import (
32
32
  init_worker,
33
- log_listener,
34
33
  log_queue,
35
34
  logger,
35
+ running_log_listener,
36
36
  set_verbosity,
37
37
  )
38
38
  from racs_tools.parallel import get_executor
@@ -1076,131 +1076,131 @@ def smooth_fits_cube(
1076
1076
  tuple[list[CubeData], list[CommonBeamData], list[Path]]: Cube data and common beam data, and output filenames.
1077
1077
  """
1078
1078
  # Required for multiprocessing logging
1079
- log_listener.start()
1080
- if dryrun:
1081
- logger.info("Doing a dry run -- no files will be saved")
1082
-
1083
- # Check early as can fail
1084
- Executor = get_executor(executor_type)
1085
-
1086
- # Check mode
1087
- logger.info(f"Mode is {mode}")
1088
- if mode == "natural":
1089
- logger.info("Smoothing each channel to a common resolution")
1090
- elif mode == "total":
1091
- logger.info("Smoothing all channels to a common resolution")
1092
- else:
1093
- raise ValueError(f"Mode must be 'natural' or 'total', not '{mode}'")
1079
+ with running_log_listener():
1080
+ if dryrun:
1081
+ logger.info("Doing a dry run -- no files will be saved")
1082
+
1083
+ # Check early as can fail
1084
+ Executor = get_executor(executor_type)
1085
+
1086
+ # Check mode
1087
+ logger.info(f"Mode is {mode}")
1088
+ if mode == "natural":
1089
+ logger.info("Smoothing each channel to a common resolution")
1090
+ elif mode == "total":
1091
+ logger.info("Smoothing all channels to a common resolution")
1092
+ else:
1093
+ raise ValueError(f"Mode must be 'natural' or 'total', not '{mode}'")
1094
1094
 
1095
- # Check cutoff
1096
- if cutoff is not None:
1097
- cutoff *= u.arcsec
1098
- logger.info(f"Cutoff is: {cutoff}")
1095
+ # Check cutoff
1096
+ if cutoff is not None:
1097
+ cutoff *= u.arcsec
1098
+ logger.info(f"Cutoff is: {cutoff}")
1099
1099
 
1100
- # Check target
1101
- conv_mode = parse_conv_mode(conv_mode)
1100
+ # Check target
1101
+ conv_mode = parse_conv_mode(conv_mode)
1102
1102
 
1103
- target_beam = _get_target_beam(bmaj=bmaj, bmin=bmin, bpa=bpa)
1103
+ target_beam = _get_target_beam(bmaj=bmaj, bmin=bmin, bpa=bpa)
1104
1104
 
1105
- files = sorted(infiles_list)
1106
- if len(files) == 0:
1107
- raise FileNotFoundError("No files found!")
1105
+ files = sorted(infiles_list)
1106
+ if len(files) == 0:
1107
+ raise FileNotFoundError("No files found!")
1108
1108
 
1109
- outdir_list: list[Path] = (
1110
- [outdir] * len(files) if outdir is not None else [f.parent for f in files]
1111
- )
1109
+ outdir_list: list[Path] = (
1110
+ [outdir] * len(files) if outdir is not None else [f.parent for f in files]
1111
+ )
1112
1112
 
1113
- cube_data_list = make_data(files, outdir_list)
1113
+ cube_data_list = make_data(files, outdir_list)
1114
1114
 
1115
- # Sanity check channel counts
1116
- nchans = np.array([cube_data.nchan for cube_data in cube_data_list])
1117
- if not all(nchans == nchans[0]):
1118
- raise ValueError(f"Unequal number of spectral channels! Got {nchans}")
1115
+ # Sanity check channel counts
1116
+ nchans = np.array([cube_data.nchan for cube_data in cube_data_list])
1117
+ if not all(nchans == nchans[0]):
1118
+ raise ValueError(f"Unequal number of spectral channels! Got {nchans}")
1119
1119
 
1120
- if isinstance(target_beam, Beams) and any(nchans != len(target_beam)):
1121
- raise ValueError(
1122
- f"Unequal length of target beams ({len(target_beam)}) to channels ({nchans})"
1123
- )
1120
+ if isinstance(target_beam, Beams) and any(nchans != len(target_beam)):
1121
+ raise ValueError(
1122
+ f"Unequal length of target beams ({len(target_beam)}) to channels ({nchans})"
1123
+ )
1124
1124
 
1125
- nchans = nchans[0]
1125
+ nchans = nchans[0]
1126
1126
 
1127
- # Check suffix
1128
- if suffix is None:
1129
- suffix = mode
1127
+ # Check suffix
1128
+ if suffix is None:
1129
+ suffix = mode
1130
1130
 
1131
- # Apply some masking
1132
- cube_data_list = masking(cube_data_list, cutoff=cutoff)
1131
+ # Apply some masking
1132
+ cube_data_list = masking(cube_data_list, cutoff=cutoff)
1133
1133
 
1134
- if not uselogs:
1135
- common_beam_data_list = commonbeamer(
1136
- cube_data_list=cube_data_list,
1137
- nchans=nchans,
1138
- conv_mode=conv_mode,
1139
- target_beam=target_beam,
1140
- mode=mode,
1141
- suffix=suffix,
1142
- circularise=circularise,
1143
- tolerance=tolerance,
1144
- nsamps=nsamps,
1145
- epsilon=epsilon,
1146
- )
1147
- else:
1148
- logger.info("Reading from convolve beamlog files")
1149
- common_beam_data_list: list[CommonBeamData] = []
1150
- for cube_data in cube_data_list:
1151
- commonbeam_log = cube_data.beamlog.with_suffix(f".{suffix}.txt")
1152
- common_beam_data = readlogs(commonbeam_log)
1153
- common_beam_data_list.append(common_beam_data)
1154
-
1155
- if dryrun:
1156
- logger.info("Doing a dryrun so all done!")
1157
- return cube_data_list, common_beam_data_list
1158
-
1159
- # Init the files in parallel
1160
- logger.info("Initialising output files")
1161
- # Init output files and retrieve file names
1162
- with Executor(
1163
- max_workers=ncores, initializer=init_worker, initargs=(log_queue, verbosity)
1164
- ) as executor:
1165
- futures = []
1166
- for cube_data, common_beam_data in zip(cube_data_list, common_beam_data_list):
1167
- future = executor.submit(
1168
- initfiles,
1169
- filename=cube_data.filename,
1170
- commonbeams=common_beam_data.commonbeams,
1171
- outdir=cube_data.outdir,
1134
+ if not uselogs:
1135
+ common_beam_data_list = commonbeamer(
1136
+ cube_data_list=cube_data_list,
1137
+ nchans=nchans,
1138
+ conv_mode=conv_mode,
1139
+ target_beam=target_beam,
1172
1140
  mode=mode,
1173
1141
  suffix=suffix,
1174
- prefix=prefix,
1175
- ref_chan=ref_chan,
1142
+ circularise=circularise,
1143
+ tolerance=tolerance,
1144
+ nsamps=nsamps,
1145
+ epsilon=epsilon,
1176
1146
  )
1177
- futures.append(future)
1178
- outfiles: list[Path] = [future.result() for future in futures]
1179
-
1180
- with Executor(
1181
- max_workers=ncores, initializer=init_worker, initargs=(log_queue, verbosity)
1182
- ) as executor:
1183
- futures = []
1184
- for cube_data, common_beam_data, outfile in zip(
1185
- cube_data_list, common_beam_data_list, outfiles
1186
- ):
1187
- for chan in range(nchans):
1147
+ else:
1148
+ logger.info("Reading from convolve beamlog files")
1149
+ common_beam_data_list: list[CommonBeamData] = []
1150
+ for cube_data in cube_data_list:
1151
+ commonbeam_log = cube_data.beamlog.with_suffix(f".{suffix}.txt")
1152
+ common_beam_data = readlogs(commonbeam_log)
1153
+ common_beam_data_list.append(common_beam_data)
1154
+
1155
+ if dryrun:
1156
+ logger.info("Doing a dryrun so all done!")
1157
+ return cube_data_list, common_beam_data_list
1158
+
1159
+ # Init the files in parallel
1160
+ logger.info("Initialising output files")
1161
+ # Init output files and retrieve file names
1162
+ with Executor(
1163
+ max_workers=ncores, initializer=init_worker, initargs=(log_queue, verbosity)
1164
+ ) as executor:
1165
+ futures = []
1166
+ for cube_data, common_beam_data in zip(
1167
+ cube_data_list, common_beam_data_list
1168
+ ):
1188
1169
  future = executor.submit(
1189
- smooth_and_write_plane,
1190
- chan=chan,
1191
- cube_data=cube_data,
1192
- common_beam_data=common_beam_data,
1193
- outfile=outfile,
1194
- conv_mode=conv_mode,
1170
+ initfiles,
1171
+ filename=cube_data.filename,
1172
+ commonbeams=common_beam_data.commonbeams,
1173
+ outdir=cube_data.outdir,
1174
+ mode=mode,
1175
+ suffix=suffix,
1176
+ prefix=prefix,
1177
+ ref_chan=ref_chan,
1195
1178
  )
1196
1179
  futures.append(future)
1197
- _ = [future.result() for future in futures]
1198
-
1199
- logger.info("Done!")
1180
+ outfiles: list[Path] = [future.result() for future in futures]
1181
+
1182
+ with Executor(
1183
+ max_workers=ncores, initializer=init_worker, initargs=(log_queue, verbosity)
1184
+ ) as executor:
1185
+ futures = []
1186
+ for cube_data, common_beam_data, outfile in zip(
1187
+ cube_data_list, common_beam_data_list, outfiles
1188
+ ):
1189
+ for chan in range(nchans):
1190
+ future = executor.submit(
1191
+ smooth_and_write_plane,
1192
+ chan=chan,
1193
+ cube_data=cube_data,
1194
+ common_beam_data=common_beam_data,
1195
+ outfile=outfile,
1196
+ conv_mode=conv_mode,
1197
+ )
1198
+ futures.append(future)
1199
+ _ = [future.result() for future in futures]
1200
1200
 
1201
- log_listener.enqueue_sentinel()
1201
+ logger.info("Done!")
1202
1202
 
1203
- return cube_data_list, common_beam_data_list, outfiles
1203
+ return cube_data_list, common_beam_data_list, outfiles
1204
1204
 
1205
1205
 
1206
1206
  def cli():
@@ -2,6 +2,9 @@
2
2
 
3
3
  import logging
4
4
  import multiprocessing as mp
5
+ import threading
6
+ from collections.abc import Iterator
7
+ from contextlib import contextmanager
5
8
  from logging.handlers import QueueHandler, QueueListener
6
9
 
7
10
  logging.captureWarnings(True)
@@ -71,8 +74,50 @@ def init_worker(log_queue: mp.Queue, verbosity: int = 0) -> None:
71
74
 
72
75
  set_verbosity(logger, verbosity)
73
76
 
74
- handler = QueueHandler(log_queue)
75
- logger.addHandler(handler)
77
+ # Keep exactly one queue handler per process. With a thread executor every
78
+ # worker shares this logger, so adding unconditionally fans each record out
79
+ # once per worker and leaves the handlers attached for later calls - where
80
+ # they keep filling the queue after the listener has stopped consuming it.
81
+ for stale in [h for h in logger.handlers if isinstance(h, QueueHandler)]:
82
+ logger.removeHandler(stale)
83
+
84
+ logger.addHandler(QueueHandler(log_queue))
76
85
 
77
86
 
78
87
  logger, log_listener, log_queue = setup_logger()
88
+
89
+ _listener_lock = threading.Lock()
90
+ _listener_users = 0
91
+ """Number of ``running_log_listener`` blocks currently active."""
92
+
93
+
94
+ @contextmanager
95
+ def running_log_listener() -> Iterator[None]:
96
+ """Run the module-level log listener for the duration of the block.
97
+
98
+ On Python 3.13, ``QueueListener.start()`` raises ``RuntimeError`` if the
99
+ listener's monitor thread is still set from a previous run. Calling
100
+ ``enqueue_sentinel()`` alone (without ``stop()``) leaves the thread
101
+ reference set even after the monitor has exited, so the listener must
102
+ always be stopped -- on every exit path -- before it can be started
103
+ again.
104
+
105
+ The listener is a module-level singleton, so nested or concurrent blocks
106
+ share the one run: it is started by the first block to enter and stopped
107
+ by the last to leave. Without that, an inner ``stop()`` would enqueue a
108
+ sentinel that the outer block's monitor thread consumes, and the
109
+ ``join()`` inside ``stop()`` would then never return.
110
+ """
111
+ global _listener_users
112
+
113
+ with _listener_lock:
114
+ if _listener_users == 0:
115
+ log_listener.start()
116
+ _listener_users += 1
117
+ try:
118
+ yield
119
+ finally:
120
+ with _listener_lock:
121
+ _listener_users -= 1
122
+ if _listener_users == 0:
123
+ log_listener.stop()
File without changes
File without changes
File without changes