asimov-ptadata 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- asimov_ptadata/__init__.py +66 -0
- asimov_ptadata/cli.py +269 -0
- asimov_ptadata/fetch.py +301 -0
- asimov_ptadata/gwb.py +352 -0
- asimov_ptadata/gwb_fit.py +502 -0
- asimov_ptadata/gwb_settings_template.yml +19 -0
- asimov_ptadata/noise.py +353 -0
- asimov_ptadata/noise_fit.py +984 -0
- asimov_ptadata/noise_settings_template.yml +18 -0
- asimov_ptadata/pipeline.py +207 -0
- asimov_ptadata/reduce.py +208 -0
- asimov_ptadata/scheduling.py +65 -0
- asimov_ptadata/settings_template.yml +27 -0
- asimov_ptadata/sources.py +50 -0
- asimov_ptadata-0.1.0.dist-info/METADATA +318 -0
- asimov_ptadata-0.1.0.dist-info/RECORD +20 -0
- asimov_ptadata-0.1.0.dist-info/WHEEL +5 -0
- asimov_ptadata-0.1.0.dist-info/entry_points.txt +7 -0
- asimov_ptadata-0.1.0.dist-info/licenses/LICENSE +21 -0
- asimov_ptadata-0.1.0.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,66 @@
|
|
|
1
|
+
"""Pulsar timing array data acquisition and reduction, with an Asimov pipeline plugin."""
|
|
2
|
+
|
|
3
|
+
import importlib.util
|
|
4
|
+
|
|
5
|
+
# Only advertise Pipeline when the 'asimov' extra is actually installed -
|
|
6
|
+
# otherwise `from asimov_ptadata import *` would try to resolve it via
|
|
7
|
+
# __getattr__ below and hit the AttributeError it deliberately raises,
|
|
8
|
+
# where previously (before Pipeline's export became lazy) it was just
|
|
9
|
+
# absent from __all__. find_spec() only checks whether the top-level
|
|
10
|
+
# 'asimov' package is importable; it doesn't import asimov_ptadata.pipeline
|
|
11
|
+
# itself, so it can't reintroduce the circular-import this laziness exists
|
|
12
|
+
# to avoid.
|
|
13
|
+
__all__ = ["Pipeline"] if importlib.util.find_spec("asimov") is not None else []
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
def __getattr__(name):
|
|
17
|
+
"""Lazily import :class:`Pipeline` on first access (PEP 562).
|
|
18
|
+
|
|
19
|
+
``asimov_ptadata.pipeline`` imports ``asimov.pipeline``, which in turn
|
|
20
|
+
imports ``asimov.analysis`` - and *that* module loads every registered
|
|
21
|
+
``asimov.pipelines`` entry point (including this package's own
|
|
22
|
+
``ptadata`` entry, ``asimov_ptadata:Pipeline``) as part of building
|
|
23
|
+
asimov's pipeline registry. If ``Pipeline`` were imported eagerly here,
|
|
24
|
+
the *very first* import of this package for any reason (e.g. just using
|
|
25
|
+
the standalone ``ptadata`` CLI, or a test importing
|
|
26
|
+
``asimov_ptadata.fetch``) would recursively re-enter this
|
|
27
|
+
still-initializing module while asimov's entry-point loader looks it up
|
|
28
|
+
- finding no ``Pipeline`` attribute yet, since we'd still be in the
|
|
29
|
+
middle of importing it. That failure is swallowed by asimov's loader
|
|
30
|
+
(a warning, not an error), so the plugin would then silently be missing
|
|
31
|
+
from ``known_pipelines`` for the rest of the process - reproduced by
|
|
32
|
+
simply reordering imports, e.g. ``import asimov_ptadata.fetch`` before
|
|
33
|
+
anything imports ``asimov.analysis``.
|
|
34
|
+
|
|
35
|
+
Deferring the import to attribute-access time (which is exactly when
|
|
36
|
+
asimov's entry-point loader actually asks for ``Pipeline``, well after
|
|
37
|
+
this module itself has finished initialising) breaks that cycle.
|
|
38
|
+
"""
|
|
39
|
+
if name == "Pipeline":
|
|
40
|
+
try:
|
|
41
|
+
from .pipeline import Pipeline
|
|
42
|
+
except ModuleNotFoundError as exc:
|
|
43
|
+
# Only treat this as "the optional extra isn't installed" if
|
|
44
|
+
# 'asimov' itself is the missing module - a broad `except
|
|
45
|
+
# ImportError` here would also swallow a real bug inside
|
|
46
|
+
# .pipeline (a missing/renamed sub-dependency, a bad relative
|
|
47
|
+
# import, ...) and misreport it as an absent extra, which is
|
|
48
|
+
# much harder to diagnose than letting it propagate normally.
|
|
49
|
+
if exc.name != "asimov":
|
|
50
|
+
raise
|
|
51
|
+
raise AttributeError(
|
|
52
|
+
"asimov_ptadata.Pipeline requires the 'asimov' extra "
|
|
53
|
+
"(pip install asimov-ptadata[asimov])"
|
|
54
|
+
) from exc
|
|
55
|
+
return Pipeline
|
|
56
|
+
raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
|
|
57
|
+
|
|
58
|
+
try:
|
|
59
|
+
from importlib.metadata import version, PackageNotFoundError
|
|
60
|
+
except ImportError:
|
|
61
|
+
from importlib_metadata import version, PackageNotFoundError
|
|
62
|
+
|
|
63
|
+
try:
|
|
64
|
+
__version__ = version(__name__)
|
|
65
|
+
except PackageNotFoundError:
|
|
66
|
+
__version__ = "unknown"
|
asimov_ptadata/cli.py
ADDED
|
@@ -0,0 +1,269 @@
|
|
|
1
|
+
"""Command-line interface for ptadata: fetch and reduce pulsar timing array data."""
|
|
2
|
+
|
|
3
|
+
import dataclasses
|
|
4
|
+
from pathlib import Path
|
|
5
|
+
|
|
6
|
+
import click
|
|
7
|
+
import yaml
|
|
8
|
+
|
|
9
|
+
from . import fetch as fetch_
|
|
10
|
+
from . import gwb_fit as gwb_fit_
|
|
11
|
+
from . import noise_fit as noise_fit_
|
|
12
|
+
from . import reduce as reduce_
|
|
13
|
+
from . import sources
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
def _echo_paths(manifest):
|
|
17
|
+
plain = {}
|
|
18
|
+
for key, value in manifest.items():
|
|
19
|
+
plain[key] = [str(v) for v in value] if isinstance(value, list) else str(value)
|
|
20
|
+
click.echo(yaml.safe_dump(plain, sort_keys=False))
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
def _resolve_release_root(release_root, release_name, cache_dir):
|
|
24
|
+
if bool(release_root) == bool(release_name):
|
|
25
|
+
raise click.UsageError("Pass exactly one of --release or --release-name")
|
|
26
|
+
if release_name:
|
|
27
|
+
return sources.resolve_release(release_name, cache_dir)
|
|
28
|
+
return release_root
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
@click.group()
|
|
32
|
+
def main():
|
|
33
|
+
"""ptadata: fetch and reduce pulsar timing array data."""
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
@main.command()
|
|
37
|
+
@click.option("--pulsar", required=True, help="Pulsar name, e.g. J1713+0747")
|
|
38
|
+
@click.option(
|
|
39
|
+
"--release", "release_root",
|
|
40
|
+
type=click.Path(exists=True, file_okay=False),
|
|
41
|
+
help="Path to a pinned local data-release checkout",
|
|
42
|
+
)
|
|
43
|
+
@click.option(
|
|
44
|
+
"--release-name", type=click.Choice(sorted(sources.RELEASES)),
|
|
45
|
+
help="Name of a known release to clone automatically, as an alternative to --release",
|
|
46
|
+
)
|
|
47
|
+
@click.option(
|
|
48
|
+
"--cache-dir", type=click.Path(file_okay=False), default="./ptadata-cache", show_default=True,
|
|
49
|
+
help="Where cloned releases are cached when using --release-name",
|
|
50
|
+
)
|
|
51
|
+
@click.option("--outdir", required=True, type=click.Path(file_okay=False), help="Directory to stage files into")
|
|
52
|
+
def fetch(pulsar, release_root, release_name, cache_dir, outdir):
|
|
53
|
+
"""Locate and stage a pulsar's par/tim/clock files from a data release."""
|
|
54
|
+
release_root = _resolve_release_root(release_root, release_name, cache_dir)
|
|
55
|
+
manifest = fetch_.fetch_pulsar(pulsar, release_root, outdir)
|
|
56
|
+
_echo_paths(manifest)
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
@main.command()
|
|
60
|
+
@click.option("--par", "par_file", required=True, type=click.Path(exists=True, dir_okay=False))
|
|
61
|
+
@click.option("--tim", "tim_files", required=True, multiple=True, type=click.Path(exists=True, dir_okay=False))
|
|
62
|
+
@click.option("--outdir", required=True, type=click.Path(file_okay=False))
|
|
63
|
+
@click.option("--sigma-threshold", default=5.0, show_default=True)
|
|
64
|
+
@click.option(
|
|
65
|
+
"--clip/--no-clip", default=True, show_default=True,
|
|
66
|
+
help="Clip pre-fit residual outliers at --sigma-threshold (see the README's 'Outlier clipping').",
|
|
67
|
+
)
|
|
68
|
+
@click.option("--refit/--no-refit", default=True, show_default=True)
|
|
69
|
+
@click.option("--ephem", default=None)
|
|
70
|
+
@click.option("--bipm-version", default=None)
|
|
71
|
+
def reduce(par_file, tim_files, outdir, sigma_threshold, clip, refit, ephem, bipm_version):
|
|
72
|
+
"""Validate and reduce TOAs for a single pulsar, writing cleaned data and a QC report."""
|
|
73
|
+
report = reduce_.reduce_pulsar(
|
|
74
|
+
par_file, list(tim_files), outdir,
|
|
75
|
+
sigma_threshold=sigma_threshold if clip else None, do_refit=refit,
|
|
76
|
+
ephem=ephem, bipm_version=bipm_version,
|
|
77
|
+
)
|
|
78
|
+
click.echo(yaml.safe_dump(report.__dict__, sort_keys=False))
|
|
79
|
+
if report.status != "pass":
|
|
80
|
+
raise SystemExit(1)
|
|
81
|
+
|
|
82
|
+
|
|
83
|
+
@main.command()
|
|
84
|
+
@click.option("--settings", "settings_file", required=True, type=click.Path(exists=True, dir_okay=False))
|
|
85
|
+
def run(settings_file):
|
|
86
|
+
"""Fetch and reduce in one step, driven by a settings file (used by the Asimov plugin)."""
|
|
87
|
+
with open(settings_file) as f:
|
|
88
|
+
settings = yaml.safe_load(f)
|
|
89
|
+
|
|
90
|
+
pulsar = settings["pulsar"]
|
|
91
|
+
rundir = Path(settings["rundir"])
|
|
92
|
+
reduction = settings.get("reduction", {})
|
|
93
|
+
release = settings["release"]
|
|
94
|
+
|
|
95
|
+
if "name" in release:
|
|
96
|
+
release_root = sources.resolve_release(release["name"], rundir / "release-cache")
|
|
97
|
+
else:
|
|
98
|
+
release_root = release["root"]
|
|
99
|
+
|
|
100
|
+
staged = fetch_.fetch_pulsar(pulsar, release_root, rundir / "staged")
|
|
101
|
+
report = reduce_.reduce_pulsar(
|
|
102
|
+
staged["par"], staged["tim"], rundir / "reduced",
|
|
103
|
+
sigma_threshold=reduction.get("sigma threshold", 5.0) if reduction.get("clip outliers", True) else None,
|
|
104
|
+
do_refit=reduction.get("refit", True),
|
|
105
|
+
ephem=settings.get("ephemeris"),
|
|
106
|
+
bipm_version=settings.get("bipm version"),
|
|
107
|
+
extra_notes=fetch_.normalisation_notes(staged["tim normalisation"]),
|
|
108
|
+
)
|
|
109
|
+
click.echo(f"ptadata run complete for {pulsar}: status={report.status}")
|
|
110
|
+
if report.status != "pass":
|
|
111
|
+
raise SystemExit(1)
|
|
112
|
+
|
|
113
|
+
|
|
114
|
+
@main.command("noise-fit")
|
|
115
|
+
@click.option("--par", "par_file", required=True, type=click.Path(exists=True, dir_okay=False))
|
|
116
|
+
@click.option("--tim", "tim_files", required=True, multiple=True, type=click.Path(exists=True, dir_okay=False))
|
|
117
|
+
@click.option("--outdir", required=True, type=click.Path(file_okay=False))
|
|
118
|
+
@click.option("--niter", default=6000, show_default=True)
|
|
119
|
+
@click.option("--burn", default=1000, show_default=True)
|
|
120
|
+
@click.option("--red-noise-components", default=10, show_default=True)
|
|
121
|
+
@click.option("--dm-noise-components", default=10, show_default=True)
|
|
122
|
+
@click.option("--ecorr/--no-ecorr", "use_ecorr", default=True, show_default=True)
|
|
123
|
+
@click.option("--dm-noise/--no-dm-noise", "use_dm_noise", default=True, show_default=True)
|
|
124
|
+
@click.option(
|
|
125
|
+
"--two-stage/--one-stage", default=True, show_default=True,
|
|
126
|
+
help="Re-sample red/DM noise with the white noise fixed at its stage-1 posterior means.",
|
|
127
|
+
)
|
|
128
|
+
@click.option("--stage2-niter", default=None, type=int, help="Stage-2 iterations (default: --niter).")
|
|
129
|
+
@click.option("--optimise-start/--prior-start", default=True, show_default=True)
|
|
130
|
+
@click.option("--min-ess", default=200.0, show_default=True)
|
|
131
|
+
@click.option("--max-split-shift", default=0.3, show_default=True)
|
|
132
|
+
@click.option(
|
|
133
|
+
"--white-noise", type=click.Choice(["map", "par", "sample"]), default="map", show_default=True,
|
|
134
|
+
help="Fix white noise at its MAP (map) or the par file's TempoNest values (par), or sample it (sample).",
|
|
135
|
+
)
|
|
136
|
+
def noise_fit(
|
|
137
|
+
par_file, tim_files, outdir, niter, burn, red_noise_components, dm_noise_components, use_ecorr, use_dm_noise,
|
|
138
|
+
two_stage, stage2_niter, optimise_start, min_ess, max_split_shift, white_noise,
|
|
139
|
+
):
|
|
140
|
+
"""Run a single-pulsar Bayesian noise fit (enterprise + PINT + PTMCMCSampler)."""
|
|
141
|
+
report = noise_fit_.run_noise_fit(
|
|
142
|
+
par_file, list(tim_files), outdir,
|
|
143
|
+
niter=niter, burn=burn,
|
|
144
|
+
red_noise_components=red_noise_components,
|
|
145
|
+
dm_noise_components=dm_noise_components,
|
|
146
|
+
use_ecorr=use_ecorr,
|
|
147
|
+
use_dm_noise=use_dm_noise,
|
|
148
|
+
two_stage=two_stage,
|
|
149
|
+
stage2_niter=stage2_niter,
|
|
150
|
+
optimise_start=optimise_start,
|
|
151
|
+
min_ess=min_ess,
|
|
152
|
+
max_split_shift=max_split_shift,
|
|
153
|
+
white_noise=white_noise,
|
|
154
|
+
)
|
|
155
|
+
click.echo(yaml.safe_dump(dataclasses.asdict(report), sort_keys=False))
|
|
156
|
+
if report.status != "complete":
|
|
157
|
+
raise SystemExit(1)
|
|
158
|
+
|
|
159
|
+
|
|
160
|
+
@main.command("noise-run")
|
|
161
|
+
@click.option("--settings", "settings_file", required=True, type=click.Path(exists=True, dir_okay=False))
|
|
162
|
+
def noise_run(settings_file):
|
|
163
|
+
"""Run noise fits for every subject in a settings file (used by the ptadata-noise Asimov pipeline)."""
|
|
164
|
+
with open(settings_file) as f:
|
|
165
|
+
settings = yaml.safe_load(f)
|
|
166
|
+
|
|
167
|
+
rundir = Path(settings["rundir"])
|
|
168
|
+
sampler = settings.get("sampler", {})
|
|
169
|
+
|
|
170
|
+
failures = []
|
|
171
|
+
for subject in settings["subjects"]:
|
|
172
|
+
outdir = rundir / "noise" / subject["name"]
|
|
173
|
+
report = noise_fit_.run_noise_fit(
|
|
174
|
+
subject["par"], subject["tim"], outdir,
|
|
175
|
+
niter=sampler.get("niter", 6000),
|
|
176
|
+
burn=sampler.get("burn", 1000),
|
|
177
|
+
red_noise_components=sampler.get("red noise components", 10),
|
|
178
|
+
dm_noise_components=sampler.get("dm noise components", 10),
|
|
179
|
+
use_ecorr=sampler.get("ecorr", True),
|
|
180
|
+
use_dm_noise=sampler.get("dm noise", True),
|
|
181
|
+
two_stage=sampler.get("two stage", True),
|
|
182
|
+
stage2_niter=sampler.get("stage 2 niter"),
|
|
183
|
+
optimise_start=sampler.get("optimise start", True),
|
|
184
|
+
min_ess=sampler.get("min ess", 200),
|
|
185
|
+
max_split_shift=sampler.get("max split shift", 0.3),
|
|
186
|
+
white_noise=sampler.get("white noise", "map"),
|
|
187
|
+
dm_dips=[tuple(w) for w in sampler.get("dm dips", []) or []],
|
|
188
|
+
)
|
|
189
|
+
click.echo(f"noise-run complete for {subject['name']}: status={report.status}")
|
|
190
|
+
if report.status != "complete":
|
|
191
|
+
failures.append(subject["name"])
|
|
192
|
+
|
|
193
|
+
if failures:
|
|
194
|
+
raise SystemExit(1)
|
|
195
|
+
|
|
196
|
+
|
|
197
|
+
@main.command("gwb-fit")
|
|
198
|
+
@click.option(
|
|
199
|
+
"--pulsars", "pulsars_file", required=True, type=click.Path(exists=True, dir_okay=False),
|
|
200
|
+
help=(
|
|
201
|
+
"YAML file listing pulsars for the joint fit: a list of mappings with "
|
|
202
|
+
"name, par, tim, and noise_params (a dict of every fixed non-timing-model "
|
|
203
|
+
"noise parameter for that pulsar, keyed by its unprefixed name) - see "
|
|
204
|
+
"asimov_ptadata.gwb_fit._build_joint_pta."
|
|
205
|
+
),
|
|
206
|
+
)
|
|
207
|
+
@click.option("--outdir", required=True, type=click.Path(file_okay=False))
|
|
208
|
+
@click.option("--niter", default=6000, show_default=True)
|
|
209
|
+
@click.option("--burn", default=1000, show_default=True)
|
|
210
|
+
@click.option("--red-noise-components", default=10, show_default=True)
|
|
211
|
+
@click.option("--dm-noise-components", default=10, show_default=True)
|
|
212
|
+
@click.option("--gwb-components", default=10, show_default=True)
|
|
213
|
+
@click.option(
|
|
214
|
+
"--gwb-gamma", default=13 / 3, show_default=True, type=float,
|
|
215
|
+
help="Fixed common-process spectral index (13/3 for a GW-driven SMBHB background).",
|
|
216
|
+
)
|
|
217
|
+
@click.option("--sample-gwb-gamma", is_flag=True, help="Sample the spectral index instead of fixing it.")
|
|
218
|
+
def gwb_fit(
|
|
219
|
+
pulsars_file, outdir, niter, burn, red_noise_components, dm_noise_components, gwb_components, gwb_gamma,
|
|
220
|
+
sample_gwb_gamma,
|
|
221
|
+
):
|
|
222
|
+
"""Run a real, joint array-wide GWB search (fixed noise, enterprise + PINT + PTMCMCSampler)."""
|
|
223
|
+
with open(pulsars_file) as f:
|
|
224
|
+
pulsars = yaml.safe_load(f)
|
|
225
|
+
report = gwb_fit_.run_gwb_fit(
|
|
226
|
+
pulsars, outdir,
|
|
227
|
+
niter=niter, burn=burn,
|
|
228
|
+
red_noise_components=red_noise_components,
|
|
229
|
+
dm_noise_components=dm_noise_components,
|
|
230
|
+
gwb_components=gwb_components,
|
|
231
|
+
gwb_gamma=None if sample_gwb_gamma else gwb_gamma,
|
|
232
|
+
)
|
|
233
|
+
click.echo(yaml.safe_dump(dataclasses.asdict(report), sort_keys=False))
|
|
234
|
+
if report.status != "complete":
|
|
235
|
+
raise SystemExit(1)
|
|
236
|
+
|
|
237
|
+
|
|
238
|
+
@main.command("gwb-run")
|
|
239
|
+
@click.option("--settings", "settings_file", required=True, type=click.Path(exists=True, dir_okay=False))
|
|
240
|
+
def gwb_run(settings_file):
|
|
241
|
+
"""Run the joint GWB search for every subject in a settings file (used by the ptadata-gwb Asimov pipeline)."""
|
|
242
|
+
with open(settings_file) as f:
|
|
243
|
+
settings = yaml.safe_load(f)
|
|
244
|
+
|
|
245
|
+
rundir = Path(settings["rundir"])
|
|
246
|
+
sampler = settings.get("sampler", {})
|
|
247
|
+
outdir = rundir / "gwb"
|
|
248
|
+
|
|
249
|
+
report = gwb_fit_.run_gwb_fit(
|
|
250
|
+
settings["pulsars"], outdir,
|
|
251
|
+
niter=sampler.get("niter", 6000),
|
|
252
|
+
burn=sampler.get("burn", 1000),
|
|
253
|
+
red_noise_components=sampler.get("red noise components", 10),
|
|
254
|
+
dm_noise_components=sampler.get("dm noise components", 10),
|
|
255
|
+
gwb_components=sampler.get("gwb components", 10),
|
|
256
|
+
gwb_gamma=sampler.get("gwb gamma"),
|
|
257
|
+
vary_pulsar_noise=sampler.get("vary pulsar noise", False),
|
|
258
|
+
orf=sampler.get("orf", "hd"),
|
|
259
|
+
)
|
|
260
|
+
click.echo(f"gwb-run complete: status={report.status}")
|
|
261
|
+
if report.status != "complete":
|
|
262
|
+
raise SystemExit(1)
|
|
263
|
+
|
|
264
|
+
|
|
265
|
+
@main.command("list-releases")
|
|
266
|
+
def list_releases():
|
|
267
|
+
"""List known PTA data releases that can be fetched by name."""
|
|
268
|
+
for name, spec in sorted(sources.RELEASES.items()):
|
|
269
|
+
click.echo(f"{name}: {spec['url']} (ref: {spec['ref']})")
|
asimov_ptadata/fetch.py
ADDED
|
@@ -0,0 +1,301 @@
|
|
|
1
|
+
"""Locate and stage pulsar timing data from a pinned data-release checkout.
|
|
2
|
+
|
|
3
|
+
A PTA data release (IPTA DR, NANOGrav Nyr, EPTA DR, ...) is distributed as a
|
|
4
|
+
tagged git repository or archive, not queried live the way GWOSC/CVMFS serve
|
|
5
|
+
GW strain data. "Fetching" here means: have a pinned local checkout of a
|
|
6
|
+
specific release, then locate and stage one pulsar's files out of it.
|
|
7
|
+
|
|
8
|
+
Real releases (checked against a live clone of IPTA DR2) commonly lay a
|
|
9
|
+
pulsar out as its own directory, with the top-level .tim file pulling in
|
|
10
|
+
per-backend files from a "tims/" subdirectory via TEMPO2 INCLUDE lines, and
|
|
11
|
+
more than one .par file present for different unit conventions (e.g. a plain
|
|
12
|
+
and a ".TDB" variant). That means a pulsar's files generally can't be
|
|
13
|
+
flat-copied by glob match alone - the tims/ subdirectory has to move with its
|
|
14
|
+
includer file for the relative INCLUDE paths to still resolve, and multiple
|
|
15
|
+
.par candidates need to be surfaced rather than guessed at silently, since
|
|
16
|
+
picking the wrong unit convention is exactly the kind of thing that silently
|
|
17
|
+
breaks array-wide consistency in a combined analysis.
|
|
18
|
+
"""
|
|
19
|
+
|
|
20
|
+
from __future__ import annotations
|
|
21
|
+
|
|
22
|
+
import re
|
|
23
|
+
import shutil
|
|
24
|
+
import subprocess
|
|
25
|
+
from pathlib import Path
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
class ReleaseNotFoundError(FileNotFoundError):
|
|
29
|
+
pass
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
class ReleaseSource:
|
|
33
|
+
"""A pinned local checkout of a PTA data release."""
|
|
34
|
+
|
|
35
|
+
def __init__(self, release_root):
|
|
36
|
+
self.release_root = Path(release_root)
|
|
37
|
+
if not self.release_root.exists():
|
|
38
|
+
raise ReleaseNotFoundError(f"Release root {self.release_root} does not exist")
|
|
39
|
+
|
|
40
|
+
def pulsar_dir(self, psr_name):
|
|
41
|
+
"""Return the release's per-pulsar directory for psr_name, or None if this
|
|
42
|
+
release isn't laid out that way (caller should fall back to a flat glob)."""
|
|
43
|
+
matches = [p for p in self.release_root.rglob(psr_name) if p.is_dir()]
|
|
44
|
+
if not matches:
|
|
45
|
+
return None
|
|
46
|
+
if len(matches) > 1:
|
|
47
|
+
raise ReleaseNotFoundError(
|
|
48
|
+
f"Multiple directories named {psr_name} found under {self.release_root} "
|
|
49
|
+
f"(likely parallel data combinations): {matches}. "
|
|
50
|
+
"Point --release at the specific combination you want."
|
|
51
|
+
)
|
|
52
|
+
return matches[0]
|
|
53
|
+
|
|
54
|
+
def par_candidates(self, search_dir, psr_name):
|
|
55
|
+
matches = sorted(search_dir.glob(f"{psr_name}*.par"))
|
|
56
|
+
if not matches:
|
|
57
|
+
raise ReleaseNotFoundError(f"No .par file found for {psr_name} under {search_dir}")
|
|
58
|
+
return matches
|
|
59
|
+
|
|
60
|
+
def tim_candidates(self, search_dir, psr_name):
|
|
61
|
+
matches = sorted(search_dir.glob(f"{psr_name}*.tim"))
|
|
62
|
+
if not matches:
|
|
63
|
+
raise ReleaseNotFoundError(f"No .tim files found for {psr_name} under {search_dir}")
|
|
64
|
+
return matches
|
|
65
|
+
|
|
66
|
+
def clock_dir(self):
|
|
67
|
+
for candidate in ("clock", "clk", "T2runtime/clock"):
|
|
68
|
+
path = self.release_root / candidate
|
|
69
|
+
if path.exists():
|
|
70
|
+
return path
|
|
71
|
+
return None
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
_FLAG_NAME = re.compile(r"^-[A-Za-z_]")
|
|
75
|
+
_NUMBER = re.compile(r"^[+-]?(\d+\.?\d*|\.\d+)([eEdD][+-]?\d+)?$")
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
def _is_tempo2_toa(fields):
|
|
79
|
+
# name freq mjd error site [flags...]
|
|
80
|
+
return len(fields) >= 5 and all(_NUMBER.match(f) for f in fields[1:4])
|
|
81
|
+
|
|
82
|
+
|
|
83
|
+
def normalise_tim_line(line):
|
|
84
|
+
"""Rewrite one tempo2 ``.tim`` line so PINT reads it the way tempo2 does.
|
|
85
|
+
|
|
86
|
+
Returns ``(line, commented, dropped_flags, addsat)``, where ``addsat`` is
|
|
87
|
+
the ``-addsat`` offset (seconds) folded into ``-to``, or None if the line
|
|
88
|
+
had none. See :func:`normalise_tim_files` for the rules applied.
|
|
89
|
+
"""
|
|
90
|
+
if line.rstrip("\r\n") == "C":
|
|
91
|
+
# A bare "C": a comment to tempo2, but PINT only knows "C " (with a
|
|
92
|
+
# space) and tries to parse it as a TOA (IndexError).
|
|
93
|
+
return "#" + line, False, [], None
|
|
94
|
+
if line.startswith("C") and len(line) > 1 and not line[1].isspace() and not line.startswith("CC "):
|
|
95
|
+
return "C " + line, True, [], None
|
|
96
|
+
|
|
97
|
+
fields = line.split()
|
|
98
|
+
if not _is_tempo2_toa(fields):
|
|
99
|
+
return line, False, [], None
|
|
100
|
+
|
|
101
|
+
flags = fields[5:]
|
|
102
|
+
kept, dropped = [], []
|
|
103
|
+
i = 0
|
|
104
|
+
while i < len(flags):
|
|
105
|
+
token = flags[i]
|
|
106
|
+
if _FLAG_NAME.match(token) and (i + 1 == len(flags) or _FLAG_NAME.match(flags[i + 1])):
|
|
107
|
+
dropped.append(token)
|
|
108
|
+
i += 1
|
|
109
|
+
continue
|
|
110
|
+
kept.append(token)
|
|
111
|
+
i += 1
|
|
112
|
+
kept, addsat = _fold_addsat_into_to(kept)
|
|
113
|
+
if not dropped and addsat is None:
|
|
114
|
+
return line, False, [], None
|
|
115
|
+
ending = line[len(line.rstrip("\r\n")):]
|
|
116
|
+
return " ".join(fields[:5] + kept) + ending, False, dropped, addsat
|
|
117
|
+
|
|
118
|
+
|
|
119
|
+
def _fold_addsat_into_to(flags):
|
|
120
|
+
"""Replace ``-addsat X`` with an equivalent ``-to`` time offset.
|
|
121
|
+
|
|
122
|
+
tempo2 adds ``-addsat`` (seconds) to the site arrival time; PINT ignores
|
|
123
|
+
the flag but applies ``-to`` (seconds) to the arrival time along with
|
|
124
|
+
the clock corrections, so the two are summed into a single ``-to``.
|
|
125
|
+
"""
|
|
126
|
+
names = flags[0::2]
|
|
127
|
+
if "-addsat" not in names or len(flags) % 2:
|
|
128
|
+
return flags, None
|
|
129
|
+
pairs = dict(zip(flags[0::2], flags[1::2]))
|
|
130
|
+
try:
|
|
131
|
+
addsat = float(pairs["-addsat"])
|
|
132
|
+
offset = addsat + float(pairs.get("-to", 0.0))
|
|
133
|
+
except ValueError:
|
|
134
|
+
return flags, None
|
|
135
|
+
out = []
|
|
136
|
+
for name, value in zip(flags[0::2], flags[1::2]):
|
|
137
|
+
if name == "-addsat":
|
|
138
|
+
continue
|
|
139
|
+
out += [name, repr(offset) if name == "-to" else value]
|
|
140
|
+
if "-to" not in pairs and offset != 0.0:
|
|
141
|
+
out += ["-to", repr(offset)]
|
|
142
|
+
return out, addsat
|
|
143
|
+
|
|
144
|
+
|
|
145
|
+
def normalise_tim_files(paths):
|
|
146
|
+
"""Normalise staged tempo2 ``.tim`` files in place for PINT.
|
|
147
|
+
|
|
148
|
+
Three tempo2 conventions that real releases (IPTA DR2 in particular) rely
|
|
149
|
+
on are read differently by PINT, so they are rewritten in the *staged*
|
|
150
|
+
copies - never in the release checkout itself:
|
|
151
|
+
|
|
152
|
+
- **Commented-out TOAs.** tempo2 treats any line whose first character
|
|
153
|
+
is an upper-case ``C`` as a comment, and releases comment out a TOA
|
|
154
|
+
by prefixing it directly (``C???? c015621.align...``,
|
|
155
|
+
``Cc054887.align...``, ``C200404522.bb ...``). PINT only recognises
|
|
156
|
+
``C ``, ``c ``, ``CC `` and ``#`` as comments, so it either reads such
|
|
157
|
+
a line as a real TOA (silently re-including data the release
|
|
158
|
+
removed) or fails to parse it. Such lines get a ``"C "`` prefix. A
|
|
159
|
+
lower-case ``c`` is *not* a comment to tempo2 - it is the first
|
|
160
|
+
letter of many archive file names - and is left alone.
|
|
161
|
+
- **Flags with no value** (``... -projid -beconfig -snr 70.99``). tempo2
|
|
162
|
+
accepts a flag followed directly by another flag or the end of the
|
|
163
|
+
line; PINT pairs flag tokens strictly two by two, so it silently
|
|
164
|
+
assigns the next flag's *name* as the value (and, with an odd number
|
|
165
|
+
of them, shifts every later flag, or fails to parse). A valueless
|
|
166
|
+
flag carries no information, so it is dropped.
|
|
167
|
+
|
|
168
|
+
- **``-addsat`` arrival-time corrections** (``... -addsat -1``). tempo2
|
|
169
|
+
adds the flag's value, in seconds, to the site arrival time; IPTA DR2
|
|
170
|
+
uses it to correct 1-second timestamp errors on some Effelsberg TOAs.
|
|
171
|
+
PINT ignores it, leaving those TOAs a second out - residuals of up to
|
|
172
|
+
half a pulse period once folded. It is replaced with PINT's ``-to``
|
|
173
|
+
time offset (seconds, applied with the clock corrections), summed
|
|
174
|
+
with any ``-to`` already on the line.
|
|
175
|
+
|
|
176
|
+
All three behaviours were checked against tempo2 itself. Only lines that
|
|
177
|
+
need it are rewritten. Returns a dict with the number of TOAs commented
|
|
178
|
+
out, valueless flags dropped and ``-addsat`` corrections converted, and
|
|
179
|
+
the files changed.
|
|
180
|
+
"""
|
|
181
|
+
summary = {"commented TOAs": 0, "valueless flags dropped": 0, "addsat corrections": 0, "files changed": []}
|
|
182
|
+
for path in paths:
|
|
183
|
+
path = Path(path)
|
|
184
|
+
# latin-1 round-trips any byte, so nothing else in the file changes.
|
|
185
|
+
with open(path, encoding="latin-1", newline="") as f:
|
|
186
|
+
lines = f.readlines()
|
|
187
|
+
changed = False
|
|
188
|
+
for i, line in enumerate(lines):
|
|
189
|
+
new, commented, dropped, addsat = normalise_tim_line(line)
|
|
190
|
+
if new != line:
|
|
191
|
+
lines[i] = new
|
|
192
|
+
changed = True
|
|
193
|
+
summary["commented TOAs"] += int(commented)
|
|
194
|
+
summary["valueless flags dropped"] += len(dropped)
|
|
195
|
+
summary["addsat corrections"] += int(addsat is not None)
|
|
196
|
+
if changed:
|
|
197
|
+
with open(path, "w", encoding="latin-1", newline="") as f:
|
|
198
|
+
f.writelines(lines)
|
|
199
|
+
summary["files changed"].append(path)
|
|
200
|
+
return summary
|
|
201
|
+
|
|
202
|
+
|
|
203
|
+
def normalisation_notes(summary):
|
|
204
|
+
"""QC-report notes describing what :func:`normalise_tim_files` changed."""
|
|
205
|
+
notes = []
|
|
206
|
+
if summary["commented TOAs"]:
|
|
207
|
+
notes.append(
|
|
208
|
+
f"{summary['commented TOAs']} TOA line(s) commented out tempo2-style (C prefix) "
|
|
209
|
+
"were rewritten as PINT comments while staging"
|
|
210
|
+
)
|
|
211
|
+
if summary["valueless flags dropped"]:
|
|
212
|
+
notes.append(f"{summary['valueless flags dropped']} valueless TOA flag(s) were dropped while staging")
|
|
213
|
+
if summary.get("addsat corrections"):
|
|
214
|
+
notes.append(
|
|
215
|
+
f"{summary['addsat corrections']} -addsat arrival-time correction(s) were converted to -to time "
|
|
216
|
+
"offsets while staging"
|
|
217
|
+
)
|
|
218
|
+
return notes
|
|
219
|
+
|
|
220
|
+
|
|
221
|
+
def _copy_into(src, outdir):
|
|
222
|
+
dst = outdir / src.name
|
|
223
|
+
shutil.copy2(src, dst)
|
|
224
|
+
return dst
|
|
225
|
+
|
|
226
|
+
|
|
227
|
+
def _par_units(path):
|
|
228
|
+
for line in path.read_text().splitlines():
|
|
229
|
+
parts = line.split()
|
|
230
|
+
if parts and parts[0] == "UNITS":
|
|
231
|
+
return parts[1] if len(parts) > 1 else None
|
|
232
|
+
return None
|
|
233
|
+
|
|
234
|
+
|
|
235
|
+
def _select_par(candidates):
|
|
236
|
+
"""Prefer a TDB-convention file over filename tidiness.
|
|
237
|
+
|
|
238
|
+
PINT only loads UNITS TDB par files (it refuses TCB outright, which is
|
|
239
|
+
tempo2's native convention and common in release par files - checked
|
|
240
|
+
against a real IPTA DR2 pulsar, where the plainest-named file was TCB
|
|
241
|
+
and only the ".TDB"-suffixed variant loaded). Among the remaining
|
|
242
|
+
candidates, prefer the fewest dotted suffixes as a tiebreak.
|
|
243
|
+
"""
|
|
244
|
+
non_tcb = [p for p in candidates if _par_units(p) != "TCB"]
|
|
245
|
+
pool = non_tcb if non_tcb else candidates
|
|
246
|
+
return min(pool, key=lambda p: p.stem.count("."))
|
|
247
|
+
|
|
248
|
+
|
|
249
|
+
def clone_release(git_url, tag, dest):
|
|
250
|
+
"""Shallow-clone a specific tagged release of a PTA data-release repository."""
|
|
251
|
+
dest = Path(dest)
|
|
252
|
+
if dest.exists():
|
|
253
|
+
return dest
|
|
254
|
+
subprocess.run(
|
|
255
|
+
["git", "clone", "--branch", tag, "--depth", "1", git_url, str(dest)],
|
|
256
|
+
check=True,
|
|
257
|
+
)
|
|
258
|
+
return dest
|
|
259
|
+
|
|
260
|
+
|
|
261
|
+
def fetch_pulsar(psr_name, release_root, outdir):
|
|
262
|
+
"""Stage a pulsar's par/tim/clock files from a release checkout into outdir.
|
|
263
|
+
|
|
264
|
+
Returns a manifest dict with keys "par" (the selected file), "par
|
|
265
|
+
candidates" (all matches found), "tim" (list), "tim normalisation" (the
|
|
266
|
+
summary from :func:`normalise_tim_files`, which has been applied to
|
|
267
|
+
every staged .tim file) and, if present, "clock dir".
|
|
268
|
+
"""
|
|
269
|
+
outdir = Path(outdir)
|
|
270
|
+
outdir.mkdir(parents=True, exist_ok=True)
|
|
271
|
+
source = ReleaseSource(release_root)
|
|
272
|
+
|
|
273
|
+
psr_dir = source.pulsar_dir(psr_name)
|
|
274
|
+
|
|
275
|
+
if psr_dir is not None:
|
|
276
|
+
# Directory-per-pulsar layout: copy the whole directory so relative
|
|
277
|
+
# INCLUDE paths (e.g. a tims/ subdirectory) keep resolving.
|
|
278
|
+
dest = outdir / psr_name
|
|
279
|
+
shutil.copytree(psr_dir, dest, dirs_exist_ok=True)
|
|
280
|
+
par_candidates = source.par_candidates(dest, psr_name)
|
|
281
|
+
tim_candidates = source.tim_candidates(dest, psr_name)
|
|
282
|
+
# INCLUDEd per-backend files hold the TOAs, so normalise every one.
|
|
283
|
+
staged_tims = sorted(dest.rglob("*.tim"))
|
|
284
|
+
else:
|
|
285
|
+
# Flat layout: no per-pulsar directory, so copy the matched files individually.
|
|
286
|
+
par_candidates = [_copy_into(p, outdir) for p in source.par_candidates(source.release_root, psr_name)]
|
|
287
|
+
tim_candidates = [_copy_into(p, outdir) for p in source.tim_candidates(source.release_root, psr_name)]
|
|
288
|
+
staged_tims = tim_candidates
|
|
289
|
+
|
|
290
|
+
manifest = {
|
|
291
|
+
"par": _select_par(par_candidates),
|
|
292
|
+
"par candidates": par_candidates,
|
|
293
|
+
"tim": tim_candidates,
|
|
294
|
+
"tim normalisation": normalise_tim_files(staged_tims),
|
|
295
|
+
}
|
|
296
|
+
|
|
297
|
+
clock_dir = source.clock_dir()
|
|
298
|
+
if clock_dir is not None:
|
|
299
|
+
manifest["clock dir"] = clock_dir
|
|
300
|
+
|
|
301
|
+
return manifest
|