asimov-ptadata 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,66 @@
1
+ """Pulsar timing array data acquisition and reduction, with an Asimov pipeline plugin."""
2
+
3
+ import importlib.util
4
+
5
+ # Only advertise Pipeline when the 'asimov' extra is actually installed -
6
+ # otherwise `from asimov_ptadata import *` would try to resolve it via
7
+ # __getattr__ below and hit the AttributeError it deliberately raises,
8
+ # where previously (before Pipeline's export became lazy) it was just
9
+ # absent from __all__. find_spec() only checks whether the top-level
10
+ # 'asimov' package is importable; it doesn't import asimov_ptadata.pipeline
11
+ # itself, so it can't reintroduce the circular-import this laziness exists
12
+ # to avoid.
13
+ __all__ = ["Pipeline"] if importlib.util.find_spec("asimov") is not None else []
14
+
15
+
16
+ def __getattr__(name):
17
+ """Lazily import :class:`Pipeline` on first access (PEP 562).
18
+
19
+ ``asimov_ptadata.pipeline`` imports ``asimov.pipeline``, which in turn
20
+ imports ``asimov.analysis`` - and *that* module loads every registered
21
+ ``asimov.pipelines`` entry point (including this package's own
22
+ ``ptadata`` entry, ``asimov_ptadata:Pipeline``) as part of building
23
+ asimov's pipeline registry. If ``Pipeline`` were imported eagerly here,
24
+ the *very first* import of this package for any reason (e.g. just using
25
+ the standalone ``ptadata`` CLI, or a test importing
26
+ ``asimov_ptadata.fetch``) would recursively re-enter this
27
+ still-initializing module while asimov's entry-point loader looks it up
28
+ - finding no ``Pipeline`` attribute yet, since we'd still be in the
29
+ middle of importing it. That failure is swallowed by asimov's loader
30
+ (a warning, not an error), so the plugin would then silently be missing
31
+ from ``known_pipelines`` for the rest of the process - reproduced by
32
+ simply reordering imports, e.g. ``import asimov_ptadata.fetch`` before
33
+ anything imports ``asimov.analysis``.
34
+
35
+ Deferring the import to attribute-access time (which is exactly when
36
+ asimov's entry-point loader actually asks for ``Pipeline``, well after
37
+ this module itself has finished initialising) breaks that cycle.
38
+ """
39
+ if name == "Pipeline":
40
+ try:
41
+ from .pipeline import Pipeline
42
+ except ModuleNotFoundError as exc:
43
+ # Only treat this as "the optional extra isn't installed" if
44
+ # 'asimov' itself is the missing module - a broad `except
45
+ # ImportError` here would also swallow a real bug inside
46
+ # .pipeline (a missing/renamed sub-dependency, a bad relative
47
+ # import, ...) and misreport it as an absent extra, which is
48
+ # much harder to diagnose than letting it propagate normally.
49
+ if exc.name != "asimov":
50
+ raise
51
+ raise AttributeError(
52
+ "asimov_ptadata.Pipeline requires the 'asimov' extra "
53
+ "(pip install asimov-ptadata[asimov])"
54
+ ) from exc
55
+ return Pipeline
56
+ raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
57
+
58
+ try:
59
+ from importlib.metadata import version, PackageNotFoundError
60
+ except ImportError:
61
+ from importlib_metadata import version, PackageNotFoundError
62
+
63
+ try:
64
+ __version__ = version(__name__)
65
+ except PackageNotFoundError:
66
+ __version__ = "unknown"
asimov_ptadata/cli.py ADDED
@@ -0,0 +1,269 @@
1
+ """Command-line interface for ptadata: fetch and reduce pulsar timing array data."""
2
+
3
+ import dataclasses
4
+ from pathlib import Path
5
+
6
+ import click
7
+ import yaml
8
+
9
+ from . import fetch as fetch_
10
+ from . import gwb_fit as gwb_fit_
11
+ from . import noise_fit as noise_fit_
12
+ from . import reduce as reduce_
13
+ from . import sources
14
+
15
+
16
+ def _echo_paths(manifest):
17
+ plain = {}
18
+ for key, value in manifest.items():
19
+ plain[key] = [str(v) for v in value] if isinstance(value, list) else str(value)
20
+ click.echo(yaml.safe_dump(plain, sort_keys=False))
21
+
22
+
23
+ def _resolve_release_root(release_root, release_name, cache_dir):
24
+ if bool(release_root) == bool(release_name):
25
+ raise click.UsageError("Pass exactly one of --release or --release-name")
26
+ if release_name:
27
+ return sources.resolve_release(release_name, cache_dir)
28
+ return release_root
29
+
30
+
31
+ @click.group()
32
+ def main():
33
+ """ptadata: fetch and reduce pulsar timing array data."""
34
+
35
+
36
+ @main.command()
37
+ @click.option("--pulsar", required=True, help="Pulsar name, e.g. J1713+0747")
38
+ @click.option(
39
+ "--release", "release_root",
40
+ type=click.Path(exists=True, file_okay=False),
41
+ help="Path to a pinned local data-release checkout",
42
+ )
43
+ @click.option(
44
+ "--release-name", type=click.Choice(sorted(sources.RELEASES)),
45
+ help="Name of a known release to clone automatically, as an alternative to --release",
46
+ )
47
+ @click.option(
48
+ "--cache-dir", type=click.Path(file_okay=False), default="./ptadata-cache", show_default=True,
49
+ help="Where cloned releases are cached when using --release-name",
50
+ )
51
+ @click.option("--outdir", required=True, type=click.Path(file_okay=False), help="Directory to stage files into")
52
+ def fetch(pulsar, release_root, release_name, cache_dir, outdir):
53
+ """Locate and stage a pulsar's par/tim/clock files from a data release."""
54
+ release_root = _resolve_release_root(release_root, release_name, cache_dir)
55
+ manifest = fetch_.fetch_pulsar(pulsar, release_root, outdir)
56
+ _echo_paths(manifest)
57
+
58
+
59
+ @main.command()
60
+ @click.option("--par", "par_file", required=True, type=click.Path(exists=True, dir_okay=False))
61
+ @click.option("--tim", "tim_files", required=True, multiple=True, type=click.Path(exists=True, dir_okay=False))
62
+ @click.option("--outdir", required=True, type=click.Path(file_okay=False))
63
+ @click.option("--sigma-threshold", default=5.0, show_default=True)
64
+ @click.option(
65
+ "--clip/--no-clip", default=True, show_default=True,
66
+ help="Clip pre-fit residual outliers at --sigma-threshold (see the README's 'Outlier clipping').",
67
+ )
68
+ @click.option("--refit/--no-refit", default=True, show_default=True)
69
+ @click.option("--ephem", default=None)
70
+ @click.option("--bipm-version", default=None)
71
+ def reduce(par_file, tim_files, outdir, sigma_threshold, clip, refit, ephem, bipm_version):
72
+ """Validate and reduce TOAs for a single pulsar, writing cleaned data and a QC report."""
73
+ report = reduce_.reduce_pulsar(
74
+ par_file, list(tim_files), outdir,
75
+ sigma_threshold=sigma_threshold if clip else None, do_refit=refit,
76
+ ephem=ephem, bipm_version=bipm_version,
77
+ )
78
+ click.echo(yaml.safe_dump(report.__dict__, sort_keys=False))
79
+ if report.status != "pass":
80
+ raise SystemExit(1)
81
+
82
+
83
+ @main.command()
84
+ @click.option("--settings", "settings_file", required=True, type=click.Path(exists=True, dir_okay=False))
85
+ def run(settings_file):
86
+ """Fetch and reduce in one step, driven by a settings file (used by the Asimov plugin)."""
87
+ with open(settings_file) as f:
88
+ settings = yaml.safe_load(f)
89
+
90
+ pulsar = settings["pulsar"]
91
+ rundir = Path(settings["rundir"])
92
+ reduction = settings.get("reduction", {})
93
+ release = settings["release"]
94
+
95
+ if "name" in release:
96
+ release_root = sources.resolve_release(release["name"], rundir / "release-cache")
97
+ else:
98
+ release_root = release["root"]
99
+
100
+ staged = fetch_.fetch_pulsar(pulsar, release_root, rundir / "staged")
101
+ report = reduce_.reduce_pulsar(
102
+ staged["par"], staged["tim"], rundir / "reduced",
103
+ sigma_threshold=reduction.get("sigma threshold", 5.0) if reduction.get("clip outliers", True) else None,
104
+ do_refit=reduction.get("refit", True),
105
+ ephem=settings.get("ephemeris"),
106
+ bipm_version=settings.get("bipm version"),
107
+ extra_notes=fetch_.normalisation_notes(staged["tim normalisation"]),
108
+ )
109
+ click.echo(f"ptadata run complete for {pulsar}: status={report.status}")
110
+ if report.status != "pass":
111
+ raise SystemExit(1)
112
+
113
+
114
+ @main.command("noise-fit")
115
+ @click.option("--par", "par_file", required=True, type=click.Path(exists=True, dir_okay=False))
116
+ @click.option("--tim", "tim_files", required=True, multiple=True, type=click.Path(exists=True, dir_okay=False))
117
+ @click.option("--outdir", required=True, type=click.Path(file_okay=False))
118
+ @click.option("--niter", default=6000, show_default=True)
119
+ @click.option("--burn", default=1000, show_default=True)
120
+ @click.option("--red-noise-components", default=10, show_default=True)
121
+ @click.option("--dm-noise-components", default=10, show_default=True)
122
+ @click.option("--ecorr/--no-ecorr", "use_ecorr", default=True, show_default=True)
123
+ @click.option("--dm-noise/--no-dm-noise", "use_dm_noise", default=True, show_default=True)
124
+ @click.option(
125
+ "--two-stage/--one-stage", default=True, show_default=True,
126
+ help="Re-sample red/DM noise with the white noise fixed at its stage-1 posterior means.",
127
+ )
128
+ @click.option("--stage2-niter", default=None, type=int, help="Stage-2 iterations (default: --niter).")
129
+ @click.option("--optimise-start/--prior-start", default=True, show_default=True)
130
+ @click.option("--min-ess", default=200.0, show_default=True)
131
+ @click.option("--max-split-shift", default=0.3, show_default=True)
132
+ @click.option(
133
+ "--white-noise", type=click.Choice(["map", "par", "sample"]), default="map", show_default=True,
134
+ help="Fix white noise at its MAP (map) or the par file's TempoNest values (par), or sample it (sample).",
135
+ )
136
+ def noise_fit(
137
+ par_file, tim_files, outdir, niter, burn, red_noise_components, dm_noise_components, use_ecorr, use_dm_noise,
138
+ two_stage, stage2_niter, optimise_start, min_ess, max_split_shift, white_noise,
139
+ ):
140
+ """Run a single-pulsar Bayesian noise fit (enterprise + PINT + PTMCMCSampler)."""
141
+ report = noise_fit_.run_noise_fit(
142
+ par_file, list(tim_files), outdir,
143
+ niter=niter, burn=burn,
144
+ red_noise_components=red_noise_components,
145
+ dm_noise_components=dm_noise_components,
146
+ use_ecorr=use_ecorr,
147
+ use_dm_noise=use_dm_noise,
148
+ two_stage=two_stage,
149
+ stage2_niter=stage2_niter,
150
+ optimise_start=optimise_start,
151
+ min_ess=min_ess,
152
+ max_split_shift=max_split_shift,
153
+ white_noise=white_noise,
154
+ )
155
+ click.echo(yaml.safe_dump(dataclasses.asdict(report), sort_keys=False))
156
+ if report.status != "complete":
157
+ raise SystemExit(1)
158
+
159
+
160
+ @main.command("noise-run")
161
+ @click.option("--settings", "settings_file", required=True, type=click.Path(exists=True, dir_okay=False))
162
+ def noise_run(settings_file):
163
+ """Run noise fits for every subject in a settings file (used by the ptadata-noise Asimov pipeline)."""
164
+ with open(settings_file) as f:
165
+ settings = yaml.safe_load(f)
166
+
167
+ rundir = Path(settings["rundir"])
168
+ sampler = settings.get("sampler", {})
169
+
170
+ failures = []
171
+ for subject in settings["subjects"]:
172
+ outdir = rundir / "noise" / subject["name"]
173
+ report = noise_fit_.run_noise_fit(
174
+ subject["par"], subject["tim"], outdir,
175
+ niter=sampler.get("niter", 6000),
176
+ burn=sampler.get("burn", 1000),
177
+ red_noise_components=sampler.get("red noise components", 10),
178
+ dm_noise_components=sampler.get("dm noise components", 10),
179
+ use_ecorr=sampler.get("ecorr", True),
180
+ use_dm_noise=sampler.get("dm noise", True),
181
+ two_stage=sampler.get("two stage", True),
182
+ stage2_niter=sampler.get("stage 2 niter"),
183
+ optimise_start=sampler.get("optimise start", True),
184
+ min_ess=sampler.get("min ess", 200),
185
+ max_split_shift=sampler.get("max split shift", 0.3),
186
+ white_noise=sampler.get("white noise", "map"),
187
+ dm_dips=[tuple(w) for w in sampler.get("dm dips", []) or []],
188
+ )
189
+ click.echo(f"noise-run complete for {subject['name']}: status={report.status}")
190
+ if report.status != "complete":
191
+ failures.append(subject["name"])
192
+
193
+ if failures:
194
+ raise SystemExit(1)
195
+
196
+
197
+ @main.command("gwb-fit")
198
+ @click.option(
199
+ "--pulsars", "pulsars_file", required=True, type=click.Path(exists=True, dir_okay=False),
200
+ help=(
201
+ "YAML file listing pulsars for the joint fit: a list of mappings with "
202
+ "name, par, tim, and noise_params (a dict of every fixed non-timing-model "
203
+ "noise parameter for that pulsar, keyed by its unprefixed name) - see "
204
+ "asimov_ptadata.gwb_fit._build_joint_pta."
205
+ ),
206
+ )
207
+ @click.option("--outdir", required=True, type=click.Path(file_okay=False))
208
+ @click.option("--niter", default=6000, show_default=True)
209
+ @click.option("--burn", default=1000, show_default=True)
210
+ @click.option("--red-noise-components", default=10, show_default=True)
211
+ @click.option("--dm-noise-components", default=10, show_default=True)
212
+ @click.option("--gwb-components", default=10, show_default=True)
213
+ @click.option(
214
+ "--gwb-gamma", default=13 / 3, show_default=True, type=float,
215
+ help="Fixed common-process spectral index (13/3 for a GW-driven SMBHB background).",
216
+ )
217
+ @click.option("--sample-gwb-gamma", is_flag=True, help="Sample the spectral index instead of fixing it.")
218
+ def gwb_fit(
219
+ pulsars_file, outdir, niter, burn, red_noise_components, dm_noise_components, gwb_components, gwb_gamma,
220
+ sample_gwb_gamma,
221
+ ):
222
+ """Run a real, joint array-wide GWB search (fixed noise, enterprise + PINT + PTMCMCSampler)."""
223
+ with open(pulsars_file) as f:
224
+ pulsars = yaml.safe_load(f)
225
+ report = gwb_fit_.run_gwb_fit(
226
+ pulsars, outdir,
227
+ niter=niter, burn=burn,
228
+ red_noise_components=red_noise_components,
229
+ dm_noise_components=dm_noise_components,
230
+ gwb_components=gwb_components,
231
+ gwb_gamma=None if sample_gwb_gamma else gwb_gamma,
232
+ )
233
+ click.echo(yaml.safe_dump(dataclasses.asdict(report), sort_keys=False))
234
+ if report.status != "complete":
235
+ raise SystemExit(1)
236
+
237
+
238
+ @main.command("gwb-run")
239
+ @click.option("--settings", "settings_file", required=True, type=click.Path(exists=True, dir_okay=False))
240
+ def gwb_run(settings_file):
241
+ """Run the joint GWB search for every subject in a settings file (used by the ptadata-gwb Asimov pipeline)."""
242
+ with open(settings_file) as f:
243
+ settings = yaml.safe_load(f)
244
+
245
+ rundir = Path(settings["rundir"])
246
+ sampler = settings.get("sampler", {})
247
+ outdir = rundir / "gwb"
248
+
249
+ report = gwb_fit_.run_gwb_fit(
250
+ settings["pulsars"], outdir,
251
+ niter=sampler.get("niter", 6000),
252
+ burn=sampler.get("burn", 1000),
253
+ red_noise_components=sampler.get("red noise components", 10),
254
+ dm_noise_components=sampler.get("dm noise components", 10),
255
+ gwb_components=sampler.get("gwb components", 10),
256
+ gwb_gamma=sampler.get("gwb gamma"),
257
+ vary_pulsar_noise=sampler.get("vary pulsar noise", False),
258
+ orf=sampler.get("orf", "hd"),
259
+ )
260
+ click.echo(f"gwb-run complete: status={report.status}")
261
+ if report.status != "complete":
262
+ raise SystemExit(1)
263
+
264
+
265
+ @main.command("list-releases")
266
+ def list_releases():
267
+ """List known PTA data releases that can be fetched by name."""
268
+ for name, spec in sorted(sources.RELEASES.items()):
269
+ click.echo(f"{name}: {spec['url']} (ref: {spec['ref']})")
@@ -0,0 +1,301 @@
1
+ """Locate and stage pulsar timing data from a pinned data-release checkout.
2
+
3
+ A PTA data release (IPTA DR, NANOGrav Nyr, EPTA DR, ...) is distributed as a
4
+ tagged git repository or archive, not queried live the way GWOSC/CVMFS serve
5
+ GW strain data. "Fetching" here means: have a pinned local checkout of a
6
+ specific release, then locate and stage one pulsar's files out of it.
7
+
8
+ Real releases (checked against a live clone of IPTA DR2) commonly lay a
9
+ pulsar out as its own directory, with the top-level .tim file pulling in
10
+ per-backend files from a "tims/" subdirectory via TEMPO2 INCLUDE lines, and
11
+ more than one .par file present for different unit conventions (e.g. a plain
12
+ and a ".TDB" variant). That means a pulsar's files generally can't be
13
+ flat-copied by glob match alone - the tims/ subdirectory has to move with its
14
+ includer file for the relative INCLUDE paths to still resolve, and multiple
15
+ .par candidates need to be surfaced rather than guessed at silently, since
16
+ picking the wrong unit convention is exactly the kind of thing that silently
17
+ breaks array-wide consistency in a combined analysis.
18
+ """
19
+
20
+ from __future__ import annotations
21
+
22
+ import re
23
+ import shutil
24
+ import subprocess
25
+ from pathlib import Path
26
+
27
+
28
+ class ReleaseNotFoundError(FileNotFoundError):
29
+ pass
30
+
31
+
32
+ class ReleaseSource:
33
+ """A pinned local checkout of a PTA data release."""
34
+
35
+ def __init__(self, release_root):
36
+ self.release_root = Path(release_root)
37
+ if not self.release_root.exists():
38
+ raise ReleaseNotFoundError(f"Release root {self.release_root} does not exist")
39
+
40
+ def pulsar_dir(self, psr_name):
41
+ """Return the release's per-pulsar directory for psr_name, or None if this
42
+ release isn't laid out that way (caller should fall back to a flat glob)."""
43
+ matches = [p for p in self.release_root.rglob(psr_name) if p.is_dir()]
44
+ if not matches:
45
+ return None
46
+ if len(matches) > 1:
47
+ raise ReleaseNotFoundError(
48
+ f"Multiple directories named {psr_name} found under {self.release_root} "
49
+ f"(likely parallel data combinations): {matches}. "
50
+ "Point --release at the specific combination you want."
51
+ )
52
+ return matches[0]
53
+
54
+ def par_candidates(self, search_dir, psr_name):
55
+ matches = sorted(search_dir.glob(f"{psr_name}*.par"))
56
+ if not matches:
57
+ raise ReleaseNotFoundError(f"No .par file found for {psr_name} under {search_dir}")
58
+ return matches
59
+
60
+ def tim_candidates(self, search_dir, psr_name):
61
+ matches = sorted(search_dir.glob(f"{psr_name}*.tim"))
62
+ if not matches:
63
+ raise ReleaseNotFoundError(f"No .tim files found for {psr_name} under {search_dir}")
64
+ return matches
65
+
66
+ def clock_dir(self):
67
+ for candidate in ("clock", "clk", "T2runtime/clock"):
68
+ path = self.release_root / candidate
69
+ if path.exists():
70
+ return path
71
+ return None
72
+
73
+
74
+ _FLAG_NAME = re.compile(r"^-[A-Za-z_]")
75
+ _NUMBER = re.compile(r"^[+-]?(\d+\.?\d*|\.\d+)([eEdD][+-]?\d+)?$")
76
+
77
+
78
+ def _is_tempo2_toa(fields):
79
+ # name freq mjd error site [flags...]
80
+ return len(fields) >= 5 and all(_NUMBER.match(f) for f in fields[1:4])
81
+
82
+
83
+ def normalise_tim_line(line):
84
+ """Rewrite one tempo2 ``.tim`` line so PINT reads it the way tempo2 does.
85
+
86
+ Returns ``(line, commented, dropped_flags, addsat)``, where ``addsat`` is
87
+ the ``-addsat`` offset (seconds) folded into ``-to``, or None if the line
88
+ had none. See :func:`normalise_tim_files` for the rules applied.
89
+ """
90
+ if line.rstrip("\r\n") == "C":
91
+ # A bare "C": a comment to tempo2, but PINT only knows "C " (with a
92
+ # space) and tries to parse it as a TOA (IndexError).
93
+ return "#" + line, False, [], None
94
+ if line.startswith("C") and len(line) > 1 and not line[1].isspace() and not line.startswith("CC "):
95
+ return "C " + line, True, [], None
96
+
97
+ fields = line.split()
98
+ if not _is_tempo2_toa(fields):
99
+ return line, False, [], None
100
+
101
+ flags = fields[5:]
102
+ kept, dropped = [], []
103
+ i = 0
104
+ while i < len(flags):
105
+ token = flags[i]
106
+ if _FLAG_NAME.match(token) and (i + 1 == len(flags) or _FLAG_NAME.match(flags[i + 1])):
107
+ dropped.append(token)
108
+ i += 1
109
+ continue
110
+ kept.append(token)
111
+ i += 1
112
+ kept, addsat = _fold_addsat_into_to(kept)
113
+ if not dropped and addsat is None:
114
+ return line, False, [], None
115
+ ending = line[len(line.rstrip("\r\n")):]
116
+ return " ".join(fields[:5] + kept) + ending, False, dropped, addsat
117
+
118
+
119
+ def _fold_addsat_into_to(flags):
120
+ """Replace ``-addsat X`` with an equivalent ``-to`` time offset.
121
+
122
+ tempo2 adds ``-addsat`` (seconds) to the site arrival time; PINT ignores
123
+ the flag but applies ``-to`` (seconds) to the arrival time along with
124
+ the clock corrections, so the two are summed into a single ``-to``.
125
+ """
126
+ names = flags[0::2]
127
+ if "-addsat" not in names or len(flags) % 2:
128
+ return flags, None
129
+ pairs = dict(zip(flags[0::2], flags[1::2]))
130
+ try:
131
+ addsat = float(pairs["-addsat"])
132
+ offset = addsat + float(pairs.get("-to", 0.0))
133
+ except ValueError:
134
+ return flags, None
135
+ out = []
136
+ for name, value in zip(flags[0::2], flags[1::2]):
137
+ if name == "-addsat":
138
+ continue
139
+ out += [name, repr(offset) if name == "-to" else value]
140
+ if "-to" not in pairs and offset != 0.0:
141
+ out += ["-to", repr(offset)]
142
+ return out, addsat
143
+
144
+
145
+ def normalise_tim_files(paths):
146
+ """Normalise staged tempo2 ``.tim`` files in place for PINT.
147
+
148
+ Three tempo2 conventions that real releases (IPTA DR2 in particular) rely
149
+ on are read differently by PINT, so they are rewritten in the *staged*
150
+ copies - never in the release checkout itself:
151
+
152
+ - **Commented-out TOAs.** tempo2 treats any line whose first character
153
+ is an upper-case ``C`` as a comment, and releases comment out a TOA
154
+ by prefixing it directly (``C???? c015621.align...``,
155
+ ``Cc054887.align...``, ``C200404522.bb ...``). PINT only recognises
156
+ ``C ``, ``c ``, ``CC `` and ``#`` as comments, so it either reads such
157
+ a line as a real TOA (silently re-including data the release
158
+ removed) or fails to parse it. Such lines get a ``"C "`` prefix. A
159
+ lower-case ``c`` is *not* a comment to tempo2 - it is the first
160
+ letter of many archive file names - and is left alone.
161
+ - **Flags with no value** (``... -projid -beconfig -snr 70.99``). tempo2
162
+ accepts a flag followed directly by another flag or the end of the
163
+ line; PINT pairs flag tokens strictly two by two, so it silently
164
+ assigns the next flag's *name* as the value (and, with an odd number
165
+ of them, shifts every later flag, or fails to parse). A valueless
166
+ flag carries no information, so it is dropped.
167
+
168
+ - **``-addsat`` arrival-time corrections** (``... -addsat -1``). tempo2
169
+ adds the flag's value, in seconds, to the site arrival time; IPTA DR2
170
+ uses it to correct 1-second timestamp errors on some Effelsberg TOAs.
171
+ PINT ignores it, leaving those TOAs a second out - residuals of up to
172
+ half a pulse period once folded. It is replaced with PINT's ``-to``
173
+ time offset (seconds, applied with the clock corrections), summed
174
+ with any ``-to`` already on the line.
175
+
176
+ All three behaviours were checked against tempo2 itself. Only lines that
177
+ need it are rewritten. Returns a dict with the number of TOAs commented
178
+ out, valueless flags dropped and ``-addsat`` corrections converted, and
179
+ the files changed.
180
+ """
181
+ summary = {"commented TOAs": 0, "valueless flags dropped": 0, "addsat corrections": 0, "files changed": []}
182
+ for path in paths:
183
+ path = Path(path)
184
+ # latin-1 round-trips any byte, so nothing else in the file changes.
185
+ with open(path, encoding="latin-1", newline="") as f:
186
+ lines = f.readlines()
187
+ changed = False
188
+ for i, line in enumerate(lines):
189
+ new, commented, dropped, addsat = normalise_tim_line(line)
190
+ if new != line:
191
+ lines[i] = new
192
+ changed = True
193
+ summary["commented TOAs"] += int(commented)
194
+ summary["valueless flags dropped"] += len(dropped)
195
+ summary["addsat corrections"] += int(addsat is not None)
196
+ if changed:
197
+ with open(path, "w", encoding="latin-1", newline="") as f:
198
+ f.writelines(lines)
199
+ summary["files changed"].append(path)
200
+ return summary
201
+
202
+
203
+ def normalisation_notes(summary):
204
+ """QC-report notes describing what :func:`normalise_tim_files` changed."""
205
+ notes = []
206
+ if summary["commented TOAs"]:
207
+ notes.append(
208
+ f"{summary['commented TOAs']} TOA line(s) commented out tempo2-style (C prefix) "
209
+ "were rewritten as PINT comments while staging"
210
+ )
211
+ if summary["valueless flags dropped"]:
212
+ notes.append(f"{summary['valueless flags dropped']} valueless TOA flag(s) were dropped while staging")
213
+ if summary.get("addsat corrections"):
214
+ notes.append(
215
+ f"{summary['addsat corrections']} -addsat arrival-time correction(s) were converted to -to time "
216
+ "offsets while staging"
217
+ )
218
+ return notes
219
+
220
+
221
+ def _copy_into(src, outdir):
222
+ dst = outdir / src.name
223
+ shutil.copy2(src, dst)
224
+ return dst
225
+
226
+
227
+ def _par_units(path):
228
+ for line in path.read_text().splitlines():
229
+ parts = line.split()
230
+ if parts and parts[0] == "UNITS":
231
+ return parts[1] if len(parts) > 1 else None
232
+ return None
233
+
234
+
235
+ def _select_par(candidates):
236
+ """Prefer a TDB-convention file over filename tidiness.
237
+
238
+ PINT only loads UNITS TDB par files (it refuses TCB outright, which is
239
+ tempo2's native convention and common in release par files - checked
240
+ against a real IPTA DR2 pulsar, where the plainest-named file was TCB
241
+ and only the ".TDB"-suffixed variant loaded). Among the remaining
242
+ candidates, prefer the fewest dotted suffixes as a tiebreak.
243
+ """
244
+ non_tcb = [p for p in candidates if _par_units(p) != "TCB"]
245
+ pool = non_tcb if non_tcb else candidates
246
+ return min(pool, key=lambda p: p.stem.count("."))
247
+
248
+
249
+ def clone_release(git_url, tag, dest):
250
+ """Shallow-clone a specific tagged release of a PTA data-release repository."""
251
+ dest = Path(dest)
252
+ if dest.exists():
253
+ return dest
254
+ subprocess.run(
255
+ ["git", "clone", "--branch", tag, "--depth", "1", git_url, str(dest)],
256
+ check=True,
257
+ )
258
+ return dest
259
+
260
+
261
+ def fetch_pulsar(psr_name, release_root, outdir):
262
+ """Stage a pulsar's par/tim/clock files from a release checkout into outdir.
263
+
264
+ Returns a manifest dict with keys "par" (the selected file), "par
265
+ candidates" (all matches found), "tim" (list), "tim normalisation" (the
266
+ summary from :func:`normalise_tim_files`, which has been applied to
267
+ every staged .tim file) and, if present, "clock dir".
268
+ """
269
+ outdir = Path(outdir)
270
+ outdir.mkdir(parents=True, exist_ok=True)
271
+ source = ReleaseSource(release_root)
272
+
273
+ psr_dir = source.pulsar_dir(psr_name)
274
+
275
+ if psr_dir is not None:
276
+ # Directory-per-pulsar layout: copy the whole directory so relative
277
+ # INCLUDE paths (e.g. a tims/ subdirectory) keep resolving.
278
+ dest = outdir / psr_name
279
+ shutil.copytree(psr_dir, dest, dirs_exist_ok=True)
280
+ par_candidates = source.par_candidates(dest, psr_name)
281
+ tim_candidates = source.tim_candidates(dest, psr_name)
282
+ # INCLUDEd per-backend files hold the TOAs, so normalise every one.
283
+ staged_tims = sorted(dest.rglob("*.tim"))
284
+ else:
285
+ # Flat layout: no per-pulsar directory, so copy the matched files individually.
286
+ par_candidates = [_copy_into(p, outdir) for p in source.par_candidates(source.release_root, psr_name)]
287
+ tim_candidates = [_copy_into(p, outdir) for p in source.tim_candidates(source.release_root, psr_name)]
288
+ staged_tims = tim_candidates
289
+
290
+ manifest = {
291
+ "par": _select_par(par_candidates),
292
+ "par candidates": par_candidates,
293
+ "tim": tim_candidates,
294
+ "tim normalisation": normalise_tim_files(staged_tims),
295
+ }
296
+
297
+ clock_dir = source.clock_dir()
298
+ if clock_dir is not None:
299
+ manifest["clock dir"] = clock_dir
300
+
301
+ return manifest