xmatcher 0.5.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- xmatcher-0.5.1/.gitignore +70 -0
- xmatcher-0.5.1/.pre-commit-config.yaml +41 -0
- xmatcher-0.5.1/CHANGELOG.md +116 -0
- xmatcher-0.5.1/CONTRIBUTING.md +62 -0
- xmatcher-0.5.1/LICENSE +21 -0
- xmatcher-0.5.1/MANIFEST.in +5 -0
- xmatcher-0.5.1/PKG-INFO +207 -0
- xmatcher-0.5.1/README.md +154 -0
- xmatcher-0.5.1/docs/algorithms.md +262 -0
- xmatcher-0.5.1/docs/api.md +415 -0
- xmatcher-0.5.1/docs/association-v1.md +229 -0
- xmatcher-0.5.1/docs/canfar-scaling-2026-10-07.csv +33 -0
- xmatcher-0.5.1/docs/canfar-scaling-2026-10-07.md +30 -0
- xmatcher-0.5.1/docs/canfar-scaling-2026-10-07.png +0 -0
- xmatcher-0.5.1/docs/catalogue-pipeline.md +142 -0
- xmatcher-0.5.1/docs/observations.md +191 -0
- xmatcher-0.5.1/docs/release-0.5.0.md +37 -0
- xmatcher-0.5.1/docs/release-0.5.1.md +95 -0
- xmatcher-0.5.1/docs/release-audit-2026-10-06.md +286 -0
- xmatcher-0.5.1/docs/usage.md +626 -0
- xmatcher-0.5.1/pixi.lock +8619 -0
- xmatcher-0.5.1/pixi.toml +35 -0
- xmatcher-0.5.1/pyproject.toml +100 -0
- xmatcher-0.5.1/scripts/analyse_canfar_scaling.py +218 -0
- xmatcher-0.5.1/scripts/bench_engines.py +252 -0
- xmatcher-0.5.1/scripts/bench_ray_union.py +149 -0
- xmatcher-0.5.1/scripts/bench_ray_union_cap.py +161 -0
- xmatcher-0.5.1/scripts/canfar-ray-job.sh +241 -0
- xmatcher-0.5.1/scripts/canfar-smoke.sh +128 -0
- xmatcher-0.5.1/scripts/check_release.py +251 -0
- xmatcher-0.5.1/scripts/install_stilts.sh +57 -0
- xmatcher-0.5.1/setup.cfg +4 -0
- xmatcher-0.5.1/src/xmatcher/__init__.py +78 -0
- xmatcher-0.5.1/src/xmatcher/__main__.py +6 -0
- xmatcher-0.5.1/src/xmatcher/association.py +2410 -0
- xmatcher-0.5.1/src/xmatcher/astro_utils.py +446 -0
- xmatcher-0.5.1/src/xmatcher/auth.py +70 -0
- xmatcher-0.5.1/src/xmatcher/bayes.py +395 -0
- xmatcher-0.5.1/src/xmatcher/candidates.py +562 -0
- xmatcher-0.5.1/src/xmatcher/cli.py +2729 -0
- xmatcher-0.5.1/src/xmatcher/crossmatch.py +2656 -0
- xmatcher-0.5.1/src/xmatcher/discovery.py +385 -0
- xmatcher-0.5.1/src/xmatcher/exceptions.py +50 -0
- xmatcher-0.5.1/src/xmatcher/fixtures/association_v1.json +126 -0
- xmatcher-0.5.1/src/xmatcher/hats_native.py +448 -0
- xmatcher-0.5.1/src/xmatcher/io_utils.py +407 -0
- xmatcher-0.5.1/src/xmatcher/matchers.py +3605 -0
- xmatcher-0.5.1/src/xmatcher/mirror.py +1985 -0
- xmatcher-0.5.1/src/xmatcher/observations.py +652 -0
- xmatcher-0.5.1/src/xmatcher/out_of_core.py +909 -0
- xmatcher-0.5.1/src/xmatcher/ray_engine.py +456 -0
- xmatcher-0.5.1/src/xmatcher/ray_union.py +1873 -0
- xmatcher-0.5.1/src/xmatcher/remote_cds.py +163 -0
- xmatcher-0.5.1/src/xmatcher/remote_tap.py +223 -0
- xmatcher-0.5.1/src/xmatcher/request.py +315 -0
- xmatcher-0.5.1/src/xmatcher/sources.py +142 -0
- xmatcher-0.5.1/src/xmatcher/stilts.py +257 -0
- xmatcher-0.5.1/src/xmatcher/storage.py +549 -0
- xmatcher-0.5.1/src/xmatcher/tap.py +102 -0
- xmatcher-0.5.1/src/xmatcher/user_config.py +192 -0
- xmatcher-0.5.1/src/xmatcher/xmatcher.yaml +847 -0
- xmatcher-0.5.1/src/xmatcher.egg-info/PKG-INFO +207 -0
- xmatcher-0.5.1/src/xmatcher.egg-info/SOURCES.txt +111 -0
- xmatcher-0.5.1/src/xmatcher.egg-info/dependency_links.txt +1 -0
- xmatcher-0.5.1/src/xmatcher.egg-info/entry_points.txt +2 -0
- xmatcher-0.5.1/src/xmatcher.egg-info/requires.txt +40 -0
- xmatcher-0.5.1/src/xmatcher.egg-info/top_level.txt +1 -0
- xmatcher-0.5.1/tests/__init__.py +1 -0
- xmatcher-0.5.1/tests/conftest.py +113 -0
- xmatcher-0.5.1/tests/inputs/id_cat1.csv +4 -0
- xmatcher-0.5.1/tests/inputs/id_cat2.csv +4 -0
- xmatcher-0.5.1/tests/inputs/my_sources.csv +4 -0
- xmatcher-0.5.1/tests/inputs/simple_sources.csv +3 -0
- xmatcher-0.5.1/tests/inputs/test_cds_input.csv +4 -0
- xmatcher-0.5.1/tests/inputs/test_coord_override_2.csv +5 -0
- xmatcher-0.5.1/tests/inputs/test_id_join_1.csv +5 -0
- xmatcher-0.5.1/tests/inputs/test_id_join_2.csv +6 -0
- xmatcher-0.5.1/tests/inputs/test_local_vs_remote_id_1.csv +4 -0
- xmatcher-0.5.1/tests/tap_fake.py +245 -0
- xmatcher-0.5.1/tests/test_association_components.py +345 -0
- xmatcher-0.5.1/tests/test_association_contract.py +64 -0
- xmatcher-0.5.1/tests/test_association_deltas.py +337 -0
- xmatcher-0.5.1/tests/test_association_equivalence.py +323 -0
- xmatcher-0.5.1/tests/test_association_release.py +177 -0
- xmatcher-0.5.1/tests/test_astro_utils.py +277 -0
- xmatcher-0.5.1/tests/test_benchmark_checks.py +72 -0
- xmatcher-0.5.1/tests/test_benchmarks.py +316 -0
- xmatcher-0.5.1/tests/test_candidate_pipeline.py +413 -0
- xmatcher-0.5.1/tests/test_canfar_ray_job.py +106 -0
- xmatcher-0.5.1/tests/test_canfar_smoke.py +34 -0
- xmatcher-0.5.1/tests/test_check_release.py +208 -0
- xmatcher-0.5.1/tests/test_cli.py +719 -0
- xmatcher-0.5.1/tests/test_cli_sync.py +207 -0
- xmatcher-0.5.1/tests/test_crossmatch.py +1098 -0
- xmatcher-0.5.1/tests/test_distributed_regressions.py +879 -0
- xmatcher-0.5.1/tests/test_epoch_spill.py +496 -0
- xmatcher-0.5.1/tests/test_fallback_policy.py +225 -0
- xmatcher-0.5.1/tests/test_hats.py +128 -0
- xmatcher-0.5.1/tests/test_hats_native.py +346 -0
- xmatcher-0.5.1/tests/test_id_join_streaming.py +149 -0
- xmatcher-0.5.1/tests/test_io_utils.py +383 -0
- xmatcher-0.5.1/tests/test_large_crossmatch.py +454 -0
- xmatcher-0.5.1/tests/test_matchers.py +3488 -0
- xmatcher-0.5.1/tests/test_mirror_sync.py +1024 -0
- xmatcher-0.5.1/tests/test_observations.py +483 -0
- xmatcher-0.5.1/tests/test_out_of_core.py +377 -0
- xmatcher-0.5.1/tests/test_ray_union.py +1201 -0
- xmatcher-0.5.1/tests/test_real_catalogues.py +570 -0
- xmatcher-0.5.1/tests/test_remote.py +562 -0
- xmatcher-0.5.1/tests/test_storage.py +562 -0
- xmatcher-0.5.1/tests/test_tap.py +57 -0
- xmatcher-0.5.1/tests/test_torchsky_engine.py +242 -0
- xmatcher-0.5.1/tests/test_user_config.py +323 -0
|
@@ -0,0 +1,70 @@
|
|
|
1
|
+
# Byte-compiled / optimized / DLL files
|
|
2
|
+
__pycache__/
|
|
3
|
+
*.py[cod]
|
|
4
|
+
*$py.class
|
|
5
|
+
|
|
6
|
+
# C extensions
|
|
7
|
+
*.so
|
|
8
|
+
|
|
9
|
+
# Distribution / packaging
|
|
10
|
+
dist/
|
|
11
|
+
build/
|
|
12
|
+
*.egg-info/
|
|
13
|
+
*.egg
|
|
14
|
+
|
|
15
|
+
# Virtual environments
|
|
16
|
+
venv/
|
|
17
|
+
env/
|
|
18
|
+
ENV/
|
|
19
|
+
.env
|
|
20
|
+
.venv
|
|
21
|
+
env.bak/
|
|
22
|
+
venv.bak/
|
|
23
|
+
|
|
24
|
+
# Unit test / coverage reports
|
|
25
|
+
htmlcov/
|
|
26
|
+
.tox/
|
|
27
|
+
.coverage
|
|
28
|
+
.coverage.*
|
|
29
|
+
.cache
|
|
30
|
+
nosetests.xml
|
|
31
|
+
coverage.xml
|
|
32
|
+
*.cover
|
|
33
|
+
.hypothesis/
|
|
34
|
+
.pytest_cache/
|
|
35
|
+
|
|
36
|
+
# Jupyter Notebook
|
|
37
|
+
.ipynb_checkpoints
|
|
38
|
+
|
|
39
|
+
# PyTorch specific
|
|
40
|
+
*.pt
|
|
41
|
+
*.pth
|
|
42
|
+
*.onnx
|
|
43
|
+
*.h5
|
|
44
|
+
checkpoint/
|
|
45
|
+
runs/
|
|
46
|
+
logs/
|
|
47
|
+
tensorboard/
|
|
48
|
+
lightning_logs/
|
|
49
|
+
|
|
50
|
+
# IDE specific files
|
|
51
|
+
.idea/
|
|
52
|
+
.vscode/
|
|
53
|
+
*.swp
|
|
54
|
+
*.swo
|
|
55
|
+
.DS_Store
|
|
56
|
+
|
|
57
|
+
# Project specific
|
|
58
|
+
data/
|
|
59
|
+
output/
|
|
60
|
+
results/
|
|
61
|
+
|
|
62
|
+
# Ignore test and benchmark output files
|
|
63
|
+
tests/outputs/
|
|
64
|
+
benchmark_results/
|
|
65
|
+
cache/
|
|
66
|
+
|
|
67
|
+
# Tool and package manager caches
|
|
68
|
+
.pixi/
|
|
69
|
+
.mypy_cache/
|
|
70
|
+
.ruff_cache/
|
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
default_install_hook_types: [pre-commit, pre-push]
|
|
2
|
+
|
|
3
|
+
repos:
|
|
4
|
+
- repo: https://github.com/pre-commit/pre-commit-hooks
|
|
5
|
+
rev: v5.0.0
|
|
6
|
+
hooks:
|
|
7
|
+
- id: trailing-whitespace
|
|
8
|
+
- id: end-of-file-fixer
|
|
9
|
+
- id: check-yaml
|
|
10
|
+
- id: check-toml
|
|
11
|
+
- id: check-added-large-files
|
|
12
|
+
- id: check-merge-conflict
|
|
13
|
+
- id: debug-statements
|
|
14
|
+
|
|
15
|
+
- repo: https://github.com/astral-sh/ruff-pre-commit
|
|
16
|
+
rev: v0.16.10
|
|
17
|
+
hooks:
|
|
18
|
+
- id: ruff-check
|
|
19
|
+
- id: ruff-format
|
|
20
|
+
|
|
21
|
+
- repo: https://github.com/codespell-project/codespell
|
|
22
|
+
rev: v2.4.3
|
|
23
|
+
hooks:
|
|
24
|
+
- id: codespell
|
|
25
|
+
|
|
26
|
+
- repo: local
|
|
27
|
+
hooks:
|
|
28
|
+
- id: mypy
|
|
29
|
+
name: type checks in the locked scientific environment
|
|
30
|
+
entry: pixi run --as-is mypy src/xmatcher tests scripts
|
|
31
|
+
language: system
|
|
32
|
+
pass_filenames: false
|
|
33
|
+
always_run: true
|
|
34
|
+
- id: ci-local-pre-push
|
|
35
|
+
name: offline test gate
|
|
36
|
+
entry: pixi run ci-local
|
|
37
|
+
language: system
|
|
38
|
+
pass_filenames: false
|
|
39
|
+
always_run: true
|
|
40
|
+
require_serial: true
|
|
41
|
+
stages: [pre-push]
|
|
@@ -0,0 +1,116 @@
|
|
|
1
|
+
# Changelog
|
|
2
|
+
|
|
3
|
+
All notable changes to `xmatcher` are documented in this file.
|
|
4
|
+
|
|
5
|
+
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
|
|
6
|
+
and this project adheres to [Semantic Versioning](https://semver.org/).
|
|
7
|
+
|
|
8
|
+
## [Unreleased]
|
|
9
|
+
|
|
10
|
+
## [0.5.1] — 2026-10-08
|
|
11
|
+
|
|
12
|
+
Published on PyPI as `xmatcher`. The `v0.5.1` tag is the GitHub release.
|
|
13
|
+
|
|
14
|
+
### Changed
|
|
15
|
+
|
|
16
|
+
- **Breaking package and command rename**: Python imports and the CLI move from
|
|
17
|
+
`xmatch` to `xmatcher`.
|
|
18
|
+
- **Breaking configuration rename**: the bundled and user configuration file
|
|
19
|
+
becomes `xmatcher.yaml`; environment variables move from the `XMATCH_` prefix
|
|
20
|
+
to `XMATCHER_`.
|
|
21
|
+
- **Changed local paths**: the default cache moves to `~/.cache/xmatcher`, and
|
|
22
|
+
the documented CANFAR output root is `/arc/projects/hats/xmatcher`.
|
|
23
|
+
- The persisted `xmatch.*` association schema identifiers remain unchanged.
|
|
24
|
+
|
|
25
|
+
### Fixed
|
|
26
|
+
|
|
27
|
+
- FITS writes preserve explicit masks, including the distinction between empty
|
|
28
|
+
strings and nulls. Reads fall back to Astropy when the fast reader could conceal masked
|
|
29
|
+
values. CDS XMatch now uses a collision-free local row key, preserving input
|
|
30
|
+
columns with the previous internal key name.
|
|
31
|
+
- Catalogue names resolve case-insensitively to their configured spelling;
|
|
32
|
+
configurations with names that differ only by case are rejected. CLI table
|
|
33
|
+
discovery and schema queries use configured archive authentication, and
|
|
34
|
+
invalid CLI configuration is reported as a command error.
|
|
35
|
+
- HTTP HATS mirroring distinguishes complete listings from failed lookups,
|
|
36
|
+
includes partition metadata, and removes stale cached files only after a
|
|
37
|
+
successful refresh. A stale file that cannot be removed fails the sync and
|
|
38
|
+
keeps its manifest entry, so the next sync retries the removal.
|
|
39
|
+
- Astrometric error floors and drift inflation preserve measured cross
|
|
40
|
+
covariance for ellipse matching, and singular covariance pairs are rejected.
|
|
41
|
+
Astropy uses the shared ellipse filter; separation-based matcher fallbacks
|
|
42
|
+
retain every candidate for `find="all"`.
|
|
43
|
+
- Post-match filters apply the same row selection to matcher scores and respect
|
|
44
|
+
the requested right-column suffix. Candidate normalization checks pair
|
|
45
|
+
endpoint namespaces against the inventory, rejects contradictory target
|
|
46
|
+
namespace evidence, and allows an explicitly named empty target population.
|
|
47
|
+
Candidate-release setup failures also clean up their lock so publication can
|
|
48
|
+
be retried.
|
|
49
|
+
- HTTP HATS refreshes publish metadata after successful data transfers and use
|
|
50
|
+
a one-byte range probe when HEAD is unsupported. HTML listing fallback retains
|
|
51
|
+
CSV partition metadata and empty-catalogue schemas. VOS replica replacement
|
|
52
|
+
preserves nested paths and existing files when backup reads fail. TAP row
|
|
53
|
+
limits include short final pages.
|
|
54
|
+
- Native HATS reads and distributed unions retain the schema of valid empty
|
|
55
|
+
catalogues while rejecting missing partitions from nonempty catalogues.
|
|
56
|
+
CANFAR scaling validation requires the observed worker count to match the
|
|
57
|
+
requested count.
|
|
58
|
+
|
|
59
|
+
## [0.5.0] — 2026-10-07
|
|
60
|
+
|
|
61
|
+
Available from the public [GitHub release at tag `v0.5.0`](https://github.com/astroai/xmatch/releases/tag/v0.5.0).
|
|
62
|
+
No PyPI package is published because another project owns the `xmatch` name.
|
|
63
|
+
|
|
64
|
+
### Added
|
|
65
|
+
|
|
66
|
+
- **Source-Preserving Observation & Candidate Releases** (`observations.py`, `candidates.py`):
|
|
67
|
+
- Publish source inventories with normalized photometry and property-evidence ledgers in immutable, checksummed Parquet releases (`write_observation_release`, `verify_observation_release`).
|
|
68
|
+
- Publish scoped sparse candidate-pair releases (`write_candidate_release`, `verify_candidate_release`) and normalize caller-supplied per-target hypothesis weights with an explicit no-match alternative (`normalize_candidate_hypotheses`).
|
|
69
|
+
- **Versioned Association Contract** (`association.py`):
|
|
70
|
+
- Deterministic `xmatch.association.v1` pair records, atomic release directories, component sidecar releases (`xmatch.association.component.release.v1`), component delta classifications (`created`, `continued`, `merged`, `split`, `retired`), and bijective member equivalence crosswalks.
|
|
71
|
+
- **HATS Mirroring & Distributed Full-Sky Union** (`mirror.py`, `ray_union.py`, `storage.py`):
|
|
72
|
+
- `xmatch sync` CLI command and `sync_catalogue` / `ensure_mirrored` APIs to mirror remote TAP and HATS catalogues (over HTTP or CANFAR `vos:`) into a local HATS cache with keyset paging, token-bucket rate limiting, endpoint failover, and resumable checkpoints.
|
|
73
|
+
- `engine="ray-union"`: distributed $N$-survey full-outer join over mirrored HATS inputs on Ray, producing partitioned HATS trees (`Norder=*/Dir=*/Npix=*.parquet`) with chunk-level resume state.
|
|
74
|
+
- `ray_engine.py`: parallel HEALPix pixel-batch cone crossmatching (`engine="ray"`).
|
|
75
|
+
- **Astrometric Kinematics & Probabilistic Matchers** (`matchers.py`, `out_of_core.py`, `bayes.py`):
|
|
76
|
+
- Six-dimensional epoch propagation (`target_epoch`) and 5-parameter astrometric covariance transport through motion Jacobians for both `skyellipse` and `skyerr` (eager and out-of-core spill paths).
|
|
77
|
+
- Wilson (2023)-inspired proper-motion drift uncertainty model (`pm_prior=True`) with per-row error quadrature inflation and optional magnitude refinement.
|
|
78
|
+
- Sutherland & Saunders (1992) Likelihood Ratio (`matcher="lr"`), Random Forest (`matcher="ml"`), XGBoost/LightGBM (`matcher="xgb"`), lightweight AUF-inspired empirical separation/background scoring (`matcher="auf"`), and a flux-augmented heuristic (`matcher="macauff"`); the latter two are not full reproductions of the published methods.
|
|
79
|
+
- Multi-condition SQL post-filtering (`MatchSpec.filter_expr`), feature-weighted ranking of spatial candidates (`MatchSpec.extra_distance_cols`), and HEALPix pixel-group batching (`MatchSpec.batch_size`; this is not a general process-memory bound).
|
|
80
|
+
- **Formats & Catalogues** (`io_utils.py`, `xmatch.yaml`):
|
|
81
|
+
- Native streaming `.tsv` and `.tab` reads and writes via Polars.
|
|
82
|
+
- Added NOIRLab Data Lab catalogues (`nsc`, `des`, `decals`, `smash`, `unwise`, `allwise_dl`, and pre-computed `*_x_gaia` crossmatch tables) and COSMOS (`cosmos-web`, `zcosmos`).
|
|
83
|
+
- Terminal progress indicator for remote TAP and VizieR downloads.
|
|
84
|
+
|
|
85
|
+
### Fixed
|
|
86
|
+
|
|
87
|
+
- Distributed HATS union now keeps output partitions spatially coherent across mixed input orders and RING/NESTED layouts, and reports non-null union routing coordinates. Tuple caps select the lowest-separation results from the complete candidate product; planner scans account for declared positional uncertainty and epoch motion.
|
|
88
|
+
- Friends-of-Friends matching searches cross-catalogue secondary links while retaining the primary-anchored component output policy.
|
|
89
|
+
- Probabilistic scores require declared positional errors or explicit error floors; the documentation states the equal-prior and isotropic-score assumptions. Remote `skyerr`, `skyellipse`, and `target_epoch` queries require an explicit search cone when service-side uncertainty or motion margins cannot be bounded.
|
|
90
|
+
- Invalid `filter_expr` SQL and missing, nonnumeric, or nonfinite declared ranking features now raise `CrossMatchError` instead of silently retaining rows or dropping/imputing features.
|
|
91
|
+
- ML synthetic negative sampling uses a local fixed seed, making repeated inputs reproducible without advancing the caller's NumPy random state.
|
|
92
|
+
- STILTS per-primary `find="best"` now uses `best1`, avoiding the symmetric one-to-one selection performed by STILTS `best`.
|
|
93
|
+
- An unreachable explicit Ray address no longer silently starts a local cluster under strict fallback policy, and the caller's address setting is preserved. Memory and storage budgets documented in GiB now use binary GiB.
|
|
94
|
+
|
|
95
|
+
### Changed
|
|
96
|
+
|
|
97
|
+
- Updated the Python requirement to $\ge 3.13$ and modernized type annotations to PEP 585 / PEP 604 built-ins. The lockfile targets Python 3.13. A targeted Python 3.14.8 / Polars 2 compatibility run passed 206 tests and skipped 24; it was not the complete release gate on that stack.
|
|
98
|
+
- Streamlined core runtime dependencies to `numpy`, `polars`, `pyarrow`, `astropy`, `scipy`, `pyyaml`, and `pyvo`, with direct PyArrow/NumPy `Table`/`MaskedColumn` conversions and optional `xmatch[cds]` extra for `astroquery`.
|
|
99
|
+
- Consolidated HATS reading and writing on the native HATS engine (`hats` + `cdshealpix`). Ray-union uses compressed interval planning and measured per-source halos for adaptive/mixed-order and RING inputs and uncertainty- or epoch-adjusted searches; it may scan additional partitions. The pairwise `hats_native` fast path remains fixed-radius, uniform-order NESTED and globally materializes other layouts or error/epoch-adjusted inputs.
|
|
100
|
+
- Removed the unusable `torchsky` package extra because that distribution is not published on PyPI. The tensor engine requires a separately installed compatible Torchsky checkout; the documented integration was tested with the sibling 0.4 development source installed editable and Torchfits 1.0.0 from PyPI.
|
|
101
|
+
|
|
102
|
+
---
|
|
103
|
+
|
|
104
|
+
## [0.4.0] — 2026-06-15
|
|
105
|
+
|
|
106
|
+
### Added
|
|
107
|
+
|
|
108
|
+
- Typed request API (`MatchRequest`, `SideOverrides`, `CrossMatch.crossmatch_request`).
|
|
109
|
+
- Multi-catalogue sequential crossmatching (`CrossMatch.crossmatch_multi`) and CLI support for $N \ge 3$ catalogues.
|
|
110
|
+
- Remote archive discovery module (`xmatch.discovery`) and CLI subcommands (`search`, `discover`, `adopt`, `list`, `describe`).
|
|
111
|
+
- Lazy bounding-cone estimation (`astro_utils.sky_extent_from_frame`) and streaming Parquet/CSV output sinks (`io_utils.write_frame`).
|
|
112
|
+
|
|
113
|
+
### Changed
|
|
114
|
+
|
|
115
|
+
- Switched Astropy–Polars table conversions (`astropy_table_to_polars`, `polars_to_astropy`) to Apache Arrow column buffers.
|
|
116
|
+
- Quoted ADQL identifiers in remote TAP queries to handle reserved words and special characters safely.
|
|
@@ -0,0 +1,62 @@
|
|
|
1
|
+
# Contributing to xmatcher
|
|
2
|
+
|
|
3
|
+
Thank you for contributing to `xmatcher`!
|
|
4
|
+
|
|
5
|
+
Before publishing, build the wheel and source archive with `uv build`, then run
|
|
6
|
+
`pixi run python scripts/check_release.py`. This verifies current source bytes,
|
|
7
|
+
package data, metadata, and the fixtures/helpers needed to run the archived tests.
|
|
8
|
+
|
|
9
|
+
## Development Setup
|
|
10
|
+
|
|
11
|
+
We recommend [Pixi](https://pixi.sh) for a reproducible environment with all compiled dependencies (`polars`, `scipy`, `astropy`, `cdshealpix`, `ray`):
|
|
12
|
+
|
|
13
|
+
```bash
|
|
14
|
+
git clone https://github.com/astroai/xmatcher.git
|
|
15
|
+
cd xmatcher
|
|
16
|
+
pixi install
|
|
17
|
+
```
|
|
18
|
+
|
|
19
|
+
On shared or NFS-mounted systems (such as CANFAR), isolate user site-packages before running Python or tests:
|
|
20
|
+
|
|
21
|
+
```bash
|
|
22
|
+
export PYTHONNOUSERSITE=1
|
|
23
|
+
unset PYTHONPATH
|
|
24
|
+
```
|
|
25
|
+
|
|
26
|
+
## Linting, Formatting, and Tests
|
|
27
|
+
|
|
28
|
+
Run the fast preflight after a small change. It checks Ruff lint, formatting, Mypy, and bytecode compilation:
|
|
29
|
+
|
|
30
|
+
```bash
|
|
31
|
+
pixi run preflight-push
|
|
32
|
+
```
|
|
33
|
+
|
|
34
|
+
Auto-format and lint with Ruff:
|
|
35
|
+
|
|
36
|
+
```bash
|
|
37
|
+
pixi run format
|
|
38
|
+
pixi run lint
|
|
39
|
+
```
|
|
40
|
+
|
|
41
|
+
Run the release gate before opening a pull request. It runs the preflight and all tests not marked `slow` or `bench`:
|
|
42
|
+
|
|
43
|
+
```bash
|
|
44
|
+
pixi run ci-local
|
|
45
|
+
```
|
|
46
|
+
|
|
47
|
+
Run targeted tests while developing, or select the full suite explicitly when you need slow and benchmark tests:
|
|
48
|
+
|
|
49
|
+
```bash
|
|
50
|
+
# Targeted module tests
|
|
51
|
+
pixi run test tests/test_matchers.py -q
|
|
52
|
+
|
|
53
|
+
# Entire pytest suite, including tests excluded from ci-local
|
|
54
|
+
pixi run test
|
|
55
|
+
```
|
|
56
|
+
|
|
57
|
+
The offline `ci-local` gate does not establish that optional integrations, remote services, credentials, or every Python version work. Run the relevant integration checks separately when changing those paths. The checked Pixi environment uses Python 3.13.
|
|
58
|
+
|
|
59
|
+
## Pull Requests
|
|
60
|
+
|
|
61
|
+
- Follow [Conventional Commits](https://www.conventionalcommits.org/) (`feat:`, `fix:`, `docs:`, `perf:`, `refactor:`, `test:`).
|
|
62
|
+
- Ensure `pixi run ci-local` and any relevant optional integration checks pass before opening a pull request against `astroai/xmatcher`.
|
xmatcher-0.5.1/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Sébastien Fabbro
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
xmatcher-0.5.1/PKG-INFO
ADDED
|
@@ -0,0 +1,207 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: xmatcher
|
|
3
|
+
Version: 0.5.1
|
|
4
|
+
Summary: High-performance, format-agnostic astronomical catalogue crossmatching and multi-survey union toolkit.
|
|
5
|
+
Author-email: Seb Fabbro <sebfabbro@gmail.com>
|
|
6
|
+
License-Expression: MIT
|
|
7
|
+
Project-URL: Homepage, https://github.com/astroai/xmatcher
|
|
8
|
+
Project-URL: Repository, https://github.com/astroai/xmatcher
|
|
9
|
+
Project-URL: Documentation, https://github.com/astroai/xmatcher/tree/main/docs
|
|
10
|
+
Project-URL: Issues, https://github.com/astroai/xmatcher/issues
|
|
11
|
+
Keywords: astronomy,crossmatch,catalogue,hats,healpix,polars,tap
|
|
12
|
+
Classifier: Development Status :: 4 - Beta
|
|
13
|
+
Classifier: Intended Audience :: Science/Research
|
|
14
|
+
Classifier: Programming Language :: Python :: 3
|
|
15
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
16
|
+
Classifier: Topic :: Scientific/Engineering :: Astronomy
|
|
17
|
+
Requires-Python: >=3.13
|
|
18
|
+
Description-Content-Type: text/markdown
|
|
19
|
+
License-File: LICENSE
|
|
20
|
+
Requires-Dist: numpy>=2.4
|
|
21
|
+
Requires-Dist: polars>=1.40
|
|
22
|
+
Requires-Dist: pyarrow>=25.0
|
|
23
|
+
Requires-Dist: astropy>=7.2
|
|
24
|
+
Requires-Dist: scipy>=1.18
|
|
25
|
+
Requires-Dist: pyyaml>=6.0.3
|
|
26
|
+
Requires-Dist: pyvo>=1.8
|
|
27
|
+
Provides-Extra: cds
|
|
28
|
+
Requires-Dist: astroquery>=0.4.11; extra == "cds"
|
|
29
|
+
Provides-Extra: torchfits
|
|
30
|
+
Requires-Dist: torchfits>=1.0.0rc4; extra == "torchfits"
|
|
31
|
+
Provides-Extra: zone
|
|
32
|
+
Requires-Dist: cdshealpix>=0.8; extra == "zone"
|
|
33
|
+
Provides-Extra: hats
|
|
34
|
+
Requires-Dist: hats>=0.7; extra == "hats"
|
|
35
|
+
Requires-Dist: cdshealpix>=0.8; extra == "hats"
|
|
36
|
+
Provides-Extra: hats-ray
|
|
37
|
+
Requires-Dist: hats>=0.7; extra == "hats-ray"
|
|
38
|
+
Requires-Dist: ray>=2.50; extra == "hats-ray"
|
|
39
|
+
Requires-Dist: cdshealpix>=0.8; extra == "hats-ray"
|
|
40
|
+
Provides-Extra: ray
|
|
41
|
+
Requires-Dist: ray>=2.50; extra == "ray"
|
|
42
|
+
Requires-Dist: cdshealpix>=0.8; extra == "ray"
|
|
43
|
+
Provides-Extra: ml
|
|
44
|
+
Requires-Dist: scikit-learn; extra == "ml"
|
|
45
|
+
Requires-Dist: xgboost; extra == "ml"
|
|
46
|
+
Requires-Dist: lightgbm; extra == "ml"
|
|
47
|
+
Requires-Dist: joblib; extra == "ml"
|
|
48
|
+
Provides-Extra: dev
|
|
49
|
+
Requires-Dist: pytest>=9.0; extra == "dev"
|
|
50
|
+
Requires-Dist: ruff>=0.15; extra == "dev"
|
|
51
|
+
Requires-Dist: mypy>=2.0; extra == "dev"
|
|
52
|
+
Dynamic: license-file
|
|
53
|
+
|
|
54
|
+
# xmatcher
|
|
55
|
+
|
|
56
|
+
`xmatcher` matches and joins astronomical catalogues from local files, HATS trees, and configured remote archives. Matching features depend on the selected engine and the metadata supplied for each catalogue; the guides document those limits and the memory trade-offs.
|
|
57
|
+
|
|
58
|
+
Version 0.5.1 is the PyPI release. The
|
|
59
|
+
[v0.5.0](https://github.com/astroai/xmatch/releases/tag/v0.5.0) GitHub release
|
|
60
|
+
keeps the old `xmatch` name. See the
|
|
61
|
+
[migration notes](docs/release-0.5.1.md).
|
|
62
|
+
|
|
63
|
+
```bash
|
|
64
|
+
xmatcher match catalog1.parquet catalog2.csv -o matches.parquet -r 1.0
|
|
65
|
+
```
|
|
66
|
+
|
|
67
|
+
Inputs can be local Parquet, CSV, TSV, or FITS files, HATS directories, Polars frames, or configured remote catalogues. Coordinate names can be configured or detected. Proper-motion propagation uses declared epoch and motion metadata; mixed coordinate frames are rejected and generic catalogue discovery does not invent positional uncertainties.
|
|
68
|
+
|
|
69
|
+
---
|
|
70
|
+
|
|
71
|
+
## 📚 Complete Documentation Suite
|
|
72
|
+
|
|
73
|
+
All detailed architectural references and user guides are organized in the [`docs/`](docs/) directory:
|
|
74
|
+
|
|
75
|
+
- **[User Guide & Cookbooks (`docs/usage.md`)](docs/usage.md)**: Installation, local and remote matching, explicit memory controls, and distributed workflows with their limits.
|
|
76
|
+
- **[Public API Reference (`docs/api.md`)](docs/api.md)**: Full Python API specifications for `CrossMatch`, `MatchRequest`, `MatchSpec`, observation/candidate release helpers, engines, storage abstractions, and exception hierarchies.
|
|
77
|
+
- **[Crossmatch Algorithms & Math (`docs/algorithms.md`)](docs/algorithms.md)**: Matcher definitions, uncertainty assumptions, score semantics, and engine behavior.
|
|
78
|
+
- **[Source & Measurement Releases (`docs/observations.md`)](docs/observations.md)**: Lazy, source-preserving measurement ledgers, photometric unit/limit normalization, property evidence, and checksummed Parquet releases.
|
|
79
|
+
- **[Candidate & Association Pipeline (`docs/catalogue-pipeline.md`)](docs/catalogue-pipeline.md)**: End-to-end workflow from pinned survey observation and sparse-candidate releases to per-target association hypotheses.
|
|
80
|
+
- **[Association Contract v1 (`docs/association-v1.md`)](docs/association-v1.md)**: Audit-grade versioned association record specification, release directories, component sidecars, and member equivalence tracking.
|
|
81
|
+
|
|
82
|
+
---
|
|
83
|
+
|
|
84
|
+
## Capabilities
|
|
85
|
+
|
|
86
|
+
- **Universal Multi-Format I/O**:
|
|
87
|
+
- Ingests and streams `.parquet`, `.csv`, `.tsv`, `.tab`, `.fits` (via high-speed Torchfits with Astropy fallback), and HATS.
|
|
88
|
+
- Remote storage connectors: POSIX filesystem, CANFAR VOSpace (`vos:`), HTTP/HTTPS, and S3.
|
|
89
|
+
- **File I/O and explicit memory controls**:
|
|
90
|
+
- Parquet, CSV, TSV, FITS, and HATS readers with Parquet/CSV/TSV/FITS outputs.
|
|
91
|
+
- Output sinks can stream serialization. Most matching engines still materialize inputs and matches; local pairwise CSV/Parquet requests can opt into the spill path with `--memory-budget-bytes` and file output. See [memory limits](docs/usage.md#pairwise-spill-matching-on-one-machine).
|
|
92
|
+
- **Astrometric and probabilistic matchers**:
|
|
93
|
+
- Simple radius (`sky`) and adaptive per-row astrometric uncertainties (`skyerr`).
|
|
94
|
+
- 2D Gaussian error ellipses via Mahalanobis distance (`skyellipse`).
|
|
95
|
+
- Proper-motion epoch propagation (`target_epoch`) when epoch and motion metadata are supplied, plus an explicit population drift model (`pm_prior`) for missing motion.
|
|
96
|
+
- Sutherland & Saunders (1992) Likelihood Ratio (`lr`) with magnitude background subtraction.
|
|
97
|
+
- Machine learning classifiers (`ml` Random Forest, `xgb` XGBoost/LightGBM) with self-match pseudo-labeling.
|
|
98
|
+
- AUF-inspired empirical separation/background scores (`auf`) and a flux-augmented heuristic (`macauff`). These are not full reproductions of the published methods and their scores are not calibrated probabilities.
|
|
99
|
+
- Bayesian positional qualification (`probabilistic=True`) or photometric qualification (`prior_columns`) with declared errors or explicit error floors, plus simultaneous $N$-way scores (`nway_match`). These scores use stated assumptions and are not population-calibrated probabilities.
|
|
100
|
+
- Friends-of-Friends transitive closure object bundles (`fof_match`).
|
|
101
|
+
- **Complete Relational Joins**:
|
|
102
|
+
- Inner (`1and2`), Left (`all1`), Right (`all2`), Full Outer (`1or2`, `all`), and Anti-joins (`1not2`, `2not1`).
|
|
103
|
+
- Relational ID joins on survey identifiers (`--id-join`).
|
|
104
|
+
- Multi-catalogue sequential chains (`crossmatch_multi`).
|
|
105
|
+
- **Local Mirroring & Offline Caching**:
|
|
106
|
+
- `xmatcher sync`: Mirror remote TAP and HATS surveys into a durable local HATS cache with keyset paging, rate limiting, and progress checkpoints.
|
|
107
|
+
- **Distributed execution**:
|
|
108
|
+
- `engine="ray"` parallelizes pairwise HEALPix candidate searches. The coordinator still holds the input frames and final result in memory.
|
|
109
|
+
- `engine="ray-union"` mirrors inputs into HATS and writes a full-sky, full-outer HATS union. It supports `sky`, `skyerr`, and `skyellipse`; compressed interval planning handles mixed-order/RING layouts and uncertainty or epoch halos in distributed tasks. Planning may scan additional partitions. Output trees add non-null `_union_ra` and `_union_dec` routing columns and declare NESTED ordering.
|
|
110
|
+
|
|
111
|
+
---
|
|
112
|
+
|
|
113
|
+
## Installation
|
|
114
|
+
|
|
115
|
+
Requires Python $\ge 3.13$. The checked Pixi environment and local release gate use Python 3.13.
|
|
116
|
+
|
|
117
|
+
### Recommended: Pixi
|
|
118
|
+
|
|
119
|
+
```bash
|
|
120
|
+
git clone https://github.com/astroai/xmatcher.git
|
|
121
|
+
cd xmatcher
|
|
122
|
+
|
|
123
|
+
# Install environment and dependencies
|
|
124
|
+
pixi install
|
|
125
|
+
|
|
126
|
+
# Run lint, formatting, bytecode, and offline test checks
|
|
127
|
+
pixi run preflight-push
|
|
128
|
+
pixi run ci-local
|
|
129
|
+
```
|
|
130
|
+
|
|
131
|
+
### Standard: Pip
|
|
132
|
+
|
|
133
|
+
```bash
|
|
134
|
+
python -m pip install "xmatcher==0.5.1"
|
|
135
|
+
|
|
136
|
+
# Optional integrations; use only the extras you need
|
|
137
|
+
python -m pip install "xmatcher[cds,hats-ray,torchfits,ml]==0.5.1"
|
|
138
|
+
```
|
|
139
|
+
|
|
140
|
+
A source checkout uses the same extras from the tree:
|
|
141
|
+
|
|
142
|
+
```bash
|
|
143
|
+
git clone https://github.com/astroai/xmatcher.git
|
|
144
|
+
cd xmatcher
|
|
145
|
+
python -m pip install ".[cds,hats-ray,torchfits,ml]"
|
|
146
|
+
```
|
|
147
|
+
|
|
148
|
+
The tensor-native `torchsky` engine is not a package extra: no `torchsky`
|
|
149
|
+
distribution is published on PyPI. To use it, install xmatcher alongside a
|
|
150
|
+
compatible Torchsky checkout. The integration was tested with the sibling
|
|
151
|
+
Torchsky 0.4 development source installed editable and Torchfits 1.0.0 from
|
|
152
|
+
PyPI.
|
|
153
|
+
|
|
154
|
+
---
|
|
155
|
+
|
|
156
|
+
## Command Line Quick Reference
|
|
157
|
+
|
|
158
|
+
| Task | Command |
|
|
159
|
+
|---|---|
|
|
160
|
+
| Match two local files | `xmatcher match cat1.parquet cat2.csv -o matches.parquet -r 1.5` |
|
|
161
|
+
| Match against remote Gaia DR3 | `xmatcher match targets.csv gaia -o targets_gaia.parquet -r 1.0` |
|
|
162
|
+
| Astrometric error ellipse match | `xmatcher match xray.fits optical.parquet --matcher skyellipse --max-error 3.0` |
|
|
163
|
+
| Proper-motion drift prior match | `xmatcher match old_survey.csv gaia --target-epoch 2016.0 --pm-prior -r 2.0` |
|
|
164
|
+
| Relational ID join | `xmatcher match cat1.parquet cat2.parquet --id-join --id1 objid --id2 objid` |
|
|
165
|
+
| Multi-survey 3-way chain | `xmatcher match gaia wise.csv 2mass.parquet -r 1.5 -o 3way.parquet` |
|
|
166
|
+
| Distributed HATS master union | `xmatcher match gaia allwise 2mass des --union --engine ray-union -o /data/master.hats` |
|
|
167
|
+
| Mirror remote surveys to local cache | `xmatcher sync gaia allwise 2mass` |
|
|
168
|
+
| Search remote TAP archives | `xmatcher search des` |
|
|
169
|
+
| Discover archive schemas | `xmatcher discover noirlab --schema nsc_dr2.object` |
|
|
170
|
+
|
|
171
|
+
---
|
|
172
|
+
|
|
173
|
+
## Python API Quickstart
|
|
174
|
+
|
|
175
|
+
```python
|
|
176
|
+
from xmatcher import CrossMatch, MatchRequest, MatchSpec
|
|
177
|
+
|
|
178
|
+
cm = CrossMatch()
|
|
179
|
+
|
|
180
|
+
# 1. Preferred typed request interface
|
|
181
|
+
req = MatchRequest(
|
|
182
|
+
cat1="optical.parquet",
|
|
183
|
+
cat2="infrared.csv",
|
|
184
|
+
spec=MatchSpec(radius_arcsec=1.5, matcher="sky", find="best"),
|
|
185
|
+
engine="fast", # SciPy cKDTree multi-threaded on 3D unit sphere
|
|
186
|
+
)
|
|
187
|
+
result_df = cm.crossmatch_request(req)
|
|
188
|
+
|
|
189
|
+
# 2. Multi-catalogue sequential intersection
|
|
190
|
+
result_3way = cm.crossmatch_multi(
|
|
191
|
+
["optical.parquet", "infrared.csv", "radio.fits"],
|
|
192
|
+
radius_arcsec=2.0,
|
|
193
|
+
join_type="1and2",
|
|
194
|
+
)
|
|
195
|
+
|
|
196
|
+
# 3. Friends-of-Friends object bundles
|
|
197
|
+
bundles = cm.fof_match(
|
|
198
|
+
["survey_a.parquet", "survey_b.parquet", "survey_c.parquet"],
|
|
199
|
+
radius_arcsec=1.5,
|
|
200
|
+
)
|
|
201
|
+
```
|
|
202
|
+
|
|
203
|
+
---
|
|
204
|
+
|
|
205
|
+
## License
|
|
206
|
+
|
|
207
|
+
MIT License. See [LICENSE](LICENSE) for details.
|