xmatcher 0.5.1__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (113) hide show
  1. xmatcher-0.5.1/.gitignore +70 -0
  2. xmatcher-0.5.1/.pre-commit-config.yaml +41 -0
  3. xmatcher-0.5.1/CHANGELOG.md +116 -0
  4. xmatcher-0.5.1/CONTRIBUTING.md +62 -0
  5. xmatcher-0.5.1/LICENSE +21 -0
  6. xmatcher-0.5.1/MANIFEST.in +5 -0
  7. xmatcher-0.5.1/PKG-INFO +207 -0
  8. xmatcher-0.5.1/README.md +154 -0
  9. xmatcher-0.5.1/docs/algorithms.md +262 -0
  10. xmatcher-0.5.1/docs/api.md +415 -0
  11. xmatcher-0.5.1/docs/association-v1.md +229 -0
  12. xmatcher-0.5.1/docs/canfar-scaling-2026-10-07.csv +33 -0
  13. xmatcher-0.5.1/docs/canfar-scaling-2026-10-07.md +30 -0
  14. xmatcher-0.5.1/docs/canfar-scaling-2026-10-07.png +0 -0
  15. xmatcher-0.5.1/docs/catalogue-pipeline.md +142 -0
  16. xmatcher-0.5.1/docs/observations.md +191 -0
  17. xmatcher-0.5.1/docs/release-0.5.0.md +37 -0
  18. xmatcher-0.5.1/docs/release-0.5.1.md +95 -0
  19. xmatcher-0.5.1/docs/release-audit-2026-10-06.md +286 -0
  20. xmatcher-0.5.1/docs/usage.md +626 -0
  21. xmatcher-0.5.1/pixi.lock +8619 -0
  22. xmatcher-0.5.1/pixi.toml +35 -0
  23. xmatcher-0.5.1/pyproject.toml +100 -0
  24. xmatcher-0.5.1/scripts/analyse_canfar_scaling.py +218 -0
  25. xmatcher-0.5.1/scripts/bench_engines.py +252 -0
  26. xmatcher-0.5.1/scripts/bench_ray_union.py +149 -0
  27. xmatcher-0.5.1/scripts/bench_ray_union_cap.py +161 -0
  28. xmatcher-0.5.1/scripts/canfar-ray-job.sh +241 -0
  29. xmatcher-0.5.1/scripts/canfar-smoke.sh +128 -0
  30. xmatcher-0.5.1/scripts/check_release.py +251 -0
  31. xmatcher-0.5.1/scripts/install_stilts.sh +57 -0
  32. xmatcher-0.5.1/setup.cfg +4 -0
  33. xmatcher-0.5.1/src/xmatcher/__init__.py +78 -0
  34. xmatcher-0.5.1/src/xmatcher/__main__.py +6 -0
  35. xmatcher-0.5.1/src/xmatcher/association.py +2410 -0
  36. xmatcher-0.5.1/src/xmatcher/astro_utils.py +446 -0
  37. xmatcher-0.5.1/src/xmatcher/auth.py +70 -0
  38. xmatcher-0.5.1/src/xmatcher/bayes.py +395 -0
  39. xmatcher-0.5.1/src/xmatcher/candidates.py +562 -0
  40. xmatcher-0.5.1/src/xmatcher/cli.py +2729 -0
  41. xmatcher-0.5.1/src/xmatcher/crossmatch.py +2656 -0
  42. xmatcher-0.5.1/src/xmatcher/discovery.py +385 -0
  43. xmatcher-0.5.1/src/xmatcher/exceptions.py +50 -0
  44. xmatcher-0.5.1/src/xmatcher/fixtures/association_v1.json +126 -0
  45. xmatcher-0.5.1/src/xmatcher/hats_native.py +448 -0
  46. xmatcher-0.5.1/src/xmatcher/io_utils.py +407 -0
  47. xmatcher-0.5.1/src/xmatcher/matchers.py +3605 -0
  48. xmatcher-0.5.1/src/xmatcher/mirror.py +1985 -0
  49. xmatcher-0.5.1/src/xmatcher/observations.py +652 -0
  50. xmatcher-0.5.1/src/xmatcher/out_of_core.py +909 -0
  51. xmatcher-0.5.1/src/xmatcher/ray_engine.py +456 -0
  52. xmatcher-0.5.1/src/xmatcher/ray_union.py +1873 -0
  53. xmatcher-0.5.1/src/xmatcher/remote_cds.py +163 -0
  54. xmatcher-0.5.1/src/xmatcher/remote_tap.py +223 -0
  55. xmatcher-0.5.1/src/xmatcher/request.py +315 -0
  56. xmatcher-0.5.1/src/xmatcher/sources.py +142 -0
  57. xmatcher-0.5.1/src/xmatcher/stilts.py +257 -0
  58. xmatcher-0.5.1/src/xmatcher/storage.py +549 -0
  59. xmatcher-0.5.1/src/xmatcher/tap.py +102 -0
  60. xmatcher-0.5.1/src/xmatcher/user_config.py +192 -0
  61. xmatcher-0.5.1/src/xmatcher/xmatcher.yaml +847 -0
  62. xmatcher-0.5.1/src/xmatcher.egg-info/PKG-INFO +207 -0
  63. xmatcher-0.5.1/src/xmatcher.egg-info/SOURCES.txt +111 -0
  64. xmatcher-0.5.1/src/xmatcher.egg-info/dependency_links.txt +1 -0
  65. xmatcher-0.5.1/src/xmatcher.egg-info/entry_points.txt +2 -0
  66. xmatcher-0.5.1/src/xmatcher.egg-info/requires.txt +40 -0
  67. xmatcher-0.5.1/src/xmatcher.egg-info/top_level.txt +1 -0
  68. xmatcher-0.5.1/tests/__init__.py +1 -0
  69. xmatcher-0.5.1/tests/conftest.py +113 -0
  70. xmatcher-0.5.1/tests/inputs/id_cat1.csv +4 -0
  71. xmatcher-0.5.1/tests/inputs/id_cat2.csv +4 -0
  72. xmatcher-0.5.1/tests/inputs/my_sources.csv +4 -0
  73. xmatcher-0.5.1/tests/inputs/simple_sources.csv +3 -0
  74. xmatcher-0.5.1/tests/inputs/test_cds_input.csv +4 -0
  75. xmatcher-0.5.1/tests/inputs/test_coord_override_2.csv +5 -0
  76. xmatcher-0.5.1/tests/inputs/test_id_join_1.csv +5 -0
  77. xmatcher-0.5.1/tests/inputs/test_id_join_2.csv +6 -0
  78. xmatcher-0.5.1/tests/inputs/test_local_vs_remote_id_1.csv +4 -0
  79. xmatcher-0.5.1/tests/tap_fake.py +245 -0
  80. xmatcher-0.5.1/tests/test_association_components.py +345 -0
  81. xmatcher-0.5.1/tests/test_association_contract.py +64 -0
  82. xmatcher-0.5.1/tests/test_association_deltas.py +337 -0
  83. xmatcher-0.5.1/tests/test_association_equivalence.py +323 -0
  84. xmatcher-0.5.1/tests/test_association_release.py +177 -0
  85. xmatcher-0.5.1/tests/test_astro_utils.py +277 -0
  86. xmatcher-0.5.1/tests/test_benchmark_checks.py +72 -0
  87. xmatcher-0.5.1/tests/test_benchmarks.py +316 -0
  88. xmatcher-0.5.1/tests/test_candidate_pipeline.py +413 -0
  89. xmatcher-0.5.1/tests/test_canfar_ray_job.py +106 -0
  90. xmatcher-0.5.1/tests/test_canfar_smoke.py +34 -0
  91. xmatcher-0.5.1/tests/test_check_release.py +208 -0
  92. xmatcher-0.5.1/tests/test_cli.py +719 -0
  93. xmatcher-0.5.1/tests/test_cli_sync.py +207 -0
  94. xmatcher-0.5.1/tests/test_crossmatch.py +1098 -0
  95. xmatcher-0.5.1/tests/test_distributed_regressions.py +879 -0
  96. xmatcher-0.5.1/tests/test_epoch_spill.py +496 -0
  97. xmatcher-0.5.1/tests/test_fallback_policy.py +225 -0
  98. xmatcher-0.5.1/tests/test_hats.py +128 -0
  99. xmatcher-0.5.1/tests/test_hats_native.py +346 -0
  100. xmatcher-0.5.1/tests/test_id_join_streaming.py +149 -0
  101. xmatcher-0.5.1/tests/test_io_utils.py +383 -0
  102. xmatcher-0.5.1/tests/test_large_crossmatch.py +454 -0
  103. xmatcher-0.5.1/tests/test_matchers.py +3488 -0
  104. xmatcher-0.5.1/tests/test_mirror_sync.py +1024 -0
  105. xmatcher-0.5.1/tests/test_observations.py +483 -0
  106. xmatcher-0.5.1/tests/test_out_of_core.py +377 -0
  107. xmatcher-0.5.1/tests/test_ray_union.py +1201 -0
  108. xmatcher-0.5.1/tests/test_real_catalogues.py +570 -0
  109. xmatcher-0.5.1/tests/test_remote.py +562 -0
  110. xmatcher-0.5.1/tests/test_storage.py +562 -0
  111. xmatcher-0.5.1/tests/test_tap.py +57 -0
  112. xmatcher-0.5.1/tests/test_torchsky_engine.py +242 -0
  113. xmatcher-0.5.1/tests/test_user_config.py +323 -0
@@ -0,0 +1,70 @@
1
+ # Byte-compiled / optimized / DLL files
2
+ __pycache__/
3
+ *.py[cod]
4
+ *$py.class
5
+
6
+ # C extensions
7
+ *.so
8
+
9
+ # Distribution / packaging
10
+ dist/
11
+ build/
12
+ *.egg-info/
13
+ *.egg
14
+
15
+ # Virtual environments
16
+ venv/
17
+ env/
18
+ ENV/
19
+ .env
20
+ .venv
21
+ env.bak/
22
+ venv.bak/
23
+
24
+ # Unit test / coverage reports
25
+ htmlcov/
26
+ .tox/
27
+ .coverage
28
+ .coverage.*
29
+ .cache
30
+ nosetests.xml
31
+ coverage.xml
32
+ *.cover
33
+ .hypothesis/
34
+ .pytest_cache/
35
+
36
+ # Jupyter Notebook
37
+ .ipynb_checkpoints
38
+
39
+ # PyTorch specific
40
+ *.pt
41
+ *.pth
42
+ *.onnx
43
+ *.h5
44
+ checkpoint/
45
+ runs/
46
+ logs/
47
+ tensorboard/
48
+ lightning_logs/
49
+
50
+ # IDE specific files
51
+ .idea/
52
+ .vscode/
53
+ *.swp
54
+ *.swo
55
+ .DS_Store
56
+
57
+ # Project specific
58
+ data/
59
+ output/
60
+ results/
61
+
62
+ # Ignore test and benchmark output files
63
+ tests/outputs/
64
+ benchmark_results/
65
+ cache/
66
+
67
+ # Tool and package manager caches
68
+ .pixi/
69
+ .mypy_cache/
70
+ .ruff_cache/
@@ -0,0 +1,41 @@
1
+ default_install_hook_types: [pre-commit, pre-push]
2
+
3
+ repos:
4
+ - repo: https://github.com/pre-commit/pre-commit-hooks
5
+ rev: v5.0.0
6
+ hooks:
7
+ - id: trailing-whitespace
8
+ - id: end-of-file-fixer
9
+ - id: check-yaml
10
+ - id: check-toml
11
+ - id: check-added-large-files
12
+ - id: check-merge-conflict
13
+ - id: debug-statements
14
+
15
+ - repo: https://github.com/astral-sh/ruff-pre-commit
16
+ rev: v0.16.10
17
+ hooks:
18
+ - id: ruff-check
19
+ - id: ruff-format
20
+
21
+ - repo: https://github.com/codespell-project/codespell
22
+ rev: v2.4.3
23
+ hooks:
24
+ - id: codespell
25
+
26
+ - repo: local
27
+ hooks:
28
+ - id: mypy
29
+ name: type checks in the locked scientific environment
30
+ entry: pixi run --as-is mypy src/xmatcher tests scripts
31
+ language: system
32
+ pass_filenames: false
33
+ always_run: true
34
+ - id: ci-local-pre-push
35
+ name: offline test gate
36
+ entry: pixi run ci-local
37
+ language: system
38
+ pass_filenames: false
39
+ always_run: true
40
+ require_serial: true
41
+ stages: [pre-push]
@@ -0,0 +1,116 @@
1
+ # Changelog
2
+
3
+ All notable changes to `xmatcher` are documented in this file.
4
+
5
+ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
6
+ and this project adheres to [Semantic Versioning](https://semver.org/).
7
+
8
+ ## [Unreleased]
9
+
10
+ ## [0.5.1] — 2026-10-08
11
+
12
+ Published on PyPI as `xmatcher`. The `v0.5.1` tag is the GitHub release.
13
+
14
+ ### Changed
15
+
16
+ - **Breaking package and command rename**: Python imports and the CLI move from
17
+ `xmatch` to `xmatcher`.
18
+ - **Breaking configuration rename**: the bundled and user configuration file
19
+ becomes `xmatcher.yaml`; environment variables move from the `XMATCH_` prefix
20
+ to `XMATCHER_`.
21
+ - **Changed local paths**: the default cache moves to `~/.cache/xmatcher`, and
22
+ the documented CANFAR output root is `/arc/projects/hats/xmatcher`.
23
+ - The persisted `xmatch.*` association schema identifiers remain unchanged.
24
+
25
+ ### Fixed
26
+
27
+ - FITS writes preserve explicit masks, including the distinction between empty
28
+ strings and nulls. Reads fall back to Astropy when the fast reader could conceal masked
29
+ values. CDS XMatch now uses a collision-free local row key, preserving input
30
+ columns with the previous internal key name.
31
+ - Catalogue names resolve case-insensitively to their configured spelling;
32
+ configurations with names that differ only by case are rejected. CLI table
33
+ discovery and schema queries use configured archive authentication, and
34
+ invalid CLI configuration is reported as a command error.
35
+ - HTTP HATS mirroring distinguishes complete listings from failed lookups,
36
+ includes partition metadata, and removes stale cached files only after a
37
+ successful refresh. A stale file that cannot be removed fails the sync and
38
+ keeps its manifest entry, so the next sync retries the removal.
39
+ - Astrometric error floors and drift inflation preserve measured cross
40
+ covariance for ellipse matching, and singular covariance pairs are rejected.
41
+ Astropy uses the shared ellipse filter; separation-based matcher fallbacks
42
+ retain every candidate for `find="all"`.
43
+ - Post-match filters apply the same row selection to matcher scores and respect
44
+ the requested right-column suffix. Candidate normalization checks pair
45
+ endpoint namespaces against the inventory, rejects contradictory target
46
+ namespace evidence, and allows an explicitly named empty target population.
47
+ Candidate-release setup failures also clean up their lock so publication can
48
+ be retried.
49
+ - HTTP HATS refreshes publish metadata after successful data transfers and use
50
+ a one-byte range probe when HEAD is unsupported. HTML listing fallback retains
51
+ CSV partition metadata and empty-catalogue schemas. VOS replica replacement
52
+ preserves nested paths and existing files when backup reads fail. TAP row
53
+ limits include short final pages.
54
+ - Native HATS reads and distributed unions retain the schema of valid empty
55
+ catalogues while rejecting missing partitions from nonempty catalogues.
56
+ CANFAR scaling validation requires the observed worker count to match the
57
+ requested count.
58
+
59
+ ## [0.5.0] — 2026-10-07
60
+
61
+ Available from the public [GitHub release at tag `v0.5.0`](https://github.com/astroai/xmatch/releases/tag/v0.5.0).
62
+ No PyPI package is published because another project owns the `xmatch` name.
63
+
64
+ ### Added
65
+
66
+ - **Source-Preserving Observation & Candidate Releases** (`observations.py`, `candidates.py`):
67
+ - Publish source inventories with normalized photometry and property-evidence ledgers in immutable, checksummed Parquet releases (`write_observation_release`, `verify_observation_release`).
68
+ - Publish scoped sparse candidate-pair releases (`write_candidate_release`, `verify_candidate_release`) and normalize caller-supplied per-target hypothesis weights with an explicit no-match alternative (`normalize_candidate_hypotheses`).
69
+ - **Versioned Association Contract** (`association.py`):
70
+ - Deterministic `xmatch.association.v1` pair records, atomic release directories, component sidecar releases (`xmatch.association.component.release.v1`), component delta classifications (`created`, `continued`, `merged`, `split`, `retired`), and bijective member equivalence crosswalks.
71
+ - **HATS Mirroring & Distributed Full-Sky Union** (`mirror.py`, `ray_union.py`, `storage.py`):
72
+ - `xmatch sync` CLI command and `sync_catalogue` / `ensure_mirrored` APIs to mirror remote TAP and HATS catalogues (over HTTP or CANFAR `vos:`) into a local HATS cache with keyset paging, token-bucket rate limiting, endpoint failover, and resumable checkpoints.
73
+ - `engine="ray-union"`: distributed $N$-survey full-outer join over mirrored HATS inputs on Ray, producing partitioned HATS trees (`Norder=*/Dir=*/Npix=*.parquet`) with chunk-level resume state.
74
+ - `ray_engine.py`: parallel HEALPix pixel-batch cone crossmatching (`engine="ray"`).
75
+ - **Astrometric Kinematics & Probabilistic Matchers** (`matchers.py`, `out_of_core.py`, `bayes.py`):
76
+ - Six-dimensional epoch propagation (`target_epoch`) and 5-parameter astrometric covariance transport through motion Jacobians for both `skyellipse` and `skyerr` (eager and out-of-core spill paths).
77
+ - Wilson (2023)-inspired proper-motion drift uncertainty model (`pm_prior=True`) with per-row error quadrature inflation and optional magnitude refinement.
78
+ - Sutherland & Saunders (1992) Likelihood Ratio (`matcher="lr"`), Random Forest (`matcher="ml"`), XGBoost/LightGBM (`matcher="xgb"`), lightweight AUF-inspired empirical separation/background scoring (`matcher="auf"`), and a flux-augmented heuristic (`matcher="macauff"`); the latter two are not full reproductions of the published methods.
79
+ - Multi-condition SQL post-filtering (`MatchSpec.filter_expr`), feature-weighted ranking of spatial candidates (`MatchSpec.extra_distance_cols`), and HEALPix pixel-group batching (`MatchSpec.batch_size`; this is not a general process-memory bound).
80
+ - **Formats & Catalogues** (`io_utils.py`, `xmatch.yaml`):
81
+ - Native streaming `.tsv` and `.tab` reads and writes via Polars.
82
+ - Added NOIRLab Data Lab catalogues (`nsc`, `des`, `decals`, `smash`, `unwise`, `allwise_dl`, and pre-computed `*_x_gaia` crossmatch tables) and COSMOS (`cosmos-web`, `zcosmos`).
83
+ - Terminal progress indicator for remote TAP and VizieR downloads.
84
+
85
+ ### Fixed
86
+
87
+ - Distributed HATS union now keeps output partitions spatially coherent across mixed input orders and RING/NESTED layouts, and reports non-null union routing coordinates. Tuple caps select the lowest-separation results from the complete candidate product; planner scans account for declared positional uncertainty and epoch motion.
88
+ - Friends-of-Friends matching searches cross-catalogue secondary links while retaining the primary-anchored component output policy.
89
+ - Probabilistic scores require declared positional errors or explicit error floors; the documentation states the equal-prior and isotropic-score assumptions. Remote `skyerr`, `skyellipse`, and `target_epoch` queries require an explicit search cone when service-side uncertainty or motion margins cannot be bounded.
90
+ - Invalid `filter_expr` SQL and missing, nonnumeric, or nonfinite declared ranking features now raise `CrossMatchError` instead of silently retaining rows or dropping/imputing features.
91
+ - ML synthetic negative sampling uses a local fixed seed, making repeated inputs reproducible without advancing the caller's NumPy random state.
92
+ - STILTS per-primary `find="best"` now uses `best1`, avoiding the symmetric one-to-one selection performed by STILTS `best`.
93
+ - An unreachable explicit Ray address no longer silently starts a local cluster under strict fallback policy, and the caller's address setting is preserved. Memory and storage budgets documented in GiB now use binary GiB.
94
+
95
+ ### Changed
96
+
97
+ - Updated the Python requirement to $\ge 3.13$ and modernized type annotations to PEP 585 / PEP 604 built-ins. The lockfile targets Python 3.13. A targeted Python 3.14.8 / Polars 2 compatibility run passed 206 tests and skipped 24; it was not the complete release gate on that stack.
98
+ - Streamlined core runtime dependencies to `numpy`, `polars`, `pyarrow`, `astropy`, `scipy`, `pyyaml`, and `pyvo`, with direct PyArrow/NumPy `Table`/`MaskedColumn` conversions and optional `xmatch[cds]` extra for `astroquery`.
99
+ - Consolidated HATS reading and writing on the native HATS engine (`hats` + `cdshealpix`). Ray-union uses compressed interval planning and measured per-source halos for adaptive/mixed-order and RING inputs and uncertainty- or epoch-adjusted searches; it may scan additional partitions. The pairwise `hats_native` fast path remains fixed-radius, uniform-order NESTED and globally materializes other layouts or error/epoch-adjusted inputs.
100
+ - Removed the unusable `torchsky` package extra because that distribution is not published on PyPI. The tensor engine requires a separately installed compatible Torchsky checkout; the documented integration was tested with the sibling 0.4 development source installed editable and Torchfits 1.0.0 from PyPI.
101
+
102
+ ---
103
+
104
+ ## [0.4.0] — 2026-06-15
105
+
106
+ ### Added
107
+
108
+ - Typed request API (`MatchRequest`, `SideOverrides`, `CrossMatch.crossmatch_request`).
109
+ - Multi-catalogue sequential crossmatching (`CrossMatch.crossmatch_multi`) and CLI support for $N \ge 3$ catalogues.
110
+ - Remote archive discovery module (`xmatch.discovery`) and CLI subcommands (`search`, `discover`, `adopt`, `list`, `describe`).
111
+ - Lazy bounding-cone estimation (`astro_utils.sky_extent_from_frame`) and streaming Parquet/CSV output sinks (`io_utils.write_frame`).
112
+
113
+ ### Changed
114
+
115
+ - Switched Astropy–Polars table conversions (`astropy_table_to_polars`, `polars_to_astropy`) to Apache Arrow column buffers.
116
+ - Quoted ADQL identifiers in remote TAP queries to handle reserved words and special characters safely.
@@ -0,0 +1,62 @@
1
+ # Contributing to xmatcher
2
+
3
+ Thank you for contributing to `xmatcher`!
4
+
5
+ Before publishing, build the wheel and source archive with `uv build`, then run
6
+ `pixi run python scripts/check_release.py`. This verifies current source bytes,
7
+ package data, metadata, and the fixtures/helpers needed to run the archived tests.
8
+
9
+ ## Development Setup
10
+
11
+ We recommend [Pixi](https://pixi.sh) for a reproducible environment with all compiled dependencies (`polars`, `scipy`, `astropy`, `cdshealpix`, `ray`):
12
+
13
+ ```bash
14
+ git clone https://github.com/astroai/xmatcher.git
15
+ cd xmatcher
16
+ pixi install
17
+ ```
18
+
19
+ On shared or NFS-mounted systems (such as CANFAR), isolate user site-packages before running Python or tests:
20
+
21
+ ```bash
22
+ export PYTHONNOUSERSITE=1
23
+ unset PYTHONPATH
24
+ ```
25
+
26
+ ## Linting, Formatting, and Tests
27
+
28
+ Run the fast preflight after a small change. It checks Ruff lint, formatting, Mypy, and bytecode compilation:
29
+
30
+ ```bash
31
+ pixi run preflight-push
32
+ ```
33
+
34
+ Auto-format and lint with Ruff:
35
+
36
+ ```bash
37
+ pixi run format
38
+ pixi run lint
39
+ ```
40
+
41
+ Run the release gate before opening a pull request. It runs the preflight and all tests not marked `slow` or `bench`:
42
+
43
+ ```bash
44
+ pixi run ci-local
45
+ ```
46
+
47
+ Run targeted tests while developing, or select the full suite explicitly when you need slow and benchmark tests:
48
+
49
+ ```bash
50
+ # Targeted module tests
51
+ pixi run test tests/test_matchers.py -q
52
+
53
+ # Entire pytest suite, including tests excluded from ci-local
54
+ pixi run test
55
+ ```
56
+
57
+ The offline `ci-local` gate does not establish that optional integrations, remote services, credentials, or every Python version work. Run the relevant integration checks separately when changing those paths. The checked Pixi environment uses Python 3.13.
58
+
59
+ ## Pull Requests
60
+
61
+ - Follow [Conventional Commits](https://www.conventionalcommits.org/) (`feat:`, `fix:`, `docs:`, `perf:`, `refactor:`, `test:`).
62
+ - Ensure `pixi run ci-local` and any relevant optional integration checks pass before opening a pull request against `astroai/xmatcher`.
xmatcher-0.5.1/LICENSE ADDED
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2026 Sébastien Fabbro
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
@@ -0,0 +1,5 @@
1
+ include CHANGELOG.md CONTRIBUTING.md pixi.toml pixi.lock
2
+ include .gitignore .pre-commit-config.yaml
3
+ include tests/*.py tests/inputs/*.csv
4
+ include docs/*.md docs/*.csv docs/*.png
5
+ include scripts/*.py scripts/*.sh
@@ -0,0 +1,207 @@
1
+ Metadata-Version: 2.4
2
+ Name: xmatcher
3
+ Version: 0.5.1
4
+ Summary: High-performance, format-agnostic astronomical catalogue crossmatching and multi-survey union toolkit.
5
+ Author-email: Seb Fabbro <sebfabbro@gmail.com>
6
+ License-Expression: MIT
7
+ Project-URL: Homepage, https://github.com/astroai/xmatcher
8
+ Project-URL: Repository, https://github.com/astroai/xmatcher
9
+ Project-URL: Documentation, https://github.com/astroai/xmatcher/tree/main/docs
10
+ Project-URL: Issues, https://github.com/astroai/xmatcher/issues
11
+ Keywords: astronomy,crossmatch,catalogue,hats,healpix,polars,tap
12
+ Classifier: Development Status :: 4 - Beta
13
+ Classifier: Intended Audience :: Science/Research
14
+ Classifier: Programming Language :: Python :: 3
15
+ Classifier: Programming Language :: Python :: 3.13
16
+ Classifier: Topic :: Scientific/Engineering :: Astronomy
17
+ Requires-Python: >=3.13
18
+ Description-Content-Type: text/markdown
19
+ License-File: LICENSE
20
+ Requires-Dist: numpy>=2.4
21
+ Requires-Dist: polars>=1.40
22
+ Requires-Dist: pyarrow>=25.0
23
+ Requires-Dist: astropy>=7.2
24
+ Requires-Dist: scipy>=1.18
25
+ Requires-Dist: pyyaml>=6.0.3
26
+ Requires-Dist: pyvo>=1.8
27
+ Provides-Extra: cds
28
+ Requires-Dist: astroquery>=0.4.11; extra == "cds"
29
+ Provides-Extra: torchfits
30
+ Requires-Dist: torchfits>=1.0.0rc4; extra == "torchfits"
31
+ Provides-Extra: zone
32
+ Requires-Dist: cdshealpix>=0.8; extra == "zone"
33
+ Provides-Extra: hats
34
+ Requires-Dist: hats>=0.7; extra == "hats"
35
+ Requires-Dist: cdshealpix>=0.8; extra == "hats"
36
+ Provides-Extra: hats-ray
37
+ Requires-Dist: hats>=0.7; extra == "hats-ray"
38
+ Requires-Dist: ray>=2.50; extra == "hats-ray"
39
+ Requires-Dist: cdshealpix>=0.8; extra == "hats-ray"
40
+ Provides-Extra: ray
41
+ Requires-Dist: ray>=2.50; extra == "ray"
42
+ Requires-Dist: cdshealpix>=0.8; extra == "ray"
43
+ Provides-Extra: ml
44
+ Requires-Dist: scikit-learn; extra == "ml"
45
+ Requires-Dist: xgboost; extra == "ml"
46
+ Requires-Dist: lightgbm; extra == "ml"
47
+ Requires-Dist: joblib; extra == "ml"
48
+ Provides-Extra: dev
49
+ Requires-Dist: pytest>=9.0; extra == "dev"
50
+ Requires-Dist: ruff>=0.15; extra == "dev"
51
+ Requires-Dist: mypy>=2.0; extra == "dev"
52
+ Dynamic: license-file
53
+
54
+ # xmatcher
55
+
56
+ `xmatcher` matches and joins astronomical catalogues from local files, HATS trees, and configured remote archives. Matching features depend on the selected engine and the metadata supplied for each catalogue; the guides document those limits and the memory trade-offs.
57
+
58
+ Version 0.5.1 is the PyPI release. The
59
+ [v0.5.0](https://github.com/astroai/xmatch/releases/tag/v0.5.0) GitHub release
60
+ keeps the old `xmatch` name. See the
61
+ [migration notes](docs/release-0.5.1.md).
62
+
63
+ ```bash
64
+ xmatcher match catalog1.parquet catalog2.csv -o matches.parquet -r 1.0
65
+ ```
66
+
67
+ Inputs can be local Parquet, CSV, TSV, or FITS files, HATS directories, Polars frames, or configured remote catalogues. Coordinate names can be configured or detected. Proper-motion propagation uses declared epoch and motion metadata; mixed coordinate frames are rejected and generic catalogue discovery does not invent positional uncertainties.
68
+
69
+ ---
70
+
71
+ ## 📚 Complete Documentation Suite
72
+
73
+ All detailed architectural references and user guides are organized in the [`docs/`](docs/) directory:
74
+
75
+ - **[User Guide & Cookbooks (`docs/usage.md`)](docs/usage.md)**: Installation, local and remote matching, explicit memory controls, and distributed workflows with their limits.
76
+ - **[Public API Reference (`docs/api.md`)](docs/api.md)**: Full Python API specifications for `CrossMatch`, `MatchRequest`, `MatchSpec`, observation/candidate release helpers, engines, storage abstractions, and exception hierarchies.
77
+ - **[Crossmatch Algorithms & Math (`docs/algorithms.md`)](docs/algorithms.md)**: Matcher definitions, uncertainty assumptions, score semantics, and engine behavior.
78
+ - **[Source & Measurement Releases (`docs/observations.md`)](docs/observations.md)**: Lazy, source-preserving measurement ledgers, photometric unit/limit normalization, property evidence, and checksummed Parquet releases.
79
+ - **[Candidate & Association Pipeline (`docs/catalogue-pipeline.md`)](docs/catalogue-pipeline.md)**: End-to-end workflow from pinned survey observation and sparse-candidate releases to per-target association hypotheses.
80
+ - **[Association Contract v1 (`docs/association-v1.md`)](docs/association-v1.md)**: Audit-grade versioned association record specification, release directories, component sidecars, and member equivalence tracking.
81
+
82
+ ---
83
+
84
+ ## Capabilities
85
+
86
+ - **Universal Multi-Format I/O**:
87
+ - Ingests and streams `.parquet`, `.csv`, `.tsv`, `.tab`, `.fits` (via high-speed Torchfits with Astropy fallback), and HATS.
88
+ - Remote storage connectors: POSIX filesystem, CANFAR VOSpace (`vos:`), HTTP/HTTPS, and S3.
89
+ - **File I/O and explicit memory controls**:
90
+ - Parquet, CSV, TSV, FITS, and HATS readers with Parquet/CSV/TSV/FITS outputs.
91
+ - Output sinks can stream serialization. Most matching engines still materialize inputs and matches; local pairwise CSV/Parquet requests can opt into the spill path with `--memory-budget-bytes` and file output. See [memory limits](docs/usage.md#pairwise-spill-matching-on-one-machine).
92
+ - **Astrometric and probabilistic matchers**:
93
+ - Simple radius (`sky`) and adaptive per-row astrometric uncertainties (`skyerr`).
94
+ - 2D Gaussian error ellipses via Mahalanobis distance (`skyellipse`).
95
+ - Proper-motion epoch propagation (`target_epoch`) when epoch and motion metadata are supplied, plus an explicit population drift model (`pm_prior`) for missing motion.
96
+ - Sutherland & Saunders (1992) Likelihood Ratio (`lr`) with magnitude background subtraction.
97
+ - Machine learning classifiers (`ml` Random Forest, `xgb` XGBoost/LightGBM) with self-match pseudo-labeling.
98
+ - AUF-inspired empirical separation/background scores (`auf`) and a flux-augmented heuristic (`macauff`). These are not full reproductions of the published methods and their scores are not calibrated probabilities.
99
+ - Bayesian positional qualification (`probabilistic=True`) or photometric qualification (`prior_columns`) with declared errors or explicit error floors, plus simultaneous $N$-way scores (`nway_match`). These scores use stated assumptions and are not population-calibrated probabilities.
100
+ - Friends-of-Friends transitive closure object bundles (`fof_match`).
101
+ - **Complete Relational Joins**:
102
+ - Inner (`1and2`), Left (`all1`), Right (`all2`), Full Outer (`1or2`, `all`), and Anti-joins (`1not2`, `2not1`).
103
+ - Relational ID joins on survey identifiers (`--id-join`).
104
+ - Multi-catalogue sequential chains (`crossmatch_multi`).
105
+ - **Local Mirroring & Offline Caching**:
106
+ - `xmatcher sync`: Mirror remote TAP and HATS surveys into a durable local HATS cache with keyset paging, rate limiting, and progress checkpoints.
107
+ - **Distributed execution**:
108
+ - `engine="ray"` parallelizes pairwise HEALPix candidate searches. The coordinator still holds the input frames and final result in memory.
109
+ - `engine="ray-union"` mirrors inputs into HATS and writes a full-sky, full-outer HATS union. It supports `sky`, `skyerr`, and `skyellipse`; compressed interval planning handles mixed-order/RING layouts and uncertainty or epoch halos in distributed tasks. Planning may scan additional partitions. Output trees add non-null `_union_ra` and `_union_dec` routing columns and declare NESTED ordering.
110
+
111
+ ---
112
+
113
+ ## Installation
114
+
115
+ Requires Python $\ge 3.13$. The checked Pixi environment and local release gate use Python 3.13.
116
+
117
+ ### Recommended: Pixi
118
+
119
+ ```bash
120
+ git clone https://github.com/astroai/xmatcher.git
121
+ cd xmatcher
122
+
123
+ # Install environment and dependencies
124
+ pixi install
125
+
126
+ # Run lint, formatting, bytecode, and offline test checks
127
+ pixi run preflight-push
128
+ pixi run ci-local
129
+ ```
130
+
131
+ ### Standard: Pip
132
+
133
+ ```bash
134
+ python -m pip install "xmatcher==0.5.1"
135
+
136
+ # Optional integrations; use only the extras you need
137
+ python -m pip install "xmatcher[cds,hats-ray,torchfits,ml]==0.5.1"
138
+ ```
139
+
140
+ A source checkout uses the same extras from the tree:
141
+
142
+ ```bash
143
+ git clone https://github.com/astroai/xmatcher.git
144
+ cd xmatcher
145
+ python -m pip install ".[cds,hats-ray,torchfits,ml]"
146
+ ```
147
+
148
+ The tensor-native `torchsky` engine is not a package extra: no `torchsky`
149
+ distribution is published on PyPI. To use it, install xmatcher alongside a
150
+ compatible Torchsky checkout. The integration was tested with the sibling
151
+ Torchsky 0.4 development source installed editable and Torchfits 1.0.0 from
152
+ PyPI.
153
+
154
+ ---
155
+
156
+ ## Command Line Quick Reference
157
+
158
+ | Task | Command |
159
+ |---|---|
160
+ | Match two local files | `xmatcher match cat1.parquet cat2.csv -o matches.parquet -r 1.5` |
161
+ | Match against remote Gaia DR3 | `xmatcher match targets.csv gaia -o targets_gaia.parquet -r 1.0` |
162
+ | Astrometric error ellipse match | `xmatcher match xray.fits optical.parquet --matcher skyellipse --max-error 3.0` |
163
+ | Proper-motion drift prior match | `xmatcher match old_survey.csv gaia --target-epoch 2016.0 --pm-prior -r 2.0` |
164
+ | Relational ID join | `xmatcher match cat1.parquet cat2.parquet --id-join --id1 objid --id2 objid` |
165
+ | Multi-survey 3-way chain | `xmatcher match gaia wise.csv 2mass.parquet -r 1.5 -o 3way.parquet` |
166
+ | Distributed HATS master union | `xmatcher match gaia allwise 2mass des --union --engine ray-union -o /data/master.hats` |
167
+ | Mirror remote surveys to local cache | `xmatcher sync gaia allwise 2mass` |
168
+ | Search remote TAP archives | `xmatcher search des` |
169
+ | Discover archive schemas | `xmatcher discover noirlab --schema nsc_dr2.object` |
170
+
171
+ ---
172
+
173
+ ## Python API Quickstart
174
+
175
+ ```python
176
+ from xmatcher import CrossMatch, MatchRequest, MatchSpec
177
+
178
+ cm = CrossMatch()
179
+
180
+ # 1. Preferred typed request interface
181
+ req = MatchRequest(
182
+ cat1="optical.parquet",
183
+ cat2="infrared.csv",
184
+ spec=MatchSpec(radius_arcsec=1.5, matcher="sky", find="best"),
185
+ engine="fast", # SciPy cKDTree multi-threaded on 3D unit sphere
186
+ )
187
+ result_df = cm.crossmatch_request(req)
188
+
189
+ # 2. Multi-catalogue sequential intersection
190
+ result_3way = cm.crossmatch_multi(
191
+ ["optical.parquet", "infrared.csv", "radio.fits"],
192
+ radius_arcsec=2.0,
193
+ join_type="1and2",
194
+ )
195
+
196
+ # 3. Friends-of-Friends object bundles
197
+ bundles = cm.fof_match(
198
+ ["survey_a.parquet", "survey_b.parquet", "survey_c.parquet"],
199
+ radius_arcsec=1.5,
200
+ )
201
+ ```
202
+
203
+ ---
204
+
205
+ ## License
206
+
207
+ MIT License. See [LICENSE](LICENSE) for details.