cosmic-crunch 2.0.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (39) hide show
  1. cosmic_crunch-2.0.0/.github/FUNDING.yml +12 -0
  2. cosmic_crunch-2.0.0/.github/workflows/ci.yml +55 -0
  3. cosmic_crunch-2.0.0/.github/workflows/publish.yml +79 -0
  4. cosmic_crunch-2.0.0/.gitignore +15 -0
  5. cosmic_crunch-2.0.0/CHANGELOG.md +72 -0
  6. cosmic_crunch-2.0.0/CITATION.cff +22 -0
  7. cosmic_crunch-2.0.0/LICENSE +21 -0
  8. cosmic_crunch-2.0.0/PKG-INFO +162 -0
  9. cosmic_crunch-2.0.0/README.md +144 -0
  10. cosmic_crunch-2.0.0/cosmic_crunch/__init__.py +19 -0
  11. cosmic_crunch-2.0.0/cosmic_crunch/_parallel.py +197 -0
  12. cosmic_crunch-2.0.0/cosmic_crunch/cli.py +369 -0
  13. cosmic_crunch-2.0.0/cosmic_crunch/convert.py +402 -0
  14. cosmic_crunch-2.0.0/cosmic_crunch/fetch.py +396 -0
  15. cosmic_crunch-2.0.0/docs/design/2026-07-07-cosmic-crunch-v2.md +173 -0
  16. cosmic_crunch-2.0.0/docs/plans/2026-07-07-repo-improvement-plan.md +202 -0
  17. cosmic_crunch-2.0.0/docs/release-checklist.md +81 -0
  18. cosmic_crunch-2.0.0/docs/site-notes.md +125 -0
  19. cosmic_crunch-2.0.0/pyproject.toml +55 -0
  20. cosmic_crunch-2.0.0/tests/conftest.py +53 -0
  21. cosmic_crunch-2.0.0/tests/fixtures/20060501_0632co1_g35_2p6.L2.txt.gz +0 -0
  22. cosmic_crunch-2.0.0/tests/fixtures/20190103_0000co1_g72_2p6.L2.txt.gz +0 -0
  23. cosmic_crunch-2.0.0/tests/fixtures/README.md +41 -0
  24. cosmic_crunch-2.0.0/tests/fixtures/cosmic1_SYNTHETIC_empty-data.L2.txt.gz +0 -0
  25. cosmic_crunch-2.0.0/tests/fixtures/cosmic1_SYNTHETIC_malicious-header.L2.txt.gz +0 -0
  26. cosmic_crunch-2.0.0/tests/fixtures/cosmic1_SYNTHETIC_multitype-differing-width.L2.txt.gz +0 -0
  27. cosmic_crunch-2.0.0/tests/fixtures/listings/00_glevels_root.html +71 -0
  28. cosmic_crunch-2.0.0/tests/fixtures/listings/01_cosmic1_postproc.html +63 -0
  29. cosmic_crunch-2.0.0/tests/fixtures/listings/02_cosmic1_postproc_y2006.html +232 -0
  30. cosmic_crunch-2.0.0/tests/fixtures/listings/03_cosmic1_2006-05-01_date.html +54 -0
  31. cosmic_crunch-2.0.0/tests/fixtures/listings/04_cosmic1_2006-05-01_L2.html +51 -0
  32. cosmic_crunch-2.0.0/tests/fixtures/listings/05_cosmic1_2006-05-01_L2_txt.html +60 -0
  33. cosmic_crunch-2.0.0/tests/fixtures/listings/README.md +40 -0
  34. cosmic_crunch-2.0.0/tests/test_convert.py +45 -0
  35. cosmic_crunch-2.0.0/tests/test_crawler.py +72 -0
  36. cosmic_crunch-2.0.0/tests/test_fetch_download.py +112 -0
  37. cosmic_crunch-2.0.0/tests/test_parser.py +85 -0
  38. cosmic_crunch-2.0.0/tests/test_retry.py +50 -0
  39. cosmic_crunch-2.0.0/tests/test_roundtrip.py +51 -0
@@ -0,0 +1,12 @@
1
+ # These are supported funding model platforms
2
+
3
+ github: [ErickShepherd] # Replace with up to 4 GitHub Sponsors-enabled usernames e.g., [user1, user2]
4
+ patreon: ErickShepherd # Replace with a single Patreon username
5
+ open_collective: # Replace with a single Open Collective username
6
+ ko_fi: # Replace with a single Ko-fi username
7
+ tidelift: # Replace with a single Tidelift platform-name/package-name e.g., npm/babel
8
+ community_bridge: # Replace with a single Community Bridge project-name e.g., cloud-foundry
9
+ liberapay: # Replace with a single Liberapay username
10
+ issuehunt: # Replace with a single IssueHunt username
11
+ otechie: # Replace with a single Otechie username
12
+ custom: ['paypal.me/ErickEShepherd', 'https://cash.app/$ErickShepherd', 'https://venmo.com/ErickShepherd', 'https://erickshepherd.com/pay/bitcoin/'] # Replace with up to 4 custom sponsorship URLs e.g., ['link1', 'link2']
@@ -0,0 +1,55 @@
1
+ name: CI
2
+
3
+ on:
4
+ push:
5
+ branches: [main, master]
6
+ pull_request:
7
+ workflow_dispatch:
8
+
9
+ jobs:
10
+ lint:
11
+ name: ruff
12
+ runs-on: ubuntu-latest
13
+ steps:
14
+ - uses: actions/checkout@v4
15
+ - uses: actions/setup-python@v5
16
+ with:
17
+ python-version: "3.12"
18
+ - name: Install ruff
19
+ run: python -m pip install ruff
20
+ - name: Run ruff
21
+ run: ruff check .
22
+
23
+ test:
24
+ name: pytest (py${{ matrix.python-version }})
25
+ runs-on: ubuntu-latest
26
+ strategy:
27
+ fail-fast: false
28
+ matrix:
29
+ python-version: ["3.10", "3.11", "3.12", "3.13"]
30
+ steps:
31
+ - uses: actions/checkout@v4
32
+ - uses: actions/setup-python@v5
33
+ with:
34
+ python-version: ${{ matrix.python-version }}
35
+ - name: Install package with test extras
36
+ run: |
37
+ python -m pip install --upgrade pip
38
+ python -m pip install -e ".[test]"
39
+ - name: Run tests (offline)
40
+ run: python -m pytest -q
41
+
42
+ build:
43
+ name: build + twine check
44
+ runs-on: ubuntu-latest
45
+ steps:
46
+ - uses: actions/checkout@v4
47
+ - uses: actions/setup-python@v5
48
+ with:
49
+ python-version: "3.12"
50
+ - name: Install build tooling
51
+ run: python -m pip install build twine
52
+ - name: Build sdist + wheel
53
+ run: python -m build
54
+ - name: Check distributions
55
+ run: twine check dist/*
@@ -0,0 +1,79 @@
1
+ name: Publish to PyPI
2
+
3
+ # Publishes the package via PyPI Trusted Publishing (OIDC) — no API token is
4
+ # stored anywhere (not in the repo, not in GitHub secrets). PyPI verifies the
5
+ # workflow's short-lived OIDC identity instead.
6
+ #
7
+ # Two ways to run:
8
+ # * A published GitHub Release -> builds and publishes to PyPI.
9
+ # * Manual "Run workflow" -> choose testpypi (dry run) or pypi.
10
+ #
11
+ # The publish jobs run in protected GitHub Environments (`pypi` / `testpypi`).
12
+ # Add yourself as a required reviewer on each (repo Settings -> Environments)
13
+ # so every upload pauses for your one-click approval before it leaves.
14
+
15
+ on:
16
+ release:
17
+ types: [released] # full releases only — pre-releases never auto-publish
18
+ workflow_dispatch:
19
+ inputs:
20
+ target:
21
+ description: "Where to publish"
22
+ type: choice
23
+ options: [testpypi, pypi]
24
+ default: testpypi
25
+
26
+ permissions:
27
+ contents: read
28
+
29
+ jobs:
30
+ build:
31
+ name: Build sdist + wheel
32
+ runs-on: ubuntu-latest
33
+ steps:
34
+ - uses: actions/checkout@v4
35
+ - uses: actions/setup-python@v5
36
+ with:
37
+ python-version: "3.12"
38
+ - name: Build
39
+ run: |
40
+ python -m pip install --upgrade build twine
41
+ python -m build
42
+ python -m twine check dist/*
43
+ - uses: actions/upload-artifact@v4
44
+ with:
45
+ name: dist
46
+ path: dist/
47
+
48
+ publish-testpypi:
49
+ name: Publish to TestPyPI
50
+ needs: build
51
+ if: github.event_name == 'workflow_dispatch' && inputs.target == 'testpypi'
52
+ runs-on: ubuntu-latest
53
+ environment: testpypi
54
+ permissions:
55
+ id-token: write
56
+ steps:
57
+ - uses: actions/download-artifact@v4
58
+ with:
59
+ name: dist
60
+ path: dist/
61
+ - uses: pypa/gh-action-pypi-publish@release/v1
62
+ with:
63
+ repository-url: https://test.pypi.org/legacy/
64
+ skip-existing: true # re-dispatching a dry run for an existing version won't fail
65
+
66
+ publish-pypi:
67
+ name: Publish to PyPI
68
+ needs: build
69
+ if: github.event_name == 'release' || (github.event_name == 'workflow_dispatch' && inputs.target == 'pypi')
70
+ runs-on: ubuntu-latest
71
+ environment: pypi
72
+ permissions:
73
+ id-token: write
74
+ steps:
75
+ - uses: actions/download-artifact@v4
76
+ with:
77
+ name: dist
78
+ path: dist/
79
+ - uses: pypa/gh-action-pypi-publish@release/v1
@@ -0,0 +1,15 @@
1
+ # Python bytecode / caches
2
+ __pycache__/
3
+ *.py[cod]
4
+ .pytest_cache/
5
+
6
+ # Packaging / build artifacts
7
+ build/
8
+ dist/
9
+ *.egg-info/
10
+
11
+ # Logs
12
+ *.log
13
+
14
+ # Downloaded data directories (default fetch SAVE_DIRECTORY)
15
+ jpl_cosmic/
@@ -0,0 +1,72 @@
1
+ # Changelog
2
+
3
+ All notable changes to this project are documented here. The format is based on
4
+ [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and this project
5
+ adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
6
+
7
+ ## [2.0.0] — 2026-07-08
8
+
9
+ The 2020 JPL site restructure broke v1 (it silently downloaded nothing), and v1
10
+ carried latent security and correctness defects. v2 is a modernization: same
11
+ purpose, packaged, tested, and fixed.
12
+
13
+ ### Added
14
+ - Packaged as the `cosmic_crunch` distribution with a single `cosmic-crunch`
15
+ console entry point (`get` and `convert` subcommands) via `pyproject.toml`
16
+ (hatchling); requires Python 3.10+.
17
+ - `--base-url` flag and `COSMIC_CRUNCH_BASE_URL` environment variable to override
18
+ the crawl root.
19
+ - `--instrument` flag (default `cosmic`) to select other instrument trees
20
+ (`champ`, `gracea`, `gracefo1`, …).
21
+ - Atomic, resumable downloads: stream to `<name>.part` then rename; skip files
22
+ already present with a matching size.
23
+ - Offline `pytest` suite and a GitHub Actions CI workflow (ruff + pytest on
24
+ Python 3.10–3.13 + `build`/`twine check`).
25
+ - `CHANGELOG.md` and `CITATION.cff`.
26
+
27
+ ### Changed
28
+ - **New crawl root** `https://genesis.jpl.nasa.gov/ftp/glevels/` (the v1
29
+ `/ftp/pub/genesis/glevels` root is dead).
30
+ - **Fail loud:** a crawl that finds zero data files now raises and exits
31
+ non-zero instead of silently "succeeding" with nothing downloaded.
32
+ - **Bounded retry:** the v1 infinite silent retry is replaced with a bounded
33
+ retry (default 3 attempts, exponential backoff, logged, re-raises on failure).
34
+ - Relicensed from AGPL-3.0 to **MIT**.
35
+ - Logging is configured only under the CLI, not at import time (importing the
36
+ package no longer creates a stray log file).
37
+ - Deduplicated the copy-pasted `parallelize`/`flatten`/retry helpers into
38
+ `cosmic_crunch/_parallel.py`; single-sourced `__version__`.
39
+
40
+ ### Fixed
41
+ - **Security:** removed `eval()` of downloaded header content (arbitrary code
42
+ execution); header values are parsed with `ast.literal_eval` + raw-string
43
+ fallback, preserving brace-set element order.
44
+ - **Security:** path traversal in downloads — a hostile or tampered directory
45
+ listing could smuggle `../` through a filename and write outside the save
46
+ directory. Downloaded filenames are now reduced to a bare leaf and confined
47
+ to the save root (irreducible unsafe names are rejected before any fetch).
48
+ - The v1 `FILES_TO_GET = -1` default silently dropped the last crawled file on
49
+ every non-test run (`[:-1]`); it now defaults to downloading everything.
50
+ - `crawl_convert` processed only the first path argument (a `return` inside the
51
+ per-path loop, plus a `paths` shadowing typo) — it now converts all paths.
52
+ - Column-name loop-leak: rows of DataTypes with differing field counts were
53
+ parsed with the wrong column names — each type now keeps its own columns.
54
+ - Single-file conversion creates its output directory; directory creation uses
55
+ `os.makedirs(exist_ok=True)`.
56
+
57
+ ### Migration
58
+ - The invocation changed: `python get_files.py …` / `python convert_files.py …`
59
+ become `cosmic-crunch get …` / `cosmic-crunch convert …`. This is the breaking
60
+ change behind the major version bump. There are no deprecation shims (the tool
61
+ was never published and has no known downstream users).
62
+
63
+ ## [1.3.x] — 2020–2021
64
+
65
+ The original two-script tool (unpackaged, AGPL-3.0, never published to PyPI):
66
+
67
+ - `get_files.py` (v1.3.1) — download JPL COSMIC ASCII data files. Created
68
+ 2020-12-11, last updated 2021-08-02.
69
+ - `convert_files.py` (v1.3.3) — convert COSMIC ASCII files to netCDF4. Created
70
+ 2021-01-28, last updated 2021-08-02.
71
+
72
+ [2.0.0]: https://github.com/ErickShepherd/cosmic-crunch/releases/tag/v2.0.0
@@ -0,0 +1,22 @@
1
+ cff-version: 1.2.0
2
+ message: "If you use this software, please cite it as below."
3
+ title: "cosmic-crunch"
4
+ abstract: >-
5
+ Download JPL GENESIS COSMIC radio-occultation ASCII data files and convert
6
+ them to netCDF4.
7
+ type: software
8
+ authors:
9
+ - family-names: "Shepherd"
10
+ given-names: "Erick"
11
+ email: "Contact@ErickShepherd.com"
12
+ version: "2.0.0"
13
+ date-released: "2026-07-08"
14
+ license: MIT
15
+ repository-code: "https://github.com/ErickShepherd/cosmic-crunch"
16
+ url: "https://github.com/ErickShepherd/cosmic-crunch"
17
+ keywords:
18
+ - COSMIC
19
+ - GENESIS
20
+ - JPL
21
+ - "radio occultation"
22
+ - netCDF4
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2020-2026 Erick Shepherd
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
@@ -0,0 +1,162 @@
1
+ Metadata-Version: 2.4
2
+ Name: cosmic-crunch
3
+ Version: 2.0.0
4
+ Summary: Download JPL GENESIS COSMIC radio-occultation ASCII files and convert them to netCDF4.
5
+ Project-URL: Homepage, https://github.com/ErickShepherd/cosmic-crunch
6
+ Author-email: Erick Shepherd <Contact@ErickShepherd.com>
7
+ License-Expression: MIT
8
+ License-File: LICENSE
9
+ Keywords: COSMIC,GENESIS,JPL,netCDF4,radio occultation
10
+ Requires-Python: >=3.10
11
+ Requires-Dist: netcdf4>=1.6
12
+ Requires-Dist: pandas>=2.0
13
+ Requires-Dist: requests>=2.28
14
+ Requires-Dist: tqdm>=4.60
15
+ Provides-Extra: test
16
+ Requires-Dist: pytest; extra == 'test'
17
+ Description-Content-Type: text/markdown
18
+
19
+ # cosmic-crunch
20
+
21
+ [![CI](https://github.com/ErickShepherd/cosmic-crunch/actions/workflows/ci.yml/badge.svg)](https://github.com/ErickShepherd/cosmic-crunch/actions/workflows/ci.yml)
22
+ [![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](LICENSE)
23
+ [![Python](https://img.shields.io/badge/python-3.10%20%7C%203.11%20%7C%203.12%20%7C%203.13-blue.svg)](pyproject.toml)
24
+
25
+ Download JPL GENESIS **COSMIC** radio-occultation ASCII data files and convert
26
+ them to **netCDF4**.
27
+
28
+ `cosmic-crunch` crawls the [JPL GENESIS](https://genesis.jpl.nasa.gov) data
29
+ archive, downloads the Level-2 ASCII occultation profiles, and (optionally)
30
+ converts them into self-describing netCDF4 files.
31
+
32
+ > **Site restructure note (2020 → 2026).** JPL restructured the GENESIS site: the
33
+ > old crawl root (`/ftp/pub/genesis/glevels`) is dead, so v1 of this tool silently
34
+ > "succeeded" while downloading nothing. v2 targets the current root
35
+ > `https://genesis.jpl.nasa.gov/ftp/glevels/` and **fails loudly** — a crawl that
36
+ > finds nothing exits non-zero instead of pretending to succeed.
37
+ >
38
+ > **Mission status.** COSMIC-1 (FORMOSAT-3, flight modules FM1–FM6, served here as
39
+ > `cosmic1/`–`cosmic6/`) was decommissioned in 2020, so this is a **static
40
+ > archive** — the data has stopped changing. (COSMIC-2 is a different mission on a
41
+ > different archive and is out of scope.)
42
+
43
+ ## Installation
44
+
45
+ ```bash
46
+ pip install -e .
47
+ ```
48
+
49
+ This installs the `cosmic-crunch` command with two subcommands, `get` and
50
+ `convert`. Python 3.10+ is required.
51
+
52
+ ## Usage
53
+
54
+ `cosmic-crunch` has a single entry point with two subcommands:
55
+
56
+ ```bash
57
+ cosmic-crunch get # crawl the GENESIS site and download ASCII files
58
+ cosmic-crunch convert # convert downloaded ASCII files to netCDF4
59
+ ```
60
+
61
+ ### `cosmic-crunch get`
62
+
63
+ ```
64
+ usage: cosmic-crunch get [-h] [--base-url BASE_URL] [--instrument INSTRUMENT]
65
+ [--year_regex YEAR_REGEX] [--date_regex DATE_REGEX]
66
+ [--processes PROCESSES] [--test] [--netcdf4]
67
+ [--skip_empty]
68
+ ```
69
+
70
+ | flag | description |
71
+ |------|-------------|
72
+ | `--base-url` | Override the crawl root. Precedence: flag > `COSMIC_CRUNCH_BASE_URL` env var > built-in default (`https://genesis.jpl.nasa.gov/ftp/glevels`). |
73
+ | `--instrument` | Instrument tree to crawl (substring filter). Defaults to `cosmic` (matches `cosmic1`–`cosmic6`); the same archive also serves `champ`, `gracea`, `gracefo1`, … |
74
+ | `--year_regex` | Download only years matching this regular expression. |
75
+ | `--date_regex` | Download only dates matching this regular expression. |
76
+ | `--processes` | Worker processes for the `multiprocessing` pool (default `1`). |
77
+ | `--test` | Download a small subset (cosmic1, 2019-01-03, 10 files) as a smoke test. |
78
+ | `--netcdf4` | Convert the downloaded ASCII files to netCDF4 afterward. |
79
+ | `--skip_empty` | Skip converting files whose data arrays are all empty. |
80
+
81
+ Downloads are **atomic and resumable**: each file is streamed to a `.part`
82
+ temporary and renamed into place only once complete, and files already present
83
+ with a matching size are skipped — so an interrupted bulk pull can simply be
84
+ re-run.
85
+
86
+ A successful run resembles:
87
+
88
+ ```
89
+ $ cosmic-crunch get --year_regex=2006 --date_regex=2006-05-02 --netcdf4 --skip_empty --processes=4
90
+ Crawling all ./cosmic<#>/postproc: 100%|████████████████████| 6/6 [00:03<00:00, 1.61it/s]
91
+ Crawling all ./cosmic<#>/.../<year>: 100%|██████████████████| 6/6 [00:03<00:00, 1.59it/s]
92
+ Crawling all ./cosmic<#>/.../<date>: 100%|██████████████████| 3/3 [00:03<00:00, 1.17s/it]
93
+ Crawling all ./cosmic<#>/.../L2/<format>: 100%|█████████████| 4/4 [00:04<00:00, 1.09s/it]
94
+ Downloading data files: 100%|███████████████████████████████| 20/20 [00:26<00:00, 1.33s/it]
95
+ Converting ASCII to netCDF4: 100%|██████████████████████████| 20/20 [00:03<00:00, 6.32it/s]
96
+
97
+ ASCII to netCDF4 conversion summary:
98
+ - Successful conversions: 17
99
+ - Skipped conversions: 3
100
+ - Conversion errors: 0
101
+ - Total number of files: 20
102
+ ```
103
+
104
+ Downloaded files are written under `./jpl_cosmic/<year>/<date>/txt/`.
105
+
106
+ ### `cosmic-crunch convert`
107
+
108
+ ```
109
+ usage: cosmic-crunch convert [-h] [--logfile LOGFILE] [--processes PROCESSES]
110
+ [--skip_empty]
111
+ path [path ...]
112
+ ```
113
+
114
+ Convert one or more ASCII `.txt.gz` files — or directories of them (crawled
115
+ recursively) — to netCDF4. netCDF4 files are written into a sibling `nc/`
116
+ directory (mirroring the `txt/` layout), or beside the source file otherwise.
117
+
118
+ ```bash
119
+ cosmic-crunch convert ./jpl_cosmic/2006/ --skip_empty --processes=4
120
+ ```
121
+
122
+ ## netCDF4 output structure
123
+
124
+ Each ASCII file becomes one netCDF4 file. The ASCII header becomes **global
125
+ attributes**; each DataType profile becomes a **group** whose variables are that
126
+ profile's columns, indexed by an `Index` dimension:
127
+
128
+ ```
129
+ netcdf 20060501_0632co1_g35_2p6.L2 {
130
+ // global attributes: ProductCreationTime, ShortName, DataSetID,
131
+ // PlatformShortName, Receiver, ... (the ASCII header)
132
+
133
+ group: COSMIC1-Profile {
134
+ dimensions: Index = <n> ;
135
+ variables: Height, Lat, Lon, Refractivity, Temperature, Pressure, WV Pressure ;
136
+ }
137
+ group: ECMWF-Profile {
138
+ dimensions: Index = <n> ;
139
+ variables: Height, Lat, Lon, Refractivity, Temperature, Pressure, WV Pressure ;
140
+ }
141
+ }
142
+ ```
143
+
144
+ Missing values in the ASCII data (`-9999`) are stored as `NaN`.
145
+
146
+ ## Security note
147
+
148
+ v1 parsed header values with `eval()`, allowing arbitrary code execution from a
149
+ malicious or corrupted data file. v2 uses `ast.literal_eval` with a raw-string
150
+ fallback — header parsing can no longer execute code.
151
+
152
+ ## Development
153
+
154
+ ```bash
155
+ pip install -e ".[test]"
156
+ python -m pytest -q # offline test suite (never touches the network)
157
+ ruff check .
158
+ ```
159
+
160
+ ## License
161
+
162
+ MIT — see [LICENSE](LICENSE).
@@ -0,0 +1,144 @@
1
+ # cosmic-crunch
2
+
3
+ [![CI](https://github.com/ErickShepherd/cosmic-crunch/actions/workflows/ci.yml/badge.svg)](https://github.com/ErickShepherd/cosmic-crunch/actions/workflows/ci.yml)
4
+ [![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](LICENSE)
5
+ [![Python](https://img.shields.io/badge/python-3.10%20%7C%203.11%20%7C%203.12%20%7C%203.13-blue.svg)](pyproject.toml)
6
+
7
+ Download JPL GENESIS **COSMIC** radio-occultation ASCII data files and convert
8
+ them to **netCDF4**.
9
+
10
+ `cosmic-crunch` crawls the [JPL GENESIS](https://genesis.jpl.nasa.gov) data
11
+ archive, downloads the Level-2 ASCII occultation profiles, and (optionally)
12
+ converts them into self-describing netCDF4 files.
13
+
14
+ > **Site restructure note (2020 → 2026).** JPL restructured the GENESIS site: the
15
+ > old crawl root (`/ftp/pub/genesis/glevels`) is dead, so v1 of this tool silently
16
+ > "succeeded" while downloading nothing. v2 targets the current root
17
+ > `https://genesis.jpl.nasa.gov/ftp/glevels/` and **fails loudly** — a crawl that
18
+ > finds nothing exits non-zero instead of pretending to succeed.
19
+ >
20
+ > **Mission status.** COSMIC-1 (FORMOSAT-3, flight modules FM1–FM6, served here as
21
+ > `cosmic1/`–`cosmic6/`) was decommissioned in 2020, so this is a **static
22
+ > archive** — the data has stopped changing. (COSMIC-2 is a different mission on a
23
+ > different archive and is out of scope.)
24
+
25
+ ## Installation
26
+
27
+ ```bash
28
+ pip install -e .
29
+ ```
30
+
31
+ This installs the `cosmic-crunch` command with two subcommands, `get` and
32
+ `convert`. Python 3.10+ is required.
33
+
34
+ ## Usage
35
+
36
+ `cosmic-crunch` has a single entry point with two subcommands:
37
+
38
+ ```bash
39
+ cosmic-crunch get # crawl the GENESIS site and download ASCII files
40
+ cosmic-crunch convert # convert downloaded ASCII files to netCDF4
41
+ ```
42
+
43
+ ### `cosmic-crunch get`
44
+
45
+ ```
46
+ usage: cosmic-crunch get [-h] [--base-url BASE_URL] [--instrument INSTRUMENT]
47
+ [--year_regex YEAR_REGEX] [--date_regex DATE_REGEX]
48
+ [--processes PROCESSES] [--test] [--netcdf4]
49
+ [--skip_empty]
50
+ ```
51
+
52
+ | flag | description |
53
+ |------|-------------|
54
+ | `--base-url` | Override the crawl root. Precedence: flag > `COSMIC_CRUNCH_BASE_URL` env var > built-in default (`https://genesis.jpl.nasa.gov/ftp/glevels`). |
55
+ | `--instrument` | Instrument tree to crawl (substring filter). Defaults to `cosmic` (matches `cosmic1`–`cosmic6`); the same archive also serves `champ`, `gracea`, `gracefo1`, … |
56
+ | `--year_regex` | Download only years matching this regular expression. |
57
+ | `--date_regex` | Download only dates matching this regular expression. |
58
+ | `--processes` | Worker processes for the `multiprocessing` pool (default `1`). |
59
+ | `--test` | Download a small subset (cosmic1, 2019-01-03, 10 files) as a smoke test. |
60
+ | `--netcdf4` | Convert the downloaded ASCII files to netCDF4 afterward. |
61
+ | `--skip_empty` | Skip converting files whose data arrays are all empty. |
62
+
63
+ Downloads are **atomic and resumable**: each file is streamed to a `.part`
64
+ temporary and renamed into place only once complete, and files already present
65
+ with a matching size are skipped — so an interrupted bulk pull can simply be
66
+ re-run.
67
+
68
+ A successful run resembles:
69
+
70
+ ```
71
+ $ cosmic-crunch get --year_regex=2006 --date_regex=2006-05-02 --netcdf4 --skip_empty --processes=4
72
+ Crawling all ./cosmic<#>/postproc: 100%|████████████████████| 6/6 [00:03<00:00, 1.61it/s]
73
+ Crawling all ./cosmic<#>/.../<year>: 100%|██████████████████| 6/6 [00:03<00:00, 1.59it/s]
74
+ Crawling all ./cosmic<#>/.../<date>: 100%|██████████████████| 3/3 [00:03<00:00, 1.17s/it]
75
+ Crawling all ./cosmic<#>/.../L2/<format>: 100%|█████████████| 4/4 [00:04<00:00, 1.09s/it]
76
+ Downloading data files: 100%|███████████████████████████████| 20/20 [00:26<00:00, 1.33s/it]
77
+ Converting ASCII to netCDF4: 100%|██████████████████████████| 20/20 [00:03<00:00, 6.32it/s]
78
+
79
+ ASCII to netCDF4 conversion summary:
80
+ - Successful conversions: 17
81
+ - Skipped conversions: 3
82
+ - Conversion errors: 0
83
+ - Total number of files: 20
84
+ ```
85
+
86
+ Downloaded files are written under `./jpl_cosmic/<year>/<date>/txt/`.
87
+
88
+ ### `cosmic-crunch convert`
89
+
90
+ ```
91
+ usage: cosmic-crunch convert [-h] [--logfile LOGFILE] [--processes PROCESSES]
92
+ [--skip_empty]
93
+ path [path ...]
94
+ ```
95
+
96
+ Convert one or more ASCII `.txt.gz` files — or directories of them (crawled
97
+ recursively) — to netCDF4. netCDF4 files are written into a sibling `nc/`
98
+ directory (mirroring the `txt/` layout), or beside the source file otherwise.
99
+
100
+ ```bash
101
+ cosmic-crunch convert ./jpl_cosmic/2006/ --skip_empty --processes=4
102
+ ```
103
+
104
+ ## netCDF4 output structure
105
+
106
+ Each ASCII file becomes one netCDF4 file. The ASCII header becomes **global
107
+ attributes**; each DataType profile becomes a **group** whose variables are that
108
+ profile's columns, indexed by an `Index` dimension:
109
+
110
+ ```
111
+ netcdf 20060501_0632co1_g35_2p6.L2 {
112
+ // global attributes: ProductCreationTime, ShortName, DataSetID,
113
+ // PlatformShortName, Receiver, ... (the ASCII header)
114
+
115
+ group: COSMIC1-Profile {
116
+ dimensions: Index = <n> ;
117
+ variables: Height, Lat, Lon, Refractivity, Temperature, Pressure, WV Pressure ;
118
+ }
119
+ group: ECMWF-Profile {
120
+ dimensions: Index = <n> ;
121
+ variables: Height, Lat, Lon, Refractivity, Temperature, Pressure, WV Pressure ;
122
+ }
123
+ }
124
+ ```
125
+
126
+ Missing values in the ASCII data (`-9999`) are stored as `NaN`.
127
+
128
+ ## Security note
129
+
130
+ v1 parsed header values with `eval()`, allowing arbitrary code execution from a
131
+ malicious or corrupted data file. v2 uses `ast.literal_eval` with a raw-string
132
+ fallback — header parsing can no longer execute code.
133
+
134
+ ## Development
135
+
136
+ ```bash
137
+ pip install -e ".[test]"
138
+ python -m pytest -q # offline test suite (never touches the network)
139
+ ruff check .
140
+ ```
141
+
142
+ ## License
143
+
144
+ MIT — see [LICENSE](LICENSE).
@@ -0,0 +1,19 @@
1
+ #!/usr/bin/env python3
2
+ # -*- coding: utf-8 -*-
3
+
4
+ '''
5
+
6
+ cosmic_crunch: download JPL GENESIS COSMIC radio-occultation ASCII files and
7
+ convert them to netCDF4.
8
+
9
+ The single source of truth for the package version lives here (``__version__``)
10
+ and is read at build time by hatchling (see ``pyproject.toml`` ->
11
+ ``[tool.hatch.version]``).
12
+
13
+ Kept intentionally light: importing the package must not import the ``fetch`` or
14
+ ``convert`` submodules, so that ``import cosmic_crunch`` has no side effects.
15
+
16
+ '''
17
+
18
+ __author__ = "Erick Edward Shepherd"
19
+ __version__ = "2.0.0"