cosmic-crunch 2.0.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- cosmic_crunch-2.0.0/.github/FUNDING.yml +12 -0
- cosmic_crunch-2.0.0/.github/workflows/ci.yml +55 -0
- cosmic_crunch-2.0.0/.github/workflows/publish.yml +79 -0
- cosmic_crunch-2.0.0/.gitignore +15 -0
- cosmic_crunch-2.0.0/CHANGELOG.md +72 -0
- cosmic_crunch-2.0.0/CITATION.cff +22 -0
- cosmic_crunch-2.0.0/LICENSE +21 -0
- cosmic_crunch-2.0.0/PKG-INFO +162 -0
- cosmic_crunch-2.0.0/README.md +144 -0
- cosmic_crunch-2.0.0/cosmic_crunch/__init__.py +19 -0
- cosmic_crunch-2.0.0/cosmic_crunch/_parallel.py +197 -0
- cosmic_crunch-2.0.0/cosmic_crunch/cli.py +369 -0
- cosmic_crunch-2.0.0/cosmic_crunch/convert.py +402 -0
- cosmic_crunch-2.0.0/cosmic_crunch/fetch.py +396 -0
- cosmic_crunch-2.0.0/docs/design/2026-07-07-cosmic-crunch-v2.md +173 -0
- cosmic_crunch-2.0.0/docs/plans/2026-07-07-repo-improvement-plan.md +202 -0
- cosmic_crunch-2.0.0/docs/release-checklist.md +81 -0
- cosmic_crunch-2.0.0/docs/site-notes.md +125 -0
- cosmic_crunch-2.0.0/pyproject.toml +55 -0
- cosmic_crunch-2.0.0/tests/conftest.py +53 -0
- cosmic_crunch-2.0.0/tests/fixtures/20060501_0632co1_g35_2p6.L2.txt.gz +0 -0
- cosmic_crunch-2.0.0/tests/fixtures/20190103_0000co1_g72_2p6.L2.txt.gz +0 -0
- cosmic_crunch-2.0.0/tests/fixtures/README.md +41 -0
- cosmic_crunch-2.0.0/tests/fixtures/cosmic1_SYNTHETIC_empty-data.L2.txt.gz +0 -0
- cosmic_crunch-2.0.0/tests/fixtures/cosmic1_SYNTHETIC_malicious-header.L2.txt.gz +0 -0
- cosmic_crunch-2.0.0/tests/fixtures/cosmic1_SYNTHETIC_multitype-differing-width.L2.txt.gz +0 -0
- cosmic_crunch-2.0.0/tests/fixtures/listings/00_glevels_root.html +71 -0
- cosmic_crunch-2.0.0/tests/fixtures/listings/01_cosmic1_postproc.html +63 -0
- cosmic_crunch-2.0.0/tests/fixtures/listings/02_cosmic1_postproc_y2006.html +232 -0
- cosmic_crunch-2.0.0/tests/fixtures/listings/03_cosmic1_2006-05-01_date.html +54 -0
- cosmic_crunch-2.0.0/tests/fixtures/listings/04_cosmic1_2006-05-01_L2.html +51 -0
- cosmic_crunch-2.0.0/tests/fixtures/listings/05_cosmic1_2006-05-01_L2_txt.html +60 -0
- cosmic_crunch-2.0.0/tests/fixtures/listings/README.md +40 -0
- cosmic_crunch-2.0.0/tests/test_convert.py +45 -0
- cosmic_crunch-2.0.0/tests/test_crawler.py +72 -0
- cosmic_crunch-2.0.0/tests/test_fetch_download.py +112 -0
- cosmic_crunch-2.0.0/tests/test_parser.py +85 -0
- cosmic_crunch-2.0.0/tests/test_retry.py +50 -0
- cosmic_crunch-2.0.0/tests/test_roundtrip.py +51 -0
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
# These are supported funding model platforms
|
|
2
|
+
|
|
3
|
+
github: [ErickShepherd] # Replace with up to 4 GitHub Sponsors-enabled usernames e.g., [user1, user2]
|
|
4
|
+
patreon: ErickShepherd # Replace with a single Patreon username
|
|
5
|
+
open_collective: # Replace with a single Open Collective username
|
|
6
|
+
ko_fi: # Replace with a single Ko-fi username
|
|
7
|
+
tidelift: # Replace with a single Tidelift platform-name/package-name e.g., npm/babel
|
|
8
|
+
community_bridge: # Replace with a single Community Bridge project-name e.g., cloud-foundry
|
|
9
|
+
liberapay: # Replace with a single Liberapay username
|
|
10
|
+
issuehunt: # Replace with a single IssueHunt username
|
|
11
|
+
otechie: # Replace with a single Otechie username
|
|
12
|
+
custom: ['paypal.me/ErickEShepherd', 'https://cash.app/$ErickShepherd', 'https://venmo.com/ErickShepherd', 'https://erickshepherd.com/pay/bitcoin/'] # Replace with up to 4 custom sponsorship URLs e.g., ['link1', 'link2']
|
|
@@ -0,0 +1,55 @@
|
|
|
1
|
+
name: CI
|
|
2
|
+
|
|
3
|
+
on:
|
|
4
|
+
push:
|
|
5
|
+
branches: [main, master]
|
|
6
|
+
pull_request:
|
|
7
|
+
workflow_dispatch:
|
|
8
|
+
|
|
9
|
+
jobs:
|
|
10
|
+
lint:
|
|
11
|
+
name: ruff
|
|
12
|
+
runs-on: ubuntu-latest
|
|
13
|
+
steps:
|
|
14
|
+
- uses: actions/checkout@v4
|
|
15
|
+
- uses: actions/setup-python@v5
|
|
16
|
+
with:
|
|
17
|
+
python-version: "3.12"
|
|
18
|
+
- name: Install ruff
|
|
19
|
+
run: python -m pip install ruff
|
|
20
|
+
- name: Run ruff
|
|
21
|
+
run: ruff check .
|
|
22
|
+
|
|
23
|
+
test:
|
|
24
|
+
name: pytest (py${{ matrix.python-version }})
|
|
25
|
+
runs-on: ubuntu-latest
|
|
26
|
+
strategy:
|
|
27
|
+
fail-fast: false
|
|
28
|
+
matrix:
|
|
29
|
+
python-version: ["3.10", "3.11", "3.12", "3.13"]
|
|
30
|
+
steps:
|
|
31
|
+
- uses: actions/checkout@v4
|
|
32
|
+
- uses: actions/setup-python@v5
|
|
33
|
+
with:
|
|
34
|
+
python-version: ${{ matrix.python-version }}
|
|
35
|
+
- name: Install package with test extras
|
|
36
|
+
run: |
|
|
37
|
+
python -m pip install --upgrade pip
|
|
38
|
+
python -m pip install -e ".[test]"
|
|
39
|
+
- name: Run tests (offline)
|
|
40
|
+
run: python -m pytest -q
|
|
41
|
+
|
|
42
|
+
build:
|
|
43
|
+
name: build + twine check
|
|
44
|
+
runs-on: ubuntu-latest
|
|
45
|
+
steps:
|
|
46
|
+
- uses: actions/checkout@v4
|
|
47
|
+
- uses: actions/setup-python@v5
|
|
48
|
+
with:
|
|
49
|
+
python-version: "3.12"
|
|
50
|
+
- name: Install build tooling
|
|
51
|
+
run: python -m pip install build twine
|
|
52
|
+
- name: Build sdist + wheel
|
|
53
|
+
run: python -m build
|
|
54
|
+
- name: Check distributions
|
|
55
|
+
run: twine check dist/*
|
|
@@ -0,0 +1,79 @@
|
|
|
1
|
+
name: Publish to PyPI
|
|
2
|
+
|
|
3
|
+
# Publishes the package via PyPI Trusted Publishing (OIDC) — no API token is
|
|
4
|
+
# stored anywhere (not in the repo, not in GitHub secrets). PyPI verifies the
|
|
5
|
+
# workflow's short-lived OIDC identity instead.
|
|
6
|
+
#
|
|
7
|
+
# Two ways to run:
|
|
8
|
+
# * A published GitHub Release -> builds and publishes to PyPI.
|
|
9
|
+
# * Manual "Run workflow" -> choose testpypi (dry run) or pypi.
|
|
10
|
+
#
|
|
11
|
+
# The publish jobs run in protected GitHub Environments (`pypi` / `testpypi`).
|
|
12
|
+
# Add yourself as a required reviewer on each (repo Settings -> Environments)
|
|
13
|
+
# so every upload pauses for your one-click approval before it leaves.
|
|
14
|
+
|
|
15
|
+
on:
|
|
16
|
+
release:
|
|
17
|
+
types: [released] # full releases only — pre-releases never auto-publish
|
|
18
|
+
workflow_dispatch:
|
|
19
|
+
inputs:
|
|
20
|
+
target:
|
|
21
|
+
description: "Where to publish"
|
|
22
|
+
type: choice
|
|
23
|
+
options: [testpypi, pypi]
|
|
24
|
+
default: testpypi
|
|
25
|
+
|
|
26
|
+
permissions:
|
|
27
|
+
contents: read
|
|
28
|
+
|
|
29
|
+
jobs:
|
|
30
|
+
build:
|
|
31
|
+
name: Build sdist + wheel
|
|
32
|
+
runs-on: ubuntu-latest
|
|
33
|
+
steps:
|
|
34
|
+
- uses: actions/checkout@v4
|
|
35
|
+
- uses: actions/setup-python@v5
|
|
36
|
+
with:
|
|
37
|
+
python-version: "3.12"
|
|
38
|
+
- name: Build
|
|
39
|
+
run: |
|
|
40
|
+
python -m pip install --upgrade build twine
|
|
41
|
+
python -m build
|
|
42
|
+
python -m twine check dist/*
|
|
43
|
+
- uses: actions/upload-artifact@v4
|
|
44
|
+
with:
|
|
45
|
+
name: dist
|
|
46
|
+
path: dist/
|
|
47
|
+
|
|
48
|
+
publish-testpypi:
|
|
49
|
+
name: Publish to TestPyPI
|
|
50
|
+
needs: build
|
|
51
|
+
if: github.event_name == 'workflow_dispatch' && inputs.target == 'testpypi'
|
|
52
|
+
runs-on: ubuntu-latest
|
|
53
|
+
environment: testpypi
|
|
54
|
+
permissions:
|
|
55
|
+
id-token: write
|
|
56
|
+
steps:
|
|
57
|
+
- uses: actions/download-artifact@v4
|
|
58
|
+
with:
|
|
59
|
+
name: dist
|
|
60
|
+
path: dist/
|
|
61
|
+
- uses: pypa/gh-action-pypi-publish@release/v1
|
|
62
|
+
with:
|
|
63
|
+
repository-url: https://test.pypi.org/legacy/
|
|
64
|
+
skip-existing: true # re-dispatching a dry run for an existing version won't fail
|
|
65
|
+
|
|
66
|
+
publish-pypi:
|
|
67
|
+
name: Publish to PyPI
|
|
68
|
+
needs: build
|
|
69
|
+
if: github.event_name == 'release' || (github.event_name == 'workflow_dispatch' && inputs.target == 'pypi')
|
|
70
|
+
runs-on: ubuntu-latest
|
|
71
|
+
environment: pypi
|
|
72
|
+
permissions:
|
|
73
|
+
id-token: write
|
|
74
|
+
steps:
|
|
75
|
+
- uses: actions/download-artifact@v4
|
|
76
|
+
with:
|
|
77
|
+
name: dist
|
|
78
|
+
path: dist/
|
|
79
|
+
- uses: pypa/gh-action-pypi-publish@release/v1
|
|
@@ -0,0 +1,72 @@
|
|
|
1
|
+
# Changelog
|
|
2
|
+
|
|
3
|
+
All notable changes to this project are documented here. The format is based on
|
|
4
|
+
[Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and this project
|
|
5
|
+
adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
|
|
6
|
+
|
|
7
|
+
## [2.0.0] — 2026-07-08
|
|
8
|
+
|
|
9
|
+
The 2020 JPL site restructure broke v1 (it silently downloaded nothing), and v1
|
|
10
|
+
carried latent security and correctness defects. v2 is a modernization: same
|
|
11
|
+
purpose, packaged, tested, and fixed.
|
|
12
|
+
|
|
13
|
+
### Added
|
|
14
|
+
- Packaged as the `cosmic_crunch` distribution with a single `cosmic-crunch`
|
|
15
|
+
console entry point (`get` and `convert` subcommands) via `pyproject.toml`
|
|
16
|
+
(hatchling); requires Python 3.10+.
|
|
17
|
+
- `--base-url` flag and `COSMIC_CRUNCH_BASE_URL` environment variable to override
|
|
18
|
+
the crawl root.
|
|
19
|
+
- `--instrument` flag (default `cosmic`) to select other instrument trees
|
|
20
|
+
(`champ`, `gracea`, `gracefo1`, …).
|
|
21
|
+
- Atomic, resumable downloads: stream to `<name>.part` then rename; skip files
|
|
22
|
+
already present with a matching size.
|
|
23
|
+
- Offline `pytest` suite and a GitHub Actions CI workflow (ruff + pytest on
|
|
24
|
+
Python 3.10–3.13 + `build`/`twine check`).
|
|
25
|
+
- `CHANGELOG.md` and `CITATION.cff`.
|
|
26
|
+
|
|
27
|
+
### Changed
|
|
28
|
+
- **New crawl root** `https://genesis.jpl.nasa.gov/ftp/glevels/` (the v1
|
|
29
|
+
`/ftp/pub/genesis/glevels` root is dead).
|
|
30
|
+
- **Fail loud:** a crawl that finds zero data files now raises and exits
|
|
31
|
+
non-zero instead of silently "succeeding" with nothing downloaded.
|
|
32
|
+
- **Bounded retry:** the v1 infinite silent retry is replaced with a bounded
|
|
33
|
+
retry (default 3 attempts, exponential backoff, logged, re-raises on failure).
|
|
34
|
+
- Relicensed from AGPL-3.0 to **MIT**.
|
|
35
|
+
- Logging is configured only under the CLI, not at import time (importing the
|
|
36
|
+
package no longer creates a stray log file).
|
|
37
|
+
- Deduplicated the copy-pasted `parallelize`/`flatten`/retry helpers into
|
|
38
|
+
`cosmic_crunch/_parallel.py`; single-sourced `__version__`.
|
|
39
|
+
|
|
40
|
+
### Fixed
|
|
41
|
+
- **Security:** removed `eval()` of downloaded header content (arbitrary code
|
|
42
|
+
execution); header values are parsed with `ast.literal_eval` + raw-string
|
|
43
|
+
fallback, preserving brace-set element order.
|
|
44
|
+
- **Security:** path traversal in downloads — a hostile or tampered directory
|
|
45
|
+
listing could smuggle `../` through a filename and write outside the save
|
|
46
|
+
directory. Downloaded filenames are now reduced to a bare leaf and confined
|
|
47
|
+
to the save root (irreducible unsafe names are rejected before any fetch).
|
|
48
|
+
- The v1 `FILES_TO_GET = -1` default silently dropped the last crawled file on
|
|
49
|
+
every non-test run (`[:-1]`); it now defaults to downloading everything.
|
|
50
|
+
- `crawl_convert` processed only the first path argument (a `return` inside the
|
|
51
|
+
per-path loop, plus a `paths` shadowing typo) — it now converts all paths.
|
|
52
|
+
- Column-name loop-leak: rows of DataTypes with differing field counts were
|
|
53
|
+
parsed with the wrong column names — each type now keeps its own columns.
|
|
54
|
+
- Single-file conversion creates its output directory; directory creation uses
|
|
55
|
+
`os.makedirs(exist_ok=True)`.
|
|
56
|
+
|
|
57
|
+
### Migration
|
|
58
|
+
- The invocation changed: `python get_files.py …` / `python convert_files.py …`
|
|
59
|
+
become `cosmic-crunch get …` / `cosmic-crunch convert …`. This is the breaking
|
|
60
|
+
change behind the major version bump. There are no deprecation shims (the tool
|
|
61
|
+
was never published and has no known downstream users).
|
|
62
|
+
|
|
63
|
+
## [1.3.x] — 2020–2021
|
|
64
|
+
|
|
65
|
+
The original two-script tool (unpackaged, AGPL-3.0, never published to PyPI):
|
|
66
|
+
|
|
67
|
+
- `get_files.py` (v1.3.1) — download JPL COSMIC ASCII data files. Created
|
|
68
|
+
2020-12-11, last updated 2021-08-02.
|
|
69
|
+
- `convert_files.py` (v1.3.3) — convert COSMIC ASCII files to netCDF4. Created
|
|
70
|
+
2021-01-28, last updated 2021-08-02.
|
|
71
|
+
|
|
72
|
+
[2.0.0]: https://github.com/ErickShepherd/cosmic-crunch/releases/tag/v2.0.0
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
cff-version: 1.2.0
|
|
2
|
+
message: "If you use this software, please cite it as below."
|
|
3
|
+
title: "cosmic-crunch"
|
|
4
|
+
abstract: >-
|
|
5
|
+
Download JPL GENESIS COSMIC radio-occultation ASCII data files and convert
|
|
6
|
+
them to netCDF4.
|
|
7
|
+
type: software
|
|
8
|
+
authors:
|
|
9
|
+
- family-names: "Shepherd"
|
|
10
|
+
given-names: "Erick"
|
|
11
|
+
email: "Contact@ErickShepherd.com"
|
|
12
|
+
version: "2.0.0"
|
|
13
|
+
date-released: "2026-07-08"
|
|
14
|
+
license: MIT
|
|
15
|
+
repository-code: "https://github.com/ErickShepherd/cosmic-crunch"
|
|
16
|
+
url: "https://github.com/ErickShepherd/cosmic-crunch"
|
|
17
|
+
keywords:
|
|
18
|
+
- COSMIC
|
|
19
|
+
- GENESIS
|
|
20
|
+
- JPL
|
|
21
|
+
- "radio occultation"
|
|
22
|
+
- netCDF4
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2020-2026 Erick Shepherd
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1,162 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: cosmic-crunch
|
|
3
|
+
Version: 2.0.0
|
|
4
|
+
Summary: Download JPL GENESIS COSMIC radio-occultation ASCII files and convert them to netCDF4.
|
|
5
|
+
Project-URL: Homepage, https://github.com/ErickShepherd/cosmic-crunch
|
|
6
|
+
Author-email: Erick Shepherd <Contact@ErickShepherd.com>
|
|
7
|
+
License-Expression: MIT
|
|
8
|
+
License-File: LICENSE
|
|
9
|
+
Keywords: COSMIC,GENESIS,JPL,netCDF4,radio occultation
|
|
10
|
+
Requires-Python: >=3.10
|
|
11
|
+
Requires-Dist: netcdf4>=1.6
|
|
12
|
+
Requires-Dist: pandas>=2.0
|
|
13
|
+
Requires-Dist: requests>=2.28
|
|
14
|
+
Requires-Dist: tqdm>=4.60
|
|
15
|
+
Provides-Extra: test
|
|
16
|
+
Requires-Dist: pytest; extra == 'test'
|
|
17
|
+
Description-Content-Type: text/markdown
|
|
18
|
+
|
|
19
|
+
# cosmic-crunch
|
|
20
|
+
|
|
21
|
+
[](https://github.com/ErickShepherd/cosmic-crunch/actions/workflows/ci.yml)
|
|
22
|
+
[](LICENSE)
|
|
23
|
+
[](pyproject.toml)
|
|
24
|
+
|
|
25
|
+
Download JPL GENESIS **COSMIC** radio-occultation ASCII data files and convert
|
|
26
|
+
them to **netCDF4**.
|
|
27
|
+
|
|
28
|
+
`cosmic-crunch` crawls the [JPL GENESIS](https://genesis.jpl.nasa.gov) data
|
|
29
|
+
archive, downloads the Level-2 ASCII occultation profiles, and (optionally)
|
|
30
|
+
converts them into self-describing netCDF4 files.
|
|
31
|
+
|
|
32
|
+
> **Site restructure note (2020 → 2026).** JPL restructured the GENESIS site: the
|
|
33
|
+
> old crawl root (`/ftp/pub/genesis/glevels`) is dead, so v1 of this tool silently
|
|
34
|
+
> "succeeded" while downloading nothing. v2 targets the current root
|
|
35
|
+
> `https://genesis.jpl.nasa.gov/ftp/glevels/` and **fails loudly** — a crawl that
|
|
36
|
+
> finds nothing exits non-zero instead of pretending to succeed.
|
|
37
|
+
>
|
|
38
|
+
> **Mission status.** COSMIC-1 (FORMOSAT-3, flight modules FM1–FM6, served here as
|
|
39
|
+
> `cosmic1/`–`cosmic6/`) was decommissioned in 2020, so this is a **static
|
|
40
|
+
> archive** — the data has stopped changing. (COSMIC-2 is a different mission on a
|
|
41
|
+
> different archive and is out of scope.)
|
|
42
|
+
|
|
43
|
+
## Installation
|
|
44
|
+
|
|
45
|
+
```bash
|
|
46
|
+
pip install -e .
|
|
47
|
+
```
|
|
48
|
+
|
|
49
|
+
This installs the `cosmic-crunch` command with two subcommands, `get` and
|
|
50
|
+
`convert`. Python 3.10+ is required.
|
|
51
|
+
|
|
52
|
+
## Usage
|
|
53
|
+
|
|
54
|
+
`cosmic-crunch` has a single entry point with two subcommands:
|
|
55
|
+
|
|
56
|
+
```bash
|
|
57
|
+
cosmic-crunch get # crawl the GENESIS site and download ASCII files
|
|
58
|
+
cosmic-crunch convert # convert downloaded ASCII files to netCDF4
|
|
59
|
+
```
|
|
60
|
+
|
|
61
|
+
### `cosmic-crunch get`
|
|
62
|
+
|
|
63
|
+
```
|
|
64
|
+
usage: cosmic-crunch get [-h] [--base-url BASE_URL] [--instrument INSTRUMENT]
|
|
65
|
+
[--year_regex YEAR_REGEX] [--date_regex DATE_REGEX]
|
|
66
|
+
[--processes PROCESSES] [--test] [--netcdf4]
|
|
67
|
+
[--skip_empty]
|
|
68
|
+
```
|
|
69
|
+
|
|
70
|
+
| flag | description |
|
|
71
|
+
|------|-------------|
|
|
72
|
+
| `--base-url` | Override the crawl root. Precedence: flag > `COSMIC_CRUNCH_BASE_URL` env var > built-in default (`https://genesis.jpl.nasa.gov/ftp/glevels`). |
|
|
73
|
+
| `--instrument` | Instrument tree to crawl (substring filter). Defaults to `cosmic` (matches `cosmic1`–`cosmic6`); the same archive also serves `champ`, `gracea`, `gracefo1`, … |
|
|
74
|
+
| `--year_regex` | Download only years matching this regular expression. |
|
|
75
|
+
| `--date_regex` | Download only dates matching this regular expression. |
|
|
76
|
+
| `--processes` | Worker processes for the `multiprocessing` pool (default `1`). |
|
|
77
|
+
| `--test` | Download a small subset (cosmic1, 2019-01-03, 10 files) as a smoke test. |
|
|
78
|
+
| `--netcdf4` | Convert the downloaded ASCII files to netCDF4 afterward. |
|
|
79
|
+
| `--skip_empty` | Skip converting files whose data arrays are all empty. |
|
|
80
|
+
|
|
81
|
+
Downloads are **atomic and resumable**: each file is streamed to a `.part`
|
|
82
|
+
temporary and renamed into place only once complete, and files already present
|
|
83
|
+
with a matching size are skipped — so an interrupted bulk pull can simply be
|
|
84
|
+
re-run.
|
|
85
|
+
|
|
86
|
+
A successful run resembles:
|
|
87
|
+
|
|
88
|
+
```
|
|
89
|
+
$ cosmic-crunch get --year_regex=2006 --date_regex=2006-05-02 --netcdf4 --skip_empty --processes=4
|
|
90
|
+
Crawling all ./cosmic<#>/postproc: 100%|████████████████████| 6/6 [00:03<00:00, 1.61it/s]
|
|
91
|
+
Crawling all ./cosmic<#>/.../<year>: 100%|██████████████████| 6/6 [00:03<00:00, 1.59it/s]
|
|
92
|
+
Crawling all ./cosmic<#>/.../<date>: 100%|██████████████████| 3/3 [00:03<00:00, 1.17s/it]
|
|
93
|
+
Crawling all ./cosmic<#>/.../L2/<format>: 100%|█████████████| 4/4 [00:04<00:00, 1.09s/it]
|
|
94
|
+
Downloading data files: 100%|███████████████████████████████| 20/20 [00:26<00:00, 1.33s/it]
|
|
95
|
+
Converting ASCII to netCDF4: 100%|██████████████████████████| 20/20 [00:03<00:00, 6.32it/s]
|
|
96
|
+
|
|
97
|
+
ASCII to netCDF4 conversion summary:
|
|
98
|
+
- Successful conversions: 17
|
|
99
|
+
- Skipped conversions: 3
|
|
100
|
+
- Conversion errors: 0
|
|
101
|
+
- Total number of files: 20
|
|
102
|
+
```
|
|
103
|
+
|
|
104
|
+
Downloaded files are written under `./jpl_cosmic/<year>/<date>/txt/`.
|
|
105
|
+
|
|
106
|
+
### `cosmic-crunch convert`
|
|
107
|
+
|
|
108
|
+
```
|
|
109
|
+
usage: cosmic-crunch convert [-h] [--logfile LOGFILE] [--processes PROCESSES]
|
|
110
|
+
[--skip_empty]
|
|
111
|
+
path [path ...]
|
|
112
|
+
```
|
|
113
|
+
|
|
114
|
+
Convert one or more ASCII `.txt.gz` files — or directories of them (crawled
|
|
115
|
+
recursively) — to netCDF4. netCDF4 files are written into a sibling `nc/`
|
|
116
|
+
directory (mirroring the `txt/` layout), or beside the source file otherwise.
|
|
117
|
+
|
|
118
|
+
```bash
|
|
119
|
+
cosmic-crunch convert ./jpl_cosmic/2006/ --skip_empty --processes=4
|
|
120
|
+
```
|
|
121
|
+
|
|
122
|
+
## netCDF4 output structure
|
|
123
|
+
|
|
124
|
+
Each ASCII file becomes one netCDF4 file. The ASCII header becomes **global
|
|
125
|
+
attributes**; each DataType profile becomes a **group** whose variables are that
|
|
126
|
+
profile's columns, indexed by an `Index` dimension:
|
|
127
|
+
|
|
128
|
+
```
|
|
129
|
+
netcdf 20060501_0632co1_g35_2p6.L2 {
|
|
130
|
+
// global attributes: ProductCreationTime, ShortName, DataSetID,
|
|
131
|
+
// PlatformShortName, Receiver, ... (the ASCII header)
|
|
132
|
+
|
|
133
|
+
group: COSMIC1-Profile {
|
|
134
|
+
dimensions: Index = <n> ;
|
|
135
|
+
variables: Height, Lat, Lon, Refractivity, Temperature, Pressure, WV Pressure ;
|
|
136
|
+
}
|
|
137
|
+
group: ECMWF-Profile {
|
|
138
|
+
dimensions: Index = <n> ;
|
|
139
|
+
variables: Height, Lat, Lon, Refractivity, Temperature, Pressure, WV Pressure ;
|
|
140
|
+
}
|
|
141
|
+
}
|
|
142
|
+
```
|
|
143
|
+
|
|
144
|
+
Missing values in the ASCII data (`-9999`) are stored as `NaN`.
|
|
145
|
+
|
|
146
|
+
## Security note
|
|
147
|
+
|
|
148
|
+
v1 parsed header values with `eval()`, allowing arbitrary code execution from a
|
|
149
|
+
malicious or corrupted data file. v2 uses `ast.literal_eval` with a raw-string
|
|
150
|
+
fallback — header parsing can no longer execute code.
|
|
151
|
+
|
|
152
|
+
## Development
|
|
153
|
+
|
|
154
|
+
```bash
|
|
155
|
+
pip install -e ".[test]"
|
|
156
|
+
python -m pytest -q # offline test suite (never touches the network)
|
|
157
|
+
ruff check .
|
|
158
|
+
```
|
|
159
|
+
|
|
160
|
+
## License
|
|
161
|
+
|
|
162
|
+
MIT — see [LICENSE](LICENSE).
|
|
@@ -0,0 +1,144 @@
|
|
|
1
|
+
# cosmic-crunch
|
|
2
|
+
|
|
3
|
+
[](https://github.com/ErickShepherd/cosmic-crunch/actions/workflows/ci.yml)
|
|
4
|
+
[](LICENSE)
|
|
5
|
+
[](pyproject.toml)
|
|
6
|
+
|
|
7
|
+
Download JPL GENESIS **COSMIC** radio-occultation ASCII data files and convert
|
|
8
|
+
them to **netCDF4**.
|
|
9
|
+
|
|
10
|
+
`cosmic-crunch` crawls the [JPL GENESIS](https://genesis.jpl.nasa.gov) data
|
|
11
|
+
archive, downloads the Level-2 ASCII occultation profiles, and (optionally)
|
|
12
|
+
converts them into self-describing netCDF4 files.
|
|
13
|
+
|
|
14
|
+
> **Site restructure note (2020 → 2026).** JPL restructured the GENESIS site: the
|
|
15
|
+
> old crawl root (`/ftp/pub/genesis/glevels`) is dead, so v1 of this tool silently
|
|
16
|
+
> "succeeded" while downloading nothing. v2 targets the current root
|
|
17
|
+
> `https://genesis.jpl.nasa.gov/ftp/glevels/` and **fails loudly** — a crawl that
|
|
18
|
+
> finds nothing exits non-zero instead of pretending to succeed.
|
|
19
|
+
>
|
|
20
|
+
> **Mission status.** COSMIC-1 (FORMOSAT-3, flight modules FM1–FM6, served here as
|
|
21
|
+
> `cosmic1/`–`cosmic6/`) was decommissioned in 2020, so this is a **static
|
|
22
|
+
> archive** — the data has stopped changing. (COSMIC-2 is a different mission on a
|
|
23
|
+
> different archive and is out of scope.)
|
|
24
|
+
|
|
25
|
+
## Installation
|
|
26
|
+
|
|
27
|
+
```bash
|
|
28
|
+
pip install -e .
|
|
29
|
+
```
|
|
30
|
+
|
|
31
|
+
This installs the `cosmic-crunch` command with two subcommands, `get` and
|
|
32
|
+
`convert`. Python 3.10+ is required.
|
|
33
|
+
|
|
34
|
+
## Usage
|
|
35
|
+
|
|
36
|
+
`cosmic-crunch` has a single entry point with two subcommands:
|
|
37
|
+
|
|
38
|
+
```bash
|
|
39
|
+
cosmic-crunch get # crawl the GENESIS site and download ASCII files
|
|
40
|
+
cosmic-crunch convert # convert downloaded ASCII files to netCDF4
|
|
41
|
+
```
|
|
42
|
+
|
|
43
|
+
### `cosmic-crunch get`
|
|
44
|
+
|
|
45
|
+
```
|
|
46
|
+
usage: cosmic-crunch get [-h] [--base-url BASE_URL] [--instrument INSTRUMENT]
|
|
47
|
+
[--year_regex YEAR_REGEX] [--date_regex DATE_REGEX]
|
|
48
|
+
[--processes PROCESSES] [--test] [--netcdf4]
|
|
49
|
+
[--skip_empty]
|
|
50
|
+
```
|
|
51
|
+
|
|
52
|
+
| flag | description |
|
|
53
|
+
|------|-------------|
|
|
54
|
+
| `--base-url` | Override the crawl root. Precedence: flag > `COSMIC_CRUNCH_BASE_URL` env var > built-in default (`https://genesis.jpl.nasa.gov/ftp/glevels`). |
|
|
55
|
+
| `--instrument` | Instrument tree to crawl (substring filter). Defaults to `cosmic` (matches `cosmic1`–`cosmic6`); the same archive also serves `champ`, `gracea`, `gracefo1`, … |
|
|
56
|
+
| `--year_regex` | Download only years matching this regular expression. |
|
|
57
|
+
| `--date_regex` | Download only dates matching this regular expression. |
|
|
58
|
+
| `--processes` | Worker processes for the `multiprocessing` pool (default `1`). |
|
|
59
|
+
| `--test` | Download a small subset (cosmic1, 2019-01-03, 10 files) as a smoke test. |
|
|
60
|
+
| `--netcdf4` | Convert the downloaded ASCII files to netCDF4 afterward. |
|
|
61
|
+
| `--skip_empty` | Skip converting files whose data arrays are all empty. |
|
|
62
|
+
|
|
63
|
+
Downloads are **atomic and resumable**: each file is streamed to a `.part`
|
|
64
|
+
temporary and renamed into place only once complete, and files already present
|
|
65
|
+
with a matching size are skipped — so an interrupted bulk pull can simply be
|
|
66
|
+
re-run.
|
|
67
|
+
|
|
68
|
+
A successful run resembles:
|
|
69
|
+
|
|
70
|
+
```
|
|
71
|
+
$ cosmic-crunch get --year_regex=2006 --date_regex=2006-05-02 --netcdf4 --skip_empty --processes=4
|
|
72
|
+
Crawling all ./cosmic<#>/postproc: 100%|████████████████████| 6/6 [00:03<00:00, 1.61it/s]
|
|
73
|
+
Crawling all ./cosmic<#>/.../<year>: 100%|██████████████████| 6/6 [00:03<00:00, 1.59it/s]
|
|
74
|
+
Crawling all ./cosmic<#>/.../<date>: 100%|██████████████████| 3/3 [00:03<00:00, 1.17s/it]
|
|
75
|
+
Crawling all ./cosmic<#>/.../L2/<format>: 100%|█████████████| 4/4 [00:04<00:00, 1.09s/it]
|
|
76
|
+
Downloading data files: 100%|███████████████████████████████| 20/20 [00:26<00:00, 1.33s/it]
|
|
77
|
+
Converting ASCII to netCDF4: 100%|██████████████████████████| 20/20 [00:03<00:00, 6.32it/s]
|
|
78
|
+
|
|
79
|
+
ASCII to netCDF4 conversion summary:
|
|
80
|
+
- Successful conversions: 17
|
|
81
|
+
- Skipped conversions: 3
|
|
82
|
+
- Conversion errors: 0
|
|
83
|
+
- Total number of files: 20
|
|
84
|
+
```
|
|
85
|
+
|
|
86
|
+
Downloaded files are written under `./jpl_cosmic/<year>/<date>/txt/`.
|
|
87
|
+
|
|
88
|
+
### `cosmic-crunch convert`
|
|
89
|
+
|
|
90
|
+
```
|
|
91
|
+
usage: cosmic-crunch convert [-h] [--logfile LOGFILE] [--processes PROCESSES]
|
|
92
|
+
[--skip_empty]
|
|
93
|
+
path [path ...]
|
|
94
|
+
```
|
|
95
|
+
|
|
96
|
+
Convert one or more ASCII `.txt.gz` files — or directories of them (crawled
|
|
97
|
+
recursively) — to netCDF4. netCDF4 files are written into a sibling `nc/`
|
|
98
|
+
directory (mirroring the `txt/` layout), or beside the source file otherwise.
|
|
99
|
+
|
|
100
|
+
```bash
|
|
101
|
+
cosmic-crunch convert ./jpl_cosmic/2006/ --skip_empty --processes=4
|
|
102
|
+
```
|
|
103
|
+
|
|
104
|
+
## netCDF4 output structure
|
|
105
|
+
|
|
106
|
+
Each ASCII file becomes one netCDF4 file. The ASCII header becomes **global
|
|
107
|
+
attributes**; each DataType profile becomes a **group** whose variables are that
|
|
108
|
+
profile's columns, indexed by an `Index` dimension:
|
|
109
|
+
|
|
110
|
+
```
|
|
111
|
+
netcdf 20060501_0632co1_g35_2p6.L2 {
|
|
112
|
+
// global attributes: ProductCreationTime, ShortName, DataSetID,
|
|
113
|
+
// PlatformShortName, Receiver, ... (the ASCII header)
|
|
114
|
+
|
|
115
|
+
group: COSMIC1-Profile {
|
|
116
|
+
dimensions: Index = <n> ;
|
|
117
|
+
variables: Height, Lat, Lon, Refractivity, Temperature, Pressure, WV Pressure ;
|
|
118
|
+
}
|
|
119
|
+
group: ECMWF-Profile {
|
|
120
|
+
dimensions: Index = <n> ;
|
|
121
|
+
variables: Height, Lat, Lon, Refractivity, Temperature, Pressure, WV Pressure ;
|
|
122
|
+
}
|
|
123
|
+
}
|
|
124
|
+
```
|
|
125
|
+
|
|
126
|
+
Missing values in the ASCII data (`-9999`) are stored as `NaN`.
|
|
127
|
+
|
|
128
|
+
## Security note
|
|
129
|
+
|
|
130
|
+
v1 parsed header values with `eval()`, allowing arbitrary code execution from a
|
|
131
|
+
malicious or corrupted data file. v2 uses `ast.literal_eval` with a raw-string
|
|
132
|
+
fallback — header parsing can no longer execute code.
|
|
133
|
+
|
|
134
|
+
## Development
|
|
135
|
+
|
|
136
|
+
```bash
|
|
137
|
+
pip install -e ".[test]"
|
|
138
|
+
python -m pytest -q # offline test suite (never touches the network)
|
|
139
|
+
ruff check .
|
|
140
|
+
```
|
|
141
|
+
|
|
142
|
+
## License
|
|
143
|
+
|
|
144
|
+
MIT — see [LICENSE](LICENSE).
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
# -*- coding: utf-8 -*-
|
|
3
|
+
|
|
4
|
+
'''
|
|
5
|
+
|
|
6
|
+
cosmic_crunch: download JPL GENESIS COSMIC radio-occultation ASCII files and
|
|
7
|
+
convert them to netCDF4.
|
|
8
|
+
|
|
9
|
+
The single source of truth for the package version lives here (``__version__``)
|
|
10
|
+
and is read at build time by hatchling (see ``pyproject.toml`` ->
|
|
11
|
+
``[tool.hatch.version]``).
|
|
12
|
+
|
|
13
|
+
Kept intentionally light: importing the package must not import the ``fetch`` or
|
|
14
|
+
``convert`` submodules, so that ``import cosmic_crunch`` has no side effects.
|
|
15
|
+
|
|
16
|
+
'''
|
|
17
|
+
|
|
18
|
+
__author__ = "Erick Edward Shepherd"
|
|
19
|
+
__version__ = "2.0.0"
|