flylight 0.12.4__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2026 Gustavo Madeira Santana
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
@@ -0,0 +1,137 @@
1
+ Metadata-Version: 2.4
2
+ Name: flylight
3
+ Version: 0.12.4
4
+ Summary: Agent-friendly local index for Janelia FlyLight Split-GAL4 resources
5
+ Author: Gustavo Madeira Santana
6
+ License-Expression: MIT
7
+ Requires-Python: >=3.11
8
+ Description-Content-Type: text/markdown
9
+ License-File: LICENSE
10
+ Dynamic: license-file
11
+
12
+ # flylight
13
+
14
+ Agent-friendly local index for Janelia FlyLight Split-GAL4 resources.
15
+
16
+ Surfaces used:
17
+
18
+ - CGI site: [splitgal4.janelia.org](https://splitgal4.janelia.org/cgi-bin/splitgal4.cgi)
19
+ - Public bucket: [janelia-flylight-imagery](https://s3.amazonaws.com/janelia-flylight-imagery/)
20
+ - Bucket docs: [README.md](https://s3.amazonaws.com/janelia-flylight-imagery/README.md)
21
+
22
+ Source order:
23
+
24
+ 1. release manifest json
25
+ 2. per-line/per-image S3 metadata json
26
+ 3. CGI release summary html
27
+
28
+ ## CLI
29
+
30
+ PyPI with `pipx`:
31
+
32
+ ```bash
33
+ pipx install flylight
34
+ ```
35
+
36
+ PyPI with plain `pip`:
37
+
38
+ ```bash
39
+ python3 -m pip install flylight
40
+ ```
41
+
42
+ Homebrew:
43
+
44
+ ```bash
45
+ brew tap gumadeiras/tap
46
+ brew install flylight
47
+ flylight --version
48
+ ```
49
+
50
+ From source:
51
+
52
+ ```bash
53
+ python3 -m pip install -e .
54
+ ```
55
+
56
+ ## Release
57
+
58
+ Current release: `v0.12.2`.
59
+
60
+ Tag pushes like `vX.Y.Z` run the release workflow: build artifacts, create a
61
+ GitHub release, publish to PyPI, and update `gumadeiras/homebrew-tap`.
62
+
63
+ Release prerequisites:
64
+
65
+ - PyPI trusted publishing configured for this repo.
66
+ - `HOMEBREW_TAP_TOKEN` repository secret can write to
67
+ `gumadeiras/homebrew-tap`.
68
+
69
+ Entry:
70
+
71
+ ```bash
72
+ flylight --help
73
+ python3 janelia_splitgal4.py
74
+ ```
75
+
76
+ Examples:
77
+
78
+ ```bash
79
+ flylight releases
80
+ flylight sync --release 'MB Paper 2014'
81
+ flylight sync --all
82
+ flylight sync --all --force
83
+ flylight sync --all --offline
84
+ flylight sync --all --refresh-cache
85
+ flylight sync --release 'Descending Neurons 2018' --workers 8
86
+ flylight sync-plan --all
87
+ flylight sync-plan --release 'MB Paper 2014' --offline
88
+ flylight reindex
89
+ flylight cache-info
90
+ flylight schema --entity line
91
+ flylight examples --topic release-diff
92
+ flylight snapshot-export --out data/flylight-snapshot.tar.gz
93
+ flylight snapshot-import data/flylight-snapshot.tar.gz --force
94
+ flylight search --expressed-in DNp04 --ad 31B08 --source-kind line-metadata
95
+ flylight search --em-cell-type EPG
96
+ flylight search-text 'DNp04 AND 31B08'
97
+ flylight search-images --area Brain --objective 20x --robot-id 3007645
98
+ flylight search-images --em-cell-type EPG
99
+ flylight show-line SS00724 --release 'Descending Neurons 2018'
100
+ flylight show-image 6878306
101
+ flylight compare-line MB005B
102
+ flylight compare-release 'MB Paper 2014' 'MB Paper 2015'
103
+ flylight show-release 'MB Paper 2014' --include-lines --genotype 34A03
104
+ flylight stats
105
+ flylight export-ndjson --entity line --release 'Descending Neurons 2018'
106
+ flylight export-ndjson --entity image --term MB005B --out data/mb005b.ndjson
107
+ flylight export-ndjson --entity release
108
+ flylight export-ndjson --entity compare-line --line MB005B
109
+ flylight export-ndjson --entity compare-release --left-release 'MB Paper 2014' --right-release 'MB Paper 2015'
110
+ ```
111
+
112
+ ## Notes
113
+
114
+ - HTTP fetches are cache-first by default; cached responses are reused until you pass `--refresh-cache`.
115
+ - `--offline` disables network access and uses cached HTTP responses only.
116
+ - cache path: `data/http_cache`
117
+ - `sync-plan` is a dry-run: source kind, cache coverage, db coverage, skip vs sync decision.
118
+ - `reindex` rebuilds derived searchable fields from stored raw image payloads; use it after schema upgrades on an existing db.
119
+ - `snapshot-export` bundles sqlite + raw manifests + HTTP cache for portable offline reuse.
120
+ - `snapshot-import` restores that bundle on another machine or working copy.
121
+ - `schema` shows agent-facing row shapes and producer commands.
122
+ - `examples` shows canned command recipes for common agent workflows.
123
+ - `sync --all` is incremental by default; unchanged releases skip.
124
+ - `cache-info` also reports cache suffix counts and oldest/newest cached timestamps.
125
+ - missing release manifest: fallback walks line dirs + metadata jsons.
126
+ - CGI summary enriches line-level fields like expressed-in, genotype, AD, DBD.
127
+ - line/image exports include normalized arrays alongside text fields.
128
+ - line/image records now include normalized `em_cell_types` derived from raw `em_cell_type` metadata when present.
129
+ - `search` supports field filters over line metadata: AD, DBD, genotype, expressed-in, robot-id, source-kind.
130
+ - `search` and `search-images` also support exact `--em-cell-type` matching.
131
+ - `search-text` uses SQLite FTS for faster boolean/full-text matching over line text fields.
132
+ - `search-images` supports field filters over image metadata: area, objective, gender, robot-id, roi.
133
+ - `compare-line` shows shared fields for the same line across synced releases.
134
+ - `compare-release` summarizes added, removed, changed, and unchanged lines between two synced releases.
135
+ - `export-ndjson` also supports `compare-line` and `compare-release` entities for agent ingest.
136
+ - local db path: `data/janelia_splitgal4.sqlite`
137
+ - raw manifest cache: `data/raw_manifests/*.json`
@@ -0,0 +1,126 @@
1
+ # flylight
2
+
3
+ Agent-friendly local index for Janelia FlyLight Split-GAL4 resources.
4
+
5
+ Surfaces used:
6
+
7
+ - CGI site: [splitgal4.janelia.org](https://splitgal4.janelia.org/cgi-bin/splitgal4.cgi)
8
+ - Public bucket: [janelia-flylight-imagery](https://s3.amazonaws.com/janelia-flylight-imagery/)
9
+ - Bucket docs: [README.md](https://s3.amazonaws.com/janelia-flylight-imagery/README.md)
10
+
11
+ Source order:
12
+
13
+ 1. release manifest json
14
+ 2. per-line/per-image S3 metadata json
15
+ 3. CGI release summary html
16
+
17
+ ## CLI
18
+
19
+ PyPI with `pipx`:
20
+
21
+ ```bash
22
+ pipx install flylight
23
+ ```
24
+
25
+ PyPI with plain `pip`:
26
+
27
+ ```bash
28
+ python3 -m pip install flylight
29
+ ```
30
+
31
+ Homebrew:
32
+
33
+ ```bash
34
+ brew tap gumadeiras/tap
35
+ brew install flylight
36
+ flylight --version
37
+ ```
38
+
39
+ From source:
40
+
41
+ ```bash
42
+ python3 -m pip install -e .
43
+ ```
44
+
45
+ ## Release
46
+
47
+ Current release: `v0.12.2`.
48
+
49
+ Tag pushes like `vX.Y.Z` run the release workflow: build artifacts, create a
50
+ GitHub release, publish to PyPI, and update `gumadeiras/homebrew-tap`.
51
+
52
+ Release prerequisites:
53
+
54
+ - PyPI trusted publishing configured for this repo.
55
+ - `HOMEBREW_TAP_TOKEN` repository secret can write to
56
+ `gumadeiras/homebrew-tap`.
57
+
58
+ Entry:
59
+
60
+ ```bash
61
+ flylight --help
62
+ python3 janelia_splitgal4.py
63
+ ```
64
+
65
+ Examples:
66
+
67
+ ```bash
68
+ flylight releases
69
+ flylight sync --release 'MB Paper 2014'
70
+ flylight sync --all
71
+ flylight sync --all --force
72
+ flylight sync --all --offline
73
+ flylight sync --all --refresh-cache
74
+ flylight sync --release 'Descending Neurons 2018' --workers 8
75
+ flylight sync-plan --all
76
+ flylight sync-plan --release 'MB Paper 2014' --offline
77
+ flylight reindex
78
+ flylight cache-info
79
+ flylight schema --entity line
80
+ flylight examples --topic release-diff
81
+ flylight snapshot-export --out data/flylight-snapshot.tar.gz
82
+ flylight snapshot-import data/flylight-snapshot.tar.gz --force
83
+ flylight search --expressed-in DNp04 --ad 31B08 --source-kind line-metadata
84
+ flylight search --em-cell-type EPG
85
+ flylight search-text 'DNp04 AND 31B08'
86
+ flylight search-images --area Brain --objective 20x --robot-id 3007645
87
+ flylight search-images --em-cell-type EPG
88
+ flylight show-line SS00724 --release 'Descending Neurons 2018'
89
+ flylight show-image 6878306
90
+ flylight compare-line MB005B
91
+ flylight compare-release 'MB Paper 2014' 'MB Paper 2015'
92
+ flylight show-release 'MB Paper 2014' --include-lines --genotype 34A03
93
+ flylight stats
94
+ flylight export-ndjson --entity line --release 'Descending Neurons 2018'
95
+ flylight export-ndjson --entity image --term MB005B --out data/mb005b.ndjson
96
+ flylight export-ndjson --entity release
97
+ flylight export-ndjson --entity compare-line --line MB005B
98
+ flylight export-ndjson --entity compare-release --left-release 'MB Paper 2014' --right-release 'MB Paper 2015'
99
+ ```
100
+
101
+ ## Notes
102
+
103
+ - HTTP fetches are cache-first by default; cached responses are reused until you pass `--refresh-cache`.
104
+ - `--offline` disables network access and uses cached HTTP responses only.
105
+ - cache path: `data/http_cache`
106
+ - `sync-plan` is a dry-run: source kind, cache coverage, db coverage, skip vs sync decision.
107
+ - `reindex` rebuilds derived searchable fields from stored raw image payloads; use it after schema upgrades on an existing db.
108
+ - `snapshot-export` bundles sqlite + raw manifests + HTTP cache for portable offline reuse.
109
+ - `snapshot-import` restores that bundle on another machine or working copy.
110
+ - `schema` shows agent-facing row shapes and producer commands.
111
+ - `examples` shows canned command recipes for common agent workflows.
112
+ - `sync --all` is incremental by default; unchanged releases skip.
113
+ - `cache-info` also reports cache suffix counts and oldest/newest cached timestamps.
114
+ - missing release manifest: fallback walks line dirs + metadata jsons.
115
+ - CGI summary enriches line-level fields like expressed-in, genotype, AD, DBD.
116
+ - line/image exports include normalized arrays alongside text fields.
117
+ - line/image records now include normalized `em_cell_types` derived from raw `em_cell_type` metadata when present.
118
+ - `search` supports field filters over line metadata: AD, DBD, genotype, expressed-in, robot-id, source-kind.
119
+ - `search` and `search-images` also support exact `--em-cell-type` matching.
120
+ - `search-text` uses SQLite FTS for faster boolean/full-text matching over line text fields.
121
+ - `search-images` supports field filters over image metadata: area, objective, gender, robot-id, roi.
122
+ - `compare-line` shows shared fields for the same line across synced releases.
123
+ - `compare-release` summarizes added, removed, changed, and unchanged lines between two synced releases.
124
+ - `export-ndjson` also supports `compare-line` and `compare-release` entities for agent ingest.
125
+ - local db path: `data/janelia_splitgal4.sqlite`
126
+ - raw manifest cache: `data/raw_manifests/*.json`
@@ -0,0 +1,137 @@
1
+ Metadata-Version: 2.4
2
+ Name: flylight
3
+ Version: 0.12.4
4
+ Summary: Agent-friendly local index for Janelia FlyLight Split-GAL4 resources
5
+ Author: Gustavo Madeira Santana
6
+ License-Expression: MIT
7
+ Requires-Python: >=3.11
8
+ Description-Content-Type: text/markdown
9
+ License-File: LICENSE
10
+ Dynamic: license-file
11
+
12
+ # flylight
13
+
14
+ Agent-friendly local index for Janelia FlyLight Split-GAL4 resources.
15
+
16
+ Surfaces used:
17
+
18
+ - CGI site: [splitgal4.janelia.org](https://splitgal4.janelia.org/cgi-bin/splitgal4.cgi)
19
+ - Public bucket: [janelia-flylight-imagery](https://s3.amazonaws.com/janelia-flylight-imagery/)
20
+ - Bucket docs: [README.md](https://s3.amazonaws.com/janelia-flylight-imagery/README.md)
21
+
22
+ Source order:
23
+
24
+ 1. release manifest json
25
+ 2. per-line/per-image S3 metadata json
26
+ 3. CGI release summary html
27
+
28
+ ## CLI
29
+
30
+ PyPI with `pipx`:
31
+
32
+ ```bash
33
+ pipx install flylight
34
+ ```
35
+
36
+ PyPI with plain `pip`:
37
+
38
+ ```bash
39
+ python3 -m pip install flylight
40
+ ```
41
+
42
+ Homebrew:
43
+
44
+ ```bash
45
+ brew tap gumadeiras/tap
46
+ brew install flylight
47
+ flylight --version
48
+ ```
49
+
50
+ From source:
51
+
52
+ ```bash
53
+ python3 -m pip install -e .
54
+ ```
55
+
56
+ ## Release
57
+
58
+ Current release: `v0.12.2`.
59
+
60
+ Tag pushes like `vX.Y.Z` run the release workflow: build artifacts, create a
61
+ GitHub release, publish to PyPI, and update `gumadeiras/homebrew-tap`.
62
+
63
+ Release prerequisites:
64
+
65
+ - PyPI trusted publishing configured for this repo.
66
+ - `HOMEBREW_TAP_TOKEN` repository secret can write to
67
+ `gumadeiras/homebrew-tap`.
68
+
69
+ Entry:
70
+
71
+ ```bash
72
+ flylight --help
73
+ python3 janelia_splitgal4.py
74
+ ```
75
+
76
+ Examples:
77
+
78
+ ```bash
79
+ flylight releases
80
+ flylight sync --release 'MB Paper 2014'
81
+ flylight sync --all
82
+ flylight sync --all --force
83
+ flylight sync --all --offline
84
+ flylight sync --all --refresh-cache
85
+ flylight sync --release 'Descending Neurons 2018' --workers 8
86
+ flylight sync-plan --all
87
+ flylight sync-plan --release 'MB Paper 2014' --offline
88
+ flylight reindex
89
+ flylight cache-info
90
+ flylight schema --entity line
91
+ flylight examples --topic release-diff
92
+ flylight snapshot-export --out data/flylight-snapshot.tar.gz
93
+ flylight snapshot-import data/flylight-snapshot.tar.gz --force
94
+ flylight search --expressed-in DNp04 --ad 31B08 --source-kind line-metadata
95
+ flylight search --em-cell-type EPG
96
+ flylight search-text 'DNp04 AND 31B08'
97
+ flylight search-images --area Brain --objective 20x --robot-id 3007645
98
+ flylight search-images --em-cell-type EPG
99
+ flylight show-line SS00724 --release 'Descending Neurons 2018'
100
+ flylight show-image 6878306
101
+ flylight compare-line MB005B
102
+ flylight compare-release 'MB Paper 2014' 'MB Paper 2015'
103
+ flylight show-release 'MB Paper 2014' --include-lines --genotype 34A03
104
+ flylight stats
105
+ flylight export-ndjson --entity line --release 'Descending Neurons 2018'
106
+ flylight export-ndjson --entity image --term MB005B --out data/mb005b.ndjson
107
+ flylight export-ndjson --entity release
108
+ flylight export-ndjson --entity compare-line --line MB005B
109
+ flylight export-ndjson --entity compare-release --left-release 'MB Paper 2014' --right-release 'MB Paper 2015'
110
+ ```
111
+
112
+ ## Notes
113
+
114
+ - HTTP fetches are cache-first by default; cached responses are reused until you pass `--refresh-cache`.
115
+ - `--offline` disables network access and uses cached HTTP responses only.
116
+ - cache path: `data/http_cache`
117
+ - `sync-plan` is a dry-run: source kind, cache coverage, db coverage, skip vs sync decision.
118
+ - `reindex` rebuilds derived searchable fields from stored raw image payloads; use it after schema upgrades on an existing db.
119
+ - `snapshot-export` bundles sqlite + raw manifests + HTTP cache for portable offline reuse.
120
+ - `snapshot-import` restores that bundle on another machine or working copy.
121
+ - `schema` shows agent-facing row shapes and producer commands.
122
+ - `examples` shows canned command recipes for common agent workflows.
123
+ - `sync --all` is incremental by default; unchanged releases skip.
124
+ - `cache-info` also reports cache suffix counts and oldest/newest cached timestamps.
125
+ - missing release manifest: fallback walks line dirs + metadata jsons.
126
+ - CGI summary enriches line-level fields like expressed-in, genotype, AD, DBD.
127
+ - line/image exports include normalized arrays alongside text fields.
128
+ - line/image records now include normalized `em_cell_types` derived from raw `em_cell_type` metadata when present.
129
+ - `search` supports field filters over line metadata: AD, DBD, genotype, expressed-in, robot-id, source-kind.
130
+ - `search` and `search-images` also support exact `--em-cell-type` matching.
131
+ - `search-text` uses SQLite FTS for faster boolean/full-text matching over line text fields.
132
+ - `search-images` supports field filters over image metadata: area, objective, gender, robot-id, roi.
133
+ - `compare-line` shows shared fields for the same line across synced releases.
134
+ - `compare-release` summarizes added, removed, changed, and unchanged lines between two synced releases.
135
+ - `export-ndjson` also supports `compare-line` and `compare-release` entities for agent ingest.
136
+ - local db path: `data/janelia_splitgal4.sqlite`
137
+ - raw manifest cache: `data/raw_manifests/*.json`
@@ -0,0 +1,25 @@
1
+ LICENSE
2
+ README.md
3
+ janelia_splitgal4.py
4
+ pyproject.toml
5
+ flylight.egg-info/PKG-INFO
6
+ flylight.egg-info/SOURCES.txt
7
+ flylight.egg-info/dependency_links.txt
8
+ flylight.egg-info/entry_points.txt
9
+ flylight.egg-info/top_level.txt
10
+ flylight_cli/__init__.py
11
+ flylight_cli/__main__.py
12
+ flylight_cli/cache.py
13
+ flylight_cli/cli.py
14
+ flylight_cli/core.py
15
+ flylight_cli/db.py
16
+ flylight_cli/examples.py
17
+ flylight_cli/normalize.py
18
+ flylight_cli/query.py
19
+ flylight_cli/records.py
20
+ flylight_cli/reindex.py
21
+ flylight_cli/schema.py
22
+ flylight_cli/snapshot.py
23
+ flylight_cli/sync_plan.py
24
+ tests/test_cli.py
25
+ tests/test_sync_plan.py
@@ -0,0 +1,2 @@
1
+ [console_scripts]
2
+ flylight = flylight_cli.cli:main
@@ -0,0 +1,2 @@
1
+ flylight_cli
2
+ janelia_splitgal4
@@ -0,0 +1,3 @@
1
+ __all__ = ["__version__"]
2
+
3
+ __version__ = "0.12.4"
@@ -0,0 +1,5 @@
1
+ from .cli import main
2
+
3
+
4
+ if __name__ == "__main__":
5
+ raise SystemExit(main())
@@ -0,0 +1,180 @@
1
+ from __future__ import annotations
2
+
3
+ import hashlib
4
+ import json
5
+ import os
6
+ import tempfile
7
+ from dataclasses import dataclass, replace
8
+ from datetime import datetime, timezone
9
+ from pathlib import Path
10
+ from urllib.parse import urlparse
11
+ from urllib.request import Request, urlopen
12
+
13
+
14
+ DEFAULT_CACHE_DIR = Path("data/http_cache")
15
+
16
+
17
+ @dataclass
18
+ class CacheOptions:
19
+ cache_dir: Path = DEFAULT_CACHE_DIR
20
+ offline: bool = False
21
+ refresh: bool = False
22
+
23
+
24
+ class OfflineCacheMiss(RuntimeError):
25
+ pass
26
+
27
+
28
+ _cache_options = CacheOptions()
29
+
30
+
31
+ def get_cache_options() -> CacheOptions:
32
+ return _cache_options
33
+
34
+
35
+ def set_cache_options(
36
+ cache_dir: Path | None = None,
37
+ offline: bool | None = None,
38
+ refresh: bool | None = None,
39
+ ) -> CacheOptions:
40
+ global _cache_options
41
+ next_options = replace(_cache_options)
42
+ if cache_dir is not None:
43
+ next_options.cache_dir = cache_dir
44
+ if offline is not None:
45
+ next_options.offline = offline
46
+ if refresh is not None:
47
+ next_options.refresh = refresh
48
+ if next_options.offline and next_options.refresh:
49
+ raise ValueError("cannot combine offline mode with refresh-cache")
50
+ _cache_options = next_options
51
+ return _cache_options
52
+
53
+
54
+ def cache_path_for_url(url: str, cache_dir: Path | None = None) -> Path:
55
+ cache_dir = cache_dir or _cache_options.cache_dir
56
+ digest = hashlib.sha256(url.encode("utf-8")).hexdigest()
57
+ suffix = Path(urlparse(url).path).suffix or ".bin"
58
+ return cache_dir / digest[:2] / f"{digest}{suffix}"
59
+
60
+
61
+ def meta_path_for_cache(cache_path: Path) -> Path:
62
+ return cache_path.with_suffix(cache_path.suffix + ".meta.json")
63
+
64
+
65
+ def ensure_cache_parent(path: Path) -> None:
66
+ path.parent.mkdir(parents=True, exist_ok=True)
67
+
68
+
69
+ def load_cached_bytes(url: str, cache_dir: Path | None = None) -> bytes | None:
70
+ cache_path = cache_path_for_url(url, cache_dir=cache_dir)
71
+ if not cache_path.exists():
72
+ return None
73
+ return cache_path.read_bytes()
74
+
75
+
76
+ def cached_at_for_path(cache_path: Path) -> str | None:
77
+ meta_path = meta_path_for_cache(cache_path)
78
+ if not meta_path.exists():
79
+ return None
80
+ meta = json.loads(meta_path.read_text(encoding="utf-8"))
81
+ cached_at = meta.get("cached_at")
82
+ if isinstance(cached_at, str) and cached_at:
83
+ return cached_at
84
+ return None
85
+
86
+
87
+ def cache_data_paths(cache_dir: Path) -> list[Path]:
88
+ return [path for path in cache_dir.rglob("*") if path.is_file() and not path.name.endswith(".meta.json")]
89
+
90
+
91
+ def cache_entry_for_url(url: str, cache_dir: Path | None = None) -> dict[str, int | str] | None:
92
+ cache_path = cache_path_for_url(url, cache_dir=cache_dir)
93
+ if not cache_path.exists():
94
+ return None
95
+ payload: dict[str, int | str] = {
96
+ "url": url,
97
+ "cache_path": str(cache_path),
98
+ "bytes": cache_path.stat().st_size,
99
+ "suffix": cache_path.suffix,
100
+ }
101
+ cached_at = cached_at_for_path(cache_path)
102
+ if cached_at:
103
+ payload["cached_at"] = cached_at
104
+ return payload
105
+
106
+
107
+ def write_atomic_bytes(path: Path, payload: bytes) -> None:
108
+ ensure_cache_parent(path)
109
+ fd, tmp_name = tempfile.mkstemp(
110
+ dir=path.parent,
111
+ prefix=f"{path.name}.",
112
+ suffix=".tmp",
113
+ )
114
+ try:
115
+ with os.fdopen(fd, "wb") as handle:
116
+ handle.write(payload)
117
+ Path(tmp_name).replace(path)
118
+ except Exception:
119
+ Path(tmp_name).unlink(missing_ok=True)
120
+ raise
121
+
122
+
123
+ def write_cached_bytes(url: str, payload: bytes, cache_dir: Path | None = None) -> Path:
124
+ cache_path = cache_path_for_url(url, cache_dir=cache_dir)
125
+ write_atomic_bytes(cache_path, payload)
126
+ meta_payload = json.dumps(
127
+ {
128
+ "url": url,
129
+ "cached_at": datetime.now(timezone.utc).replace(microsecond=0).isoformat(),
130
+ "bytes": len(payload),
131
+ },
132
+ indent=2,
133
+ sort_keys=True,
134
+ )
135
+ write_atomic_bytes(meta_path_for_cache(cache_path), meta_payload.encode("utf-8"))
136
+ return cache_path
137
+
138
+
139
+ def fetch_bytes(url: str, user_agent: str) -> bytes:
140
+ options = get_cache_options()
141
+ cached = None if options.refresh else load_cached_bytes(url, cache_dir=options.cache_dir)
142
+ if cached is not None:
143
+ return cached
144
+ if options.offline:
145
+ raise OfflineCacheMiss(f"offline cache miss for {url}; rerun without --offline to warm the cache")
146
+ req = Request(url, headers={"User-Agent": user_agent})
147
+ with urlopen(req) as resp:
148
+ payload = resp.read()
149
+ write_cached_bytes(url, payload, cache_dir=options.cache_dir)
150
+ return payload
151
+
152
+
153
+ def cache_stats(cache_dir: Path | None = None) -> dict[str, int | str]:
154
+ cache_dir = cache_dir or _cache_options.cache_dir
155
+ if not cache_dir.exists():
156
+ return {
157
+ "cache_dir": str(cache_dir),
158
+ "entries": 0,
159
+ "bytes": 0,
160
+ "suffix_counts": {},
161
+ "oldest_cached_at": None,
162
+ "newest_cached_at": None,
163
+ }
164
+ paths = cache_data_paths(cache_dir)
165
+ suffix_counts: dict[str, int] = {}
166
+ cached_at_values: list[str] = []
167
+ for path in paths:
168
+ suffix = path.suffix or ".bin"
169
+ suffix_counts[suffix] = suffix_counts.get(suffix, 0) + 1
170
+ cached_at = cached_at_for_path(path)
171
+ if cached_at:
172
+ cached_at_values.append(cached_at)
173
+ return {
174
+ "cache_dir": str(cache_dir),
175
+ "entries": len(paths),
176
+ "bytes": sum(path.stat().st_size for path in paths),
177
+ "suffix_counts": suffix_counts,
178
+ "oldest_cached_at": min(cached_at_values) if cached_at_values else None,
179
+ "newest_cached_at": max(cached_at_values) if cached_at_values else None,
180
+ }