flylight 0.12.4__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- flylight-0.12.4/LICENSE +21 -0
- flylight-0.12.4/PKG-INFO +137 -0
- flylight-0.12.4/README.md +126 -0
- flylight-0.12.4/flylight.egg-info/PKG-INFO +137 -0
- flylight-0.12.4/flylight.egg-info/SOURCES.txt +25 -0
- flylight-0.12.4/flylight.egg-info/dependency_links.txt +1 -0
- flylight-0.12.4/flylight.egg-info/entry_points.txt +2 -0
- flylight-0.12.4/flylight.egg-info/top_level.txt +2 -0
- flylight-0.12.4/flylight_cli/__init__.py +3 -0
- flylight-0.12.4/flylight_cli/__main__.py +5 -0
- flylight-0.12.4/flylight_cli/cache.py +180 -0
- flylight-0.12.4/flylight_cli/cli.py +766 -0
- flylight-0.12.4/flylight_cli/core.py +557 -0
- flylight-0.12.4/flylight_cli/db.py +131 -0
- flylight-0.12.4/flylight_cli/examples.py +55 -0
- flylight-0.12.4/flylight_cli/normalize.py +79 -0
- flylight-0.12.4/flylight_cli/query.py +126 -0
- flylight-0.12.4/flylight_cli/records.py +360 -0
- flylight-0.12.4/flylight_cli/reindex.py +75 -0
- flylight-0.12.4/flylight_cli/schema.py +147 -0
- flylight-0.12.4/flylight_cli/snapshot.py +135 -0
- flylight-0.12.4/flylight_cli/sync_plan.py +108 -0
- flylight-0.12.4/janelia_splitgal4.py +6 -0
- flylight-0.12.4/pyproject.toml +25 -0
- flylight-0.12.4/setup.cfg +4 -0
- flylight-0.12.4/tests/test_cli.py +842 -0
- flylight-0.12.4/tests/test_sync_plan.py +120 -0
flylight-0.12.4/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Gustavo Madeira Santana
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
flylight-0.12.4/PKG-INFO
ADDED
|
@@ -0,0 +1,137 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: flylight
|
|
3
|
+
Version: 0.12.4
|
|
4
|
+
Summary: Agent-friendly local index for Janelia FlyLight Split-GAL4 resources
|
|
5
|
+
Author: Gustavo Madeira Santana
|
|
6
|
+
License-Expression: MIT
|
|
7
|
+
Requires-Python: >=3.11
|
|
8
|
+
Description-Content-Type: text/markdown
|
|
9
|
+
License-File: LICENSE
|
|
10
|
+
Dynamic: license-file
|
|
11
|
+
|
|
12
|
+
# flylight
|
|
13
|
+
|
|
14
|
+
Agent-friendly local index for Janelia FlyLight Split-GAL4 resources.
|
|
15
|
+
|
|
16
|
+
Surfaces used:
|
|
17
|
+
|
|
18
|
+
- CGI site: [splitgal4.janelia.org](https://splitgal4.janelia.org/cgi-bin/splitgal4.cgi)
|
|
19
|
+
- Public bucket: [janelia-flylight-imagery](https://s3.amazonaws.com/janelia-flylight-imagery/)
|
|
20
|
+
- Bucket docs: [README.md](https://s3.amazonaws.com/janelia-flylight-imagery/README.md)
|
|
21
|
+
|
|
22
|
+
Source order:
|
|
23
|
+
|
|
24
|
+
1. release manifest json
|
|
25
|
+
2. per-line/per-image S3 metadata json
|
|
26
|
+
3. CGI release summary html
|
|
27
|
+
|
|
28
|
+
## CLI
|
|
29
|
+
|
|
30
|
+
PyPI with `pipx`:
|
|
31
|
+
|
|
32
|
+
```bash
|
|
33
|
+
pipx install flylight
|
|
34
|
+
```
|
|
35
|
+
|
|
36
|
+
PyPI with plain `pip`:
|
|
37
|
+
|
|
38
|
+
```bash
|
|
39
|
+
python3 -m pip install flylight
|
|
40
|
+
```
|
|
41
|
+
|
|
42
|
+
Homebrew:
|
|
43
|
+
|
|
44
|
+
```bash
|
|
45
|
+
brew tap gumadeiras/tap
|
|
46
|
+
brew install flylight
|
|
47
|
+
flylight --version
|
|
48
|
+
```
|
|
49
|
+
|
|
50
|
+
From source:
|
|
51
|
+
|
|
52
|
+
```bash
|
|
53
|
+
python3 -m pip install -e .
|
|
54
|
+
```
|
|
55
|
+
|
|
56
|
+
## Release
|
|
57
|
+
|
|
58
|
+
Current release: `v0.12.2`.
|
|
59
|
+
|
|
60
|
+
Tag pushes like `vX.Y.Z` run the release workflow: build artifacts, create a
|
|
61
|
+
GitHub release, publish to PyPI, and update `gumadeiras/homebrew-tap`.
|
|
62
|
+
|
|
63
|
+
Release prerequisites:
|
|
64
|
+
|
|
65
|
+
- PyPI trusted publishing configured for this repo.
|
|
66
|
+
- `HOMEBREW_TAP_TOKEN` repository secret can write to
|
|
67
|
+
`gumadeiras/homebrew-tap`.
|
|
68
|
+
|
|
69
|
+
Entry:
|
|
70
|
+
|
|
71
|
+
```bash
|
|
72
|
+
flylight --help
|
|
73
|
+
python3 janelia_splitgal4.py
|
|
74
|
+
```
|
|
75
|
+
|
|
76
|
+
Examples:
|
|
77
|
+
|
|
78
|
+
```bash
|
|
79
|
+
flylight releases
|
|
80
|
+
flylight sync --release 'MB Paper 2014'
|
|
81
|
+
flylight sync --all
|
|
82
|
+
flylight sync --all --force
|
|
83
|
+
flylight sync --all --offline
|
|
84
|
+
flylight sync --all --refresh-cache
|
|
85
|
+
flylight sync --release 'Descending Neurons 2018' --workers 8
|
|
86
|
+
flylight sync-plan --all
|
|
87
|
+
flylight sync-plan --release 'MB Paper 2014' --offline
|
|
88
|
+
flylight reindex
|
|
89
|
+
flylight cache-info
|
|
90
|
+
flylight schema --entity line
|
|
91
|
+
flylight examples --topic release-diff
|
|
92
|
+
flylight snapshot-export --out data/flylight-snapshot.tar.gz
|
|
93
|
+
flylight snapshot-import data/flylight-snapshot.tar.gz --force
|
|
94
|
+
flylight search --expressed-in DNp04 --ad 31B08 --source-kind line-metadata
|
|
95
|
+
flylight search --em-cell-type EPG
|
|
96
|
+
flylight search-text 'DNp04 AND 31B08'
|
|
97
|
+
flylight search-images --area Brain --objective 20x --robot-id 3007645
|
|
98
|
+
flylight search-images --em-cell-type EPG
|
|
99
|
+
flylight show-line SS00724 --release 'Descending Neurons 2018'
|
|
100
|
+
flylight show-image 6878306
|
|
101
|
+
flylight compare-line MB005B
|
|
102
|
+
flylight compare-release 'MB Paper 2014' 'MB Paper 2015'
|
|
103
|
+
flylight show-release 'MB Paper 2014' --include-lines --genotype 34A03
|
|
104
|
+
flylight stats
|
|
105
|
+
flylight export-ndjson --entity line --release 'Descending Neurons 2018'
|
|
106
|
+
flylight export-ndjson --entity image --term MB005B --out data/mb005b.ndjson
|
|
107
|
+
flylight export-ndjson --entity release
|
|
108
|
+
flylight export-ndjson --entity compare-line --line MB005B
|
|
109
|
+
flylight export-ndjson --entity compare-release --left-release 'MB Paper 2014' --right-release 'MB Paper 2015'
|
|
110
|
+
```
|
|
111
|
+
|
|
112
|
+
## Notes
|
|
113
|
+
|
|
114
|
+
- HTTP fetches are cache-first by default; cached responses are reused until you pass `--refresh-cache`.
|
|
115
|
+
- `--offline` disables network access and uses cached HTTP responses only.
|
|
116
|
+
- cache path: `data/http_cache`
|
|
117
|
+
- `sync-plan` is a dry-run: source kind, cache coverage, db coverage, skip vs sync decision.
|
|
118
|
+
- `reindex` rebuilds derived searchable fields from stored raw image payloads; use it after schema upgrades on an existing db.
|
|
119
|
+
- `snapshot-export` bundles sqlite + raw manifests + HTTP cache for portable offline reuse.
|
|
120
|
+
- `snapshot-import` restores that bundle on another machine or working copy.
|
|
121
|
+
- `schema` shows agent-facing row shapes and producer commands.
|
|
122
|
+
- `examples` shows canned command recipes for common agent workflows.
|
|
123
|
+
- `sync --all` is incremental by default; unchanged releases skip.
|
|
124
|
+
- `cache-info` also reports cache suffix counts and oldest/newest cached timestamps.
|
|
125
|
+
- missing release manifest: fallback walks line dirs + metadata jsons.
|
|
126
|
+
- CGI summary enriches line-level fields like expressed-in, genotype, AD, DBD.
|
|
127
|
+
- line/image exports include normalized arrays alongside text fields.
|
|
128
|
+
- line/image records now include normalized `em_cell_types` derived from raw `em_cell_type` metadata when present.
|
|
129
|
+
- `search` supports field filters over line metadata: AD, DBD, genotype, expressed-in, robot-id, source-kind.
|
|
130
|
+
- `search` and `search-images` also support exact `--em-cell-type` matching.
|
|
131
|
+
- `search-text` uses SQLite FTS for faster boolean/full-text matching over line text fields.
|
|
132
|
+
- `search-images` supports field filters over image metadata: area, objective, gender, robot-id, roi.
|
|
133
|
+
- `compare-line` shows shared fields for the same line across synced releases.
|
|
134
|
+
- `compare-release` summarizes added, removed, changed, and unchanged lines between two synced releases.
|
|
135
|
+
- `export-ndjson` also supports `compare-line` and `compare-release` entities for agent ingest.
|
|
136
|
+
- local db path: `data/janelia_splitgal4.sqlite`
|
|
137
|
+
- raw manifest cache: `data/raw_manifests/*.json`
|
|
@@ -0,0 +1,126 @@
|
|
|
1
|
+
# flylight
|
|
2
|
+
|
|
3
|
+
Agent-friendly local index for Janelia FlyLight Split-GAL4 resources.
|
|
4
|
+
|
|
5
|
+
Surfaces used:
|
|
6
|
+
|
|
7
|
+
- CGI site: [splitgal4.janelia.org](https://splitgal4.janelia.org/cgi-bin/splitgal4.cgi)
|
|
8
|
+
- Public bucket: [janelia-flylight-imagery](https://s3.amazonaws.com/janelia-flylight-imagery/)
|
|
9
|
+
- Bucket docs: [README.md](https://s3.amazonaws.com/janelia-flylight-imagery/README.md)
|
|
10
|
+
|
|
11
|
+
Source order:
|
|
12
|
+
|
|
13
|
+
1. release manifest json
|
|
14
|
+
2. per-line/per-image S3 metadata json
|
|
15
|
+
3. CGI release summary html
|
|
16
|
+
|
|
17
|
+
## CLI
|
|
18
|
+
|
|
19
|
+
PyPI with `pipx`:
|
|
20
|
+
|
|
21
|
+
```bash
|
|
22
|
+
pipx install flylight
|
|
23
|
+
```
|
|
24
|
+
|
|
25
|
+
PyPI with plain `pip`:
|
|
26
|
+
|
|
27
|
+
```bash
|
|
28
|
+
python3 -m pip install flylight
|
|
29
|
+
```
|
|
30
|
+
|
|
31
|
+
Homebrew:
|
|
32
|
+
|
|
33
|
+
```bash
|
|
34
|
+
brew tap gumadeiras/tap
|
|
35
|
+
brew install flylight
|
|
36
|
+
flylight --version
|
|
37
|
+
```
|
|
38
|
+
|
|
39
|
+
From source:
|
|
40
|
+
|
|
41
|
+
```bash
|
|
42
|
+
python3 -m pip install -e .
|
|
43
|
+
```
|
|
44
|
+
|
|
45
|
+
## Release
|
|
46
|
+
|
|
47
|
+
Current release: `v0.12.2`.
|
|
48
|
+
|
|
49
|
+
Tag pushes like `vX.Y.Z` run the release workflow: build artifacts, create a
|
|
50
|
+
GitHub release, publish to PyPI, and update `gumadeiras/homebrew-tap`.
|
|
51
|
+
|
|
52
|
+
Release prerequisites:
|
|
53
|
+
|
|
54
|
+
- PyPI trusted publishing configured for this repo.
|
|
55
|
+
- `HOMEBREW_TAP_TOKEN` repository secret can write to
|
|
56
|
+
`gumadeiras/homebrew-tap`.
|
|
57
|
+
|
|
58
|
+
Entry:
|
|
59
|
+
|
|
60
|
+
```bash
|
|
61
|
+
flylight --help
|
|
62
|
+
python3 janelia_splitgal4.py
|
|
63
|
+
```
|
|
64
|
+
|
|
65
|
+
Examples:
|
|
66
|
+
|
|
67
|
+
```bash
|
|
68
|
+
flylight releases
|
|
69
|
+
flylight sync --release 'MB Paper 2014'
|
|
70
|
+
flylight sync --all
|
|
71
|
+
flylight sync --all --force
|
|
72
|
+
flylight sync --all --offline
|
|
73
|
+
flylight sync --all --refresh-cache
|
|
74
|
+
flylight sync --release 'Descending Neurons 2018' --workers 8
|
|
75
|
+
flylight sync-plan --all
|
|
76
|
+
flylight sync-plan --release 'MB Paper 2014' --offline
|
|
77
|
+
flylight reindex
|
|
78
|
+
flylight cache-info
|
|
79
|
+
flylight schema --entity line
|
|
80
|
+
flylight examples --topic release-diff
|
|
81
|
+
flylight snapshot-export --out data/flylight-snapshot.tar.gz
|
|
82
|
+
flylight snapshot-import data/flylight-snapshot.tar.gz --force
|
|
83
|
+
flylight search --expressed-in DNp04 --ad 31B08 --source-kind line-metadata
|
|
84
|
+
flylight search --em-cell-type EPG
|
|
85
|
+
flylight search-text 'DNp04 AND 31B08'
|
|
86
|
+
flylight search-images --area Brain --objective 20x --robot-id 3007645
|
|
87
|
+
flylight search-images --em-cell-type EPG
|
|
88
|
+
flylight show-line SS00724 --release 'Descending Neurons 2018'
|
|
89
|
+
flylight show-image 6878306
|
|
90
|
+
flylight compare-line MB005B
|
|
91
|
+
flylight compare-release 'MB Paper 2014' 'MB Paper 2015'
|
|
92
|
+
flylight show-release 'MB Paper 2014' --include-lines --genotype 34A03
|
|
93
|
+
flylight stats
|
|
94
|
+
flylight export-ndjson --entity line --release 'Descending Neurons 2018'
|
|
95
|
+
flylight export-ndjson --entity image --term MB005B --out data/mb005b.ndjson
|
|
96
|
+
flylight export-ndjson --entity release
|
|
97
|
+
flylight export-ndjson --entity compare-line --line MB005B
|
|
98
|
+
flylight export-ndjson --entity compare-release --left-release 'MB Paper 2014' --right-release 'MB Paper 2015'
|
|
99
|
+
```
|
|
100
|
+
|
|
101
|
+
## Notes
|
|
102
|
+
|
|
103
|
+
- HTTP fetches are cache-first by default; cached responses are reused until you pass `--refresh-cache`.
|
|
104
|
+
- `--offline` disables network access and uses cached HTTP responses only.
|
|
105
|
+
- cache path: `data/http_cache`
|
|
106
|
+
- `sync-plan` is a dry-run: source kind, cache coverage, db coverage, skip vs sync decision.
|
|
107
|
+
- `reindex` rebuilds derived searchable fields from stored raw image payloads; use it after schema upgrades on an existing db.
|
|
108
|
+
- `snapshot-export` bundles sqlite + raw manifests + HTTP cache for portable offline reuse.
|
|
109
|
+
- `snapshot-import` restores that bundle on another machine or working copy.
|
|
110
|
+
- `schema` shows agent-facing row shapes and producer commands.
|
|
111
|
+
- `examples` shows canned command recipes for common agent workflows.
|
|
112
|
+
- `sync --all` is incremental by default; unchanged releases skip.
|
|
113
|
+
- `cache-info` also reports cache suffix counts and oldest/newest cached timestamps.
|
|
114
|
+
- missing release manifest: fallback walks line dirs + metadata jsons.
|
|
115
|
+
- CGI summary enriches line-level fields like expressed-in, genotype, AD, DBD.
|
|
116
|
+
- line/image exports include normalized arrays alongside text fields.
|
|
117
|
+
- line/image records now include normalized `em_cell_types` derived from raw `em_cell_type` metadata when present.
|
|
118
|
+
- `search` supports field filters over line metadata: AD, DBD, genotype, expressed-in, robot-id, source-kind.
|
|
119
|
+
- `search` and `search-images` also support exact `--em-cell-type` matching.
|
|
120
|
+
- `search-text` uses SQLite FTS for faster boolean/full-text matching over line text fields.
|
|
121
|
+
- `search-images` supports field filters over image metadata: area, objective, gender, robot-id, roi.
|
|
122
|
+
- `compare-line` shows shared fields for the same line across synced releases.
|
|
123
|
+
- `compare-release` summarizes added, removed, changed, and unchanged lines between two synced releases.
|
|
124
|
+
- `export-ndjson` also supports `compare-line` and `compare-release` entities for agent ingest.
|
|
125
|
+
- local db path: `data/janelia_splitgal4.sqlite`
|
|
126
|
+
- raw manifest cache: `data/raw_manifests/*.json`
|
|
@@ -0,0 +1,137 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: flylight
|
|
3
|
+
Version: 0.12.4
|
|
4
|
+
Summary: Agent-friendly local index for Janelia FlyLight Split-GAL4 resources
|
|
5
|
+
Author: Gustavo Madeira Santana
|
|
6
|
+
License-Expression: MIT
|
|
7
|
+
Requires-Python: >=3.11
|
|
8
|
+
Description-Content-Type: text/markdown
|
|
9
|
+
License-File: LICENSE
|
|
10
|
+
Dynamic: license-file
|
|
11
|
+
|
|
12
|
+
# flylight
|
|
13
|
+
|
|
14
|
+
Agent-friendly local index for Janelia FlyLight Split-GAL4 resources.
|
|
15
|
+
|
|
16
|
+
Surfaces used:
|
|
17
|
+
|
|
18
|
+
- CGI site: [splitgal4.janelia.org](https://splitgal4.janelia.org/cgi-bin/splitgal4.cgi)
|
|
19
|
+
- Public bucket: [janelia-flylight-imagery](https://s3.amazonaws.com/janelia-flylight-imagery/)
|
|
20
|
+
- Bucket docs: [README.md](https://s3.amazonaws.com/janelia-flylight-imagery/README.md)
|
|
21
|
+
|
|
22
|
+
Source order:
|
|
23
|
+
|
|
24
|
+
1. release manifest json
|
|
25
|
+
2. per-line/per-image S3 metadata json
|
|
26
|
+
3. CGI release summary html
|
|
27
|
+
|
|
28
|
+
## CLI
|
|
29
|
+
|
|
30
|
+
PyPI with `pipx`:
|
|
31
|
+
|
|
32
|
+
```bash
|
|
33
|
+
pipx install flylight
|
|
34
|
+
```
|
|
35
|
+
|
|
36
|
+
PyPI with plain `pip`:
|
|
37
|
+
|
|
38
|
+
```bash
|
|
39
|
+
python3 -m pip install flylight
|
|
40
|
+
```
|
|
41
|
+
|
|
42
|
+
Homebrew:
|
|
43
|
+
|
|
44
|
+
```bash
|
|
45
|
+
brew tap gumadeiras/tap
|
|
46
|
+
brew install flylight
|
|
47
|
+
flylight --version
|
|
48
|
+
```
|
|
49
|
+
|
|
50
|
+
From source:
|
|
51
|
+
|
|
52
|
+
```bash
|
|
53
|
+
python3 -m pip install -e .
|
|
54
|
+
```
|
|
55
|
+
|
|
56
|
+
## Release
|
|
57
|
+
|
|
58
|
+
Current release: `v0.12.2`.
|
|
59
|
+
|
|
60
|
+
Tag pushes like `vX.Y.Z` run the release workflow: build artifacts, create a
|
|
61
|
+
GitHub release, publish to PyPI, and update `gumadeiras/homebrew-tap`.
|
|
62
|
+
|
|
63
|
+
Release prerequisites:
|
|
64
|
+
|
|
65
|
+
- PyPI trusted publishing configured for this repo.
|
|
66
|
+
- `HOMEBREW_TAP_TOKEN` repository secret can write to
|
|
67
|
+
`gumadeiras/homebrew-tap`.
|
|
68
|
+
|
|
69
|
+
Entry:
|
|
70
|
+
|
|
71
|
+
```bash
|
|
72
|
+
flylight --help
|
|
73
|
+
python3 janelia_splitgal4.py
|
|
74
|
+
```
|
|
75
|
+
|
|
76
|
+
Examples:
|
|
77
|
+
|
|
78
|
+
```bash
|
|
79
|
+
flylight releases
|
|
80
|
+
flylight sync --release 'MB Paper 2014'
|
|
81
|
+
flylight sync --all
|
|
82
|
+
flylight sync --all --force
|
|
83
|
+
flylight sync --all --offline
|
|
84
|
+
flylight sync --all --refresh-cache
|
|
85
|
+
flylight sync --release 'Descending Neurons 2018' --workers 8
|
|
86
|
+
flylight sync-plan --all
|
|
87
|
+
flylight sync-plan --release 'MB Paper 2014' --offline
|
|
88
|
+
flylight reindex
|
|
89
|
+
flylight cache-info
|
|
90
|
+
flylight schema --entity line
|
|
91
|
+
flylight examples --topic release-diff
|
|
92
|
+
flylight snapshot-export --out data/flylight-snapshot.tar.gz
|
|
93
|
+
flylight snapshot-import data/flylight-snapshot.tar.gz --force
|
|
94
|
+
flylight search --expressed-in DNp04 --ad 31B08 --source-kind line-metadata
|
|
95
|
+
flylight search --em-cell-type EPG
|
|
96
|
+
flylight search-text 'DNp04 AND 31B08'
|
|
97
|
+
flylight search-images --area Brain --objective 20x --robot-id 3007645
|
|
98
|
+
flylight search-images --em-cell-type EPG
|
|
99
|
+
flylight show-line SS00724 --release 'Descending Neurons 2018'
|
|
100
|
+
flylight show-image 6878306
|
|
101
|
+
flylight compare-line MB005B
|
|
102
|
+
flylight compare-release 'MB Paper 2014' 'MB Paper 2015'
|
|
103
|
+
flylight show-release 'MB Paper 2014' --include-lines --genotype 34A03
|
|
104
|
+
flylight stats
|
|
105
|
+
flylight export-ndjson --entity line --release 'Descending Neurons 2018'
|
|
106
|
+
flylight export-ndjson --entity image --term MB005B --out data/mb005b.ndjson
|
|
107
|
+
flylight export-ndjson --entity release
|
|
108
|
+
flylight export-ndjson --entity compare-line --line MB005B
|
|
109
|
+
flylight export-ndjson --entity compare-release --left-release 'MB Paper 2014' --right-release 'MB Paper 2015'
|
|
110
|
+
```
|
|
111
|
+
|
|
112
|
+
## Notes
|
|
113
|
+
|
|
114
|
+
- HTTP fetches are cache-first by default; cached responses are reused until you pass `--refresh-cache`.
|
|
115
|
+
- `--offline` disables network access and uses cached HTTP responses only.
|
|
116
|
+
- cache path: `data/http_cache`
|
|
117
|
+
- `sync-plan` is a dry-run: source kind, cache coverage, db coverage, skip vs sync decision.
|
|
118
|
+
- `reindex` rebuilds derived searchable fields from stored raw image payloads; use it after schema upgrades on an existing db.
|
|
119
|
+
- `snapshot-export` bundles sqlite + raw manifests + HTTP cache for portable offline reuse.
|
|
120
|
+
- `snapshot-import` restores that bundle on another machine or working copy.
|
|
121
|
+
- `schema` shows agent-facing row shapes and producer commands.
|
|
122
|
+
- `examples` shows canned command recipes for common agent workflows.
|
|
123
|
+
- `sync --all` is incremental by default; unchanged releases skip.
|
|
124
|
+
- `cache-info` also reports cache suffix counts and oldest/newest cached timestamps.
|
|
125
|
+
- missing release manifest: fallback walks line dirs + metadata jsons.
|
|
126
|
+
- CGI summary enriches line-level fields like expressed-in, genotype, AD, DBD.
|
|
127
|
+
- line/image exports include normalized arrays alongside text fields.
|
|
128
|
+
- line/image records now include normalized `em_cell_types` derived from raw `em_cell_type` metadata when present.
|
|
129
|
+
- `search` supports field filters over line metadata: AD, DBD, genotype, expressed-in, robot-id, source-kind.
|
|
130
|
+
- `search` and `search-images` also support exact `--em-cell-type` matching.
|
|
131
|
+
- `search-text` uses SQLite FTS for faster boolean/full-text matching over line text fields.
|
|
132
|
+
- `search-images` supports field filters over image metadata: area, objective, gender, robot-id, roi.
|
|
133
|
+
- `compare-line` shows shared fields for the same line across synced releases.
|
|
134
|
+
- `compare-release` summarizes added, removed, changed, and unchanged lines between two synced releases.
|
|
135
|
+
- `export-ndjson` also supports `compare-line` and `compare-release` entities for agent ingest.
|
|
136
|
+
- local db path: `data/janelia_splitgal4.sqlite`
|
|
137
|
+
- raw manifest cache: `data/raw_manifests/*.json`
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
LICENSE
|
|
2
|
+
README.md
|
|
3
|
+
janelia_splitgal4.py
|
|
4
|
+
pyproject.toml
|
|
5
|
+
flylight.egg-info/PKG-INFO
|
|
6
|
+
flylight.egg-info/SOURCES.txt
|
|
7
|
+
flylight.egg-info/dependency_links.txt
|
|
8
|
+
flylight.egg-info/entry_points.txt
|
|
9
|
+
flylight.egg-info/top_level.txt
|
|
10
|
+
flylight_cli/__init__.py
|
|
11
|
+
flylight_cli/__main__.py
|
|
12
|
+
flylight_cli/cache.py
|
|
13
|
+
flylight_cli/cli.py
|
|
14
|
+
flylight_cli/core.py
|
|
15
|
+
flylight_cli/db.py
|
|
16
|
+
flylight_cli/examples.py
|
|
17
|
+
flylight_cli/normalize.py
|
|
18
|
+
flylight_cli/query.py
|
|
19
|
+
flylight_cli/records.py
|
|
20
|
+
flylight_cli/reindex.py
|
|
21
|
+
flylight_cli/schema.py
|
|
22
|
+
flylight_cli/snapshot.py
|
|
23
|
+
flylight_cli/sync_plan.py
|
|
24
|
+
tests/test_cli.py
|
|
25
|
+
tests/test_sync_plan.py
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
|
|
@@ -0,0 +1,180 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import hashlib
|
|
4
|
+
import json
|
|
5
|
+
import os
|
|
6
|
+
import tempfile
|
|
7
|
+
from dataclasses import dataclass, replace
|
|
8
|
+
from datetime import datetime, timezone
|
|
9
|
+
from pathlib import Path
|
|
10
|
+
from urllib.parse import urlparse
|
|
11
|
+
from urllib.request import Request, urlopen
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
DEFAULT_CACHE_DIR = Path("data/http_cache")
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
@dataclass
|
|
18
|
+
class CacheOptions:
|
|
19
|
+
cache_dir: Path = DEFAULT_CACHE_DIR
|
|
20
|
+
offline: bool = False
|
|
21
|
+
refresh: bool = False
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
class OfflineCacheMiss(RuntimeError):
|
|
25
|
+
pass
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
_cache_options = CacheOptions()
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
def get_cache_options() -> CacheOptions:
|
|
32
|
+
return _cache_options
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
def set_cache_options(
|
|
36
|
+
cache_dir: Path | None = None,
|
|
37
|
+
offline: bool | None = None,
|
|
38
|
+
refresh: bool | None = None,
|
|
39
|
+
) -> CacheOptions:
|
|
40
|
+
global _cache_options
|
|
41
|
+
next_options = replace(_cache_options)
|
|
42
|
+
if cache_dir is not None:
|
|
43
|
+
next_options.cache_dir = cache_dir
|
|
44
|
+
if offline is not None:
|
|
45
|
+
next_options.offline = offline
|
|
46
|
+
if refresh is not None:
|
|
47
|
+
next_options.refresh = refresh
|
|
48
|
+
if next_options.offline and next_options.refresh:
|
|
49
|
+
raise ValueError("cannot combine offline mode with refresh-cache")
|
|
50
|
+
_cache_options = next_options
|
|
51
|
+
return _cache_options
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
def cache_path_for_url(url: str, cache_dir: Path | None = None) -> Path:
|
|
55
|
+
cache_dir = cache_dir or _cache_options.cache_dir
|
|
56
|
+
digest = hashlib.sha256(url.encode("utf-8")).hexdigest()
|
|
57
|
+
suffix = Path(urlparse(url).path).suffix or ".bin"
|
|
58
|
+
return cache_dir / digest[:2] / f"{digest}{suffix}"
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
def meta_path_for_cache(cache_path: Path) -> Path:
|
|
62
|
+
return cache_path.with_suffix(cache_path.suffix + ".meta.json")
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
def ensure_cache_parent(path: Path) -> None:
|
|
66
|
+
path.parent.mkdir(parents=True, exist_ok=True)
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
def load_cached_bytes(url: str, cache_dir: Path | None = None) -> bytes | None:
|
|
70
|
+
cache_path = cache_path_for_url(url, cache_dir=cache_dir)
|
|
71
|
+
if not cache_path.exists():
|
|
72
|
+
return None
|
|
73
|
+
return cache_path.read_bytes()
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
def cached_at_for_path(cache_path: Path) -> str | None:
|
|
77
|
+
meta_path = meta_path_for_cache(cache_path)
|
|
78
|
+
if not meta_path.exists():
|
|
79
|
+
return None
|
|
80
|
+
meta = json.loads(meta_path.read_text(encoding="utf-8"))
|
|
81
|
+
cached_at = meta.get("cached_at")
|
|
82
|
+
if isinstance(cached_at, str) and cached_at:
|
|
83
|
+
return cached_at
|
|
84
|
+
return None
|
|
85
|
+
|
|
86
|
+
|
|
87
|
+
def cache_data_paths(cache_dir: Path) -> list[Path]:
|
|
88
|
+
return [path for path in cache_dir.rglob("*") if path.is_file() and not path.name.endswith(".meta.json")]
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
def cache_entry_for_url(url: str, cache_dir: Path | None = None) -> dict[str, int | str] | None:
|
|
92
|
+
cache_path = cache_path_for_url(url, cache_dir=cache_dir)
|
|
93
|
+
if not cache_path.exists():
|
|
94
|
+
return None
|
|
95
|
+
payload: dict[str, int | str] = {
|
|
96
|
+
"url": url,
|
|
97
|
+
"cache_path": str(cache_path),
|
|
98
|
+
"bytes": cache_path.stat().st_size,
|
|
99
|
+
"suffix": cache_path.suffix,
|
|
100
|
+
}
|
|
101
|
+
cached_at = cached_at_for_path(cache_path)
|
|
102
|
+
if cached_at:
|
|
103
|
+
payload["cached_at"] = cached_at
|
|
104
|
+
return payload
|
|
105
|
+
|
|
106
|
+
|
|
107
|
+
def write_atomic_bytes(path: Path, payload: bytes) -> None:
|
|
108
|
+
ensure_cache_parent(path)
|
|
109
|
+
fd, tmp_name = tempfile.mkstemp(
|
|
110
|
+
dir=path.parent,
|
|
111
|
+
prefix=f"{path.name}.",
|
|
112
|
+
suffix=".tmp",
|
|
113
|
+
)
|
|
114
|
+
try:
|
|
115
|
+
with os.fdopen(fd, "wb") as handle:
|
|
116
|
+
handle.write(payload)
|
|
117
|
+
Path(tmp_name).replace(path)
|
|
118
|
+
except Exception:
|
|
119
|
+
Path(tmp_name).unlink(missing_ok=True)
|
|
120
|
+
raise
|
|
121
|
+
|
|
122
|
+
|
|
123
|
+
def write_cached_bytes(url: str, payload: bytes, cache_dir: Path | None = None) -> Path:
|
|
124
|
+
cache_path = cache_path_for_url(url, cache_dir=cache_dir)
|
|
125
|
+
write_atomic_bytes(cache_path, payload)
|
|
126
|
+
meta_payload = json.dumps(
|
|
127
|
+
{
|
|
128
|
+
"url": url,
|
|
129
|
+
"cached_at": datetime.now(timezone.utc).replace(microsecond=0).isoformat(),
|
|
130
|
+
"bytes": len(payload),
|
|
131
|
+
},
|
|
132
|
+
indent=2,
|
|
133
|
+
sort_keys=True,
|
|
134
|
+
)
|
|
135
|
+
write_atomic_bytes(meta_path_for_cache(cache_path), meta_payload.encode("utf-8"))
|
|
136
|
+
return cache_path
|
|
137
|
+
|
|
138
|
+
|
|
139
|
+
def fetch_bytes(url: str, user_agent: str) -> bytes:
|
|
140
|
+
options = get_cache_options()
|
|
141
|
+
cached = None if options.refresh else load_cached_bytes(url, cache_dir=options.cache_dir)
|
|
142
|
+
if cached is not None:
|
|
143
|
+
return cached
|
|
144
|
+
if options.offline:
|
|
145
|
+
raise OfflineCacheMiss(f"offline cache miss for {url}; rerun without --offline to warm the cache")
|
|
146
|
+
req = Request(url, headers={"User-Agent": user_agent})
|
|
147
|
+
with urlopen(req) as resp:
|
|
148
|
+
payload = resp.read()
|
|
149
|
+
write_cached_bytes(url, payload, cache_dir=options.cache_dir)
|
|
150
|
+
return payload
|
|
151
|
+
|
|
152
|
+
|
|
153
|
+
def cache_stats(cache_dir: Path | None = None) -> dict[str, int | str]:
|
|
154
|
+
cache_dir = cache_dir or _cache_options.cache_dir
|
|
155
|
+
if not cache_dir.exists():
|
|
156
|
+
return {
|
|
157
|
+
"cache_dir": str(cache_dir),
|
|
158
|
+
"entries": 0,
|
|
159
|
+
"bytes": 0,
|
|
160
|
+
"suffix_counts": {},
|
|
161
|
+
"oldest_cached_at": None,
|
|
162
|
+
"newest_cached_at": None,
|
|
163
|
+
}
|
|
164
|
+
paths = cache_data_paths(cache_dir)
|
|
165
|
+
suffix_counts: dict[str, int] = {}
|
|
166
|
+
cached_at_values: list[str] = []
|
|
167
|
+
for path in paths:
|
|
168
|
+
suffix = path.suffix or ".bin"
|
|
169
|
+
suffix_counts[suffix] = suffix_counts.get(suffix, 0) + 1
|
|
170
|
+
cached_at = cached_at_for_path(path)
|
|
171
|
+
if cached_at:
|
|
172
|
+
cached_at_values.append(cached_at)
|
|
173
|
+
return {
|
|
174
|
+
"cache_dir": str(cache_dir),
|
|
175
|
+
"entries": len(paths),
|
|
176
|
+
"bytes": sum(path.stat().st_size for path in paths),
|
|
177
|
+
"suffix_counts": suffix_counts,
|
|
178
|
+
"oldest_cached_at": min(cached_at_values) if cached_at_values else None,
|
|
179
|
+
"newest_cached_at": max(cached_at_values) if cached_at_values else None,
|
|
180
|
+
}
|