nmcp-precomputed 3.0.7__tar.gz → 3.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {nmcp_precomputed-3.0.7 → nmcp_precomputed-3.1.0}/.github/workflows/publish.yml +2 -4
- nmcp_precomputed-3.1.0/Dockerfile +14 -0
- {nmcp_precomputed-3.0.7 → nmcp_precomputed-3.1.0}/PKG-INFO +7 -3
- nmcp_precomputed-3.1.0/README.md +150 -0
- {nmcp_precomputed-3.0.7 → nmcp_precomputed-3.1.0}/Taskfile.dist.yaml +6 -2
- {nmcp_precomputed-3.0.7 → nmcp_precomputed-3.1.0}/Test.Dockerfile +1 -1
- {nmcp_precomputed-3.0.7 → nmcp_precomputed-3.1.0}/pyproject.toml +8 -3
- {nmcp_precomputed-3.0.7 → nmcp_precomputed-3.1.0}/src/nmcp/__init__.py +2 -1
- {nmcp_precomputed-3.0.7 → nmcp_precomputed-3.1.0}/src/nmcp/__main__.py +3 -3
- nmcp_precomputed-3.1.0/src/nmcp/api/__init__.py +1 -0
- nmcp_precomputed-3.1.0/src/nmcp/api/app.py +90 -0
- nmcp_precomputed-3.1.0/src/nmcp/api/models.py +68 -0
- nmcp_precomputed-3.1.0/src/nmcp/api/output_location.py +80 -0
- {nmcp_precomputed-3.0.7 → nmcp_precomputed-3.1.0}/src/nmcp/data/remote_data_client.py +24 -0
- {nmcp_precomputed-3.0.7 → nmcp_precomputed-3.1.0}/src/nmcp/precomputed/__init__.py +3 -2
- {nmcp_precomputed-3.0.7 → nmcp_precomputed-3.1.0}/src/nmcp/precomputed/nmcp_precomputed.py +56 -51
- {nmcp_precomputed-3.0.7 → nmcp_precomputed-3.1.0}/src/nmcp/precomputed/segment_tag_property.py +14 -2
- {nmcp_precomputed-3.0.7 → nmcp_precomputed-3.1.0}/src/nmcp/precomputed_worker.py +50 -11
- nmcp_precomputed-3.1.0/src/nmcp/service.py +33 -0
- nmcp_precomputed-3.1.0/uv.lock +3624 -0
- nmcp_precomputed-3.0.7/Dockerfile +0 -9
- nmcp_precomputed-3.0.7/README.md +0 -70
- nmcp_precomputed-3.0.7/version.json +0 -4
- {nmcp_precomputed-3.0.7 → nmcp_precomputed-3.1.0}/.gitattributes +0 -0
- {nmcp_precomputed-3.0.7 → nmcp_precomputed-3.1.0}/.gitignore +0 -0
- {nmcp_precomputed-3.0.7 → nmcp_precomputed-3.1.0}/dev.sh +0 -0
- {nmcp_precomputed-3.0.7 → nmcp_precomputed-3.1.0}/docker-compose.yml +0 -0
- {nmcp_precomputed-3.0.7 → nmcp_precomputed-3.1.0}/docker-entry.sh +0 -0
- {nmcp_precomputed-3.0.7 → nmcp_precomputed-3.1.0}/src/nmcp/data/__init__.py +0 -0
- {nmcp_precomputed-3.0.7 → nmcp_precomputed-3.1.0}/src/nmcp/data/gql/__init__.py +0 -0
- {nmcp_precomputed-3.0.7 → nmcp_precomputed-3.1.0}/src/nmcp/data/gql/precomputed_queries.gql +0 -0
- {nmcp_precomputed-3.0.7 → nmcp_precomputed-3.1.0}/src/nmcp/data/portal_reconstruction.py +0 -0
- {nmcp_precomputed-3.0.7 → nmcp_precomputed-3.1.0}/src/nmcp/data/precomputed_entry.py +0 -0
- {nmcp_precomputed-3.0.7 → nmcp_precomputed-3.1.0}/src/nmcp/export/__init__.py +0 -0
- {nmcp_precomputed-3.0.7 → nmcp_precomputed-3.1.0}/src/nmcp/export/export_format.py +0 -0
- {nmcp_precomputed-3.0.7 → nmcp_precomputed-3.1.0}/src/nmcp/export/reconstruction_space.py +0 -0
- {nmcp_precomputed-3.0.7 → nmcp_precomputed-3.1.0}/src/nmcp/from_portal_json.py +0 -0
- {nmcp_precomputed-3.0.7 → nmcp_precomputed-3.1.0}/src/nmcp/from_service.py +0 -0
- {nmcp_precomputed-3.0.7 → nmcp_precomputed-3.1.0}/src/nmcp/list_skeletons.py +0 -0
- {nmcp_precomputed-3.0.7 → nmcp_precomputed-3.1.0}/src/nmcp/precomputed/neuron_structure.py +0 -0
- {nmcp_precomputed-3.0.7 → nmcp_precomputed-3.1.0}/src/nmcp/precomputed/nmcp_skeleton.py +0 -0
- {nmcp_precomputed-3.0.7 → nmcp_precomputed-3.1.0}/src/nmcp/precomputed/segment_info.py +0 -0
- {nmcp_precomputed-3.0.7 → nmcp_precomputed-3.1.0}/src/nmcp/precomputed/segment_property.py +0 -0
- {nmcp_precomputed-3.0.7 → nmcp_precomputed-3.1.0}/src/nmcp/remove_skeleton.py +0 -0
|
@@ -37,10 +37,8 @@ jobs:
|
|
|
37
37
|
with:
|
|
38
38
|
version: 3.x
|
|
39
39
|
repo-token: ${{ secrets.GITHUB_TOKEN }}
|
|
40
|
-
- name:
|
|
41
|
-
uses:
|
|
42
|
-
with:
|
|
43
|
-
node-version: "18.x"
|
|
40
|
+
- name: Install uv
|
|
41
|
+
uses: astral-sh/setup-uv@v6
|
|
44
42
|
- name: Login to Github Packages
|
|
45
43
|
uses: docker/login-action@v2
|
|
46
44
|
with:
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
FROM python:3.10
|
|
2
|
+
|
|
3
|
+
# Supplied by the Taskfile from pyproject.toml; selects the wheel `uv build` wrote for this version.
|
|
4
|
+
ARG VERSION
|
|
5
|
+
|
|
6
|
+
WORKDIR /app
|
|
7
|
+
|
|
8
|
+
COPY --chmod=755 docker-entry.sh ./
|
|
9
|
+
|
|
10
|
+
COPY dist/nmcp_precomputed-${VERSION}-py3-none-any.whl ./
|
|
11
|
+
|
|
12
|
+
RUN pip install --no-cache-dir "./nmcp_precomputed-${VERSION}-py3-none-any.whl[numerics,api]"
|
|
13
|
+
|
|
14
|
+
CMD ["./docker-entry.sh"]
|
|
@@ -1,18 +1,22 @@
|
|
|
1
|
-
Metadata-Version: 2.
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
2
|
Name: nmcp-precomputed
|
|
3
|
-
Version: 3.0
|
|
3
|
+
Version: 3.1.0
|
|
4
4
|
Summary: Neuron Morphology Community Portal Precomputed Toolbox
|
|
5
5
|
License: MIT
|
|
6
6
|
Requires-Python: <3.13,>=3.10
|
|
7
7
|
Requires-Dist: gql[requests]
|
|
8
|
+
Provides-Extra: api
|
|
9
|
+
Requires-Dist: fastapi>=0.162.2; extra == 'api'
|
|
10
|
+
Requires-Dist: uvicorn>=0.54.0; extra == 'api'
|
|
8
11
|
Provides-Extra: numerics
|
|
9
12
|
Requires-Dist: allensdk; extra == 'numerics'
|
|
10
13
|
Requires-Dist: cloud-files; extra == 'numerics'
|
|
11
14
|
Requires-Dist: cloud-volume; extra == 'numerics'
|
|
12
|
-
Requires-Dist: numpy<1.
|
|
15
|
+
Requires-Dist: numpy<2,>=1.23.5; extra == 'numerics'
|
|
13
16
|
Requires-Dist: pandas; extra == 'numerics'
|
|
14
17
|
Provides-Extra: publish
|
|
15
18
|
Requires-Dist: build; extra == 'publish'
|
|
16
19
|
Requires-Dist: twine; extra == 'publish'
|
|
17
20
|
Provides-Extra: test
|
|
21
|
+
Requires-Dist: httpx>=0.28.1; extra == 'test'
|
|
18
22
|
Requires-Dist: pytest==8.2.0; extra == 'test'
|
|
@@ -0,0 +1,150 @@
|
|
|
1
|
+
# NMCP Precomputed
|
|
2
|
+
The NMCP Precomputed service generates reconstruction skeletons in Neuroglancer's precomputed format. It periodically
|
|
3
|
+
polls the API service for reconstructions that are ready for new or updated skeletons.
|
|
4
|
+
|
|
5
|
+
Given the current behavior or Neuroglancer layers and the NMCP viewer feature requirements, the service currently
|
|
6
|
+
generated separate data sources for the full reconstruction, as well as axon-only and dendrite-only versions.
|
|
7
|
+
|
|
8
|
+
The service uses the chunked API for acquiring reconstruction data and should be compatible with dense reconstruction
|
|
9
|
+
sources.
|
|
10
|
+
|
|
11
|
+
## Local Development
|
|
12
|
+
A local S3-compatible service may be used during local development to avoid AWS or other cloud service storage charges.
|
|
13
|
+
|
|
14
|
+
_Note that at this time, cloud-files only appears to support this option from Linux, WSL, or MacOS._
|
|
15
|
+
|
|
16
|
+
One functional option is to run a local [MinIO](https://www.min.io/) Docker container. The following steps describe the
|
|
17
|
+
process (and assume the default configuration of port 9000 for MinIO). For additional details see either the MinIO
|
|
18
|
+
documentation, [cloud-files](https://github.com/seung-lab/cloud-files), and
|
|
19
|
+
[cloud-volume](https://github.com/seung-lab/cloud-volume).
|
|
20
|
+
|
|
21
|
+
A preconfigured instance is defined in `docker-compose.yml` that can be started with the following script
|
|
22
|
+
|
|
23
|
+
```bash
|
|
24
|
+
docker compose -p nmcp up -d
|
|
25
|
+
```
|
|
26
|
+
|
|
27
|
+
Attach to the running container to configure the precomputed bucket.
|
|
28
|
+
|
|
29
|
+
Find the container id
|
|
30
|
+
```bash
|
|
31
|
+
docker ps
|
|
32
|
+
```
|
|
33
|
+
|
|
34
|
+
Attach to the instance
|
|
35
|
+
```bash
|
|
36
|
+
docker exec -it <container-id> /bin/bash
|
|
37
|
+
```
|
|
38
|
+
|
|
39
|
+
Define an `mc` alias for the server, create the bucket, and allow public access for the Neuroglancer viewer. The
|
|
40
|
+
following assumes the username/password defined in `docker-compose.yml`. This is performed in the container after
|
|
41
|
+
attaching above, not on the host (unless you choose to install the `mc` tools on your host machine).
|
|
42
|
+
```bash
|
|
43
|
+
mc alias set myminio http://localhost:9000 root password
|
|
44
|
+
mc mb myminio/aind-neuron-morphology-community-portal-local/ngv01/
|
|
45
|
+
mc anonymous set public myminio/aind-neuron-morphology-community-portal-local/ngv01
|
|
46
|
+
```
|
|
47
|
+
|
|
48
|
+
Exit the container instance. On the host (in the Python environment created for this project w/cloud-files installed),
|
|
49
|
+
create an alias for the server.
|
|
50
|
+
|
|
51
|
+
```bash
|
|
52
|
+
cloudfiles alias add minio s3://http://127.0.0.1:9000/
|
|
53
|
+
```
|
|
54
|
+
|
|
55
|
+
Add the username and password as a secrets file in `~/.cloudvolume/secrets/minio-secret.json` (assumes defaults used
|
|
56
|
+
in the compose file).
|
|
57
|
+
```json
|
|
58
|
+
{
|
|
59
|
+
"AWS_ACCESS_KEY_ID": "root",
|
|
60
|
+
"AWS_SECRET_ACCESS_KEY": "password"
|
|
61
|
+
}
|
|
62
|
+
```
|
|
63
|
+
|
|
64
|
+
Although the bucket is set for public access in this development example, adding the authentication when generating
|
|
65
|
+
the precomputed data mimics the typical behavior in actual deployments.
|
|
66
|
+
|
|
67
|
+
When running the precomputed worker, pass the alias as the precomputed output argument, _e.g._, `-o minio://aind-neuron-morphology-community-portal-local/ngv01`
|
|
68
|
+
|
|
69
|
+
Here the current standard path is used as the base location (`ngv01`), however anything can used so long as it is also
|
|
70
|
+
used in the `NMCP_PRECOMPUTED_OUTPUT` environment variable for `nmcp-client`.
|
|
71
|
+
|
|
72
|
+
## On-Demand API
|
|
73
|
+
In addition to polling, the service exposes an HTTP API that converts a posted portal-format reconstruction into a
|
|
74
|
+
complete, self-contained precomputed dataset (one Neuroglancer layer holding a single skeleton) at a location chosen by
|
|
75
|
+
the caller. It uses the same conversion as the worker, so the skeleton output is identical.
|
|
76
|
+
|
|
77
|
+
The API runs in the same process as the polling worker. API requests run concurrently with worker generations and never
|
|
78
|
+
wait for one to finish; they are serialized only with other API requests.
|
|
79
|
+
|
|
80
|
+
### Startup
|
|
81
|
+
`python -m nmcp` (and therefore `docker-entry.sh`) starts both. Before either runs, it:
|
|
82
|
+
|
|
83
|
+
1. validates and normalizes the worker output location (`-o`, required);
|
|
84
|
+
2. loads the Allen structure tree used for soma-location tags.
|
|
85
|
+
|
|
86
|
+
If either step fails, the process exits so the container restarts it. After the first successful load, the tree is
|
|
87
|
+
read from allensdk's `mouse_connectivity/` cache, so later restarts do not need network access.
|
|
88
|
+
|
|
89
|
+
The worker output location is normalized at startup: protocol-only (`s3://`, `gs://`, `file://` or a configured
|
|
90
|
+
cloud-files alias), with no trailing slash and no `precomputed://` prefix or `|precomputed:` suffix. The worker writes
|
|
91
|
+
under the normalized location, and a warning is logged when the configured value differed from it. Set
|
|
92
|
+
`NMCP_PRECOMPUTED_OUTPUT` in normalized form, because `nmcp-client` composes its own dataset paths (`<value>/full`, …)
|
|
93
|
+
from the same value and must read the same datasets the worker writes.
|
|
94
|
+
|
|
95
|
+
| Environment variable | Default | Purpose |
|
|
96
|
+
|---|---|---|
|
|
97
|
+
| `NMCP_PRECOMPUTED_API_HOST` | `0.0.0.0` | Interface the API listens on |
|
|
98
|
+
| `NMCP_PRECOMPUTED_API_PORT` | `5000` | Port the API listens on |
|
|
99
|
+
|
|
100
|
+
### Endpoints
|
|
101
|
+
`GET /` returns `"NMCP Precomputed API"`.
|
|
102
|
+
|
|
103
|
+
`POST /generateskeleton` takes the output location, an optional segment id (default `1`, range `1` to `2^64 - 1`) and
|
|
104
|
+
the reconstruction in the portal export format:
|
|
105
|
+
|
|
106
|
+
```json
|
|
107
|
+
{
|
|
108
|
+
"location": "s3://bucket/path/to/dataset",
|
|
109
|
+
"segmentId": 1,
|
|
110
|
+
"reconstruction": {
|
|
111
|
+
"id": "…",
|
|
112
|
+
"neuron": {"label": "N004", "specimen": {"label": "648434", "genotype": "Dbh-Cre_KI"}},
|
|
113
|
+
"nodes": [{"index": 1, "structure": 1, "x": 0.0, "y": 0.0, "z": 0.0, "radius": 1.0, "parentIndex": -1, "atlasStructure": 202}]
|
|
114
|
+
}
|
|
115
|
+
}
|
|
116
|
+
```
|
|
117
|
+
|
|
118
|
+
It returns once the dataset is written:
|
|
119
|
+
|
|
120
|
+
```json
|
|
121
|
+
{
|
|
122
|
+
"serviceVersion": "3.0.7",
|
|
123
|
+
"reconstructionId": "…",
|
|
124
|
+
"result": {"location": "s3://bucket/path/to/dataset", "segmentId": 1, "nodeCount": 9, "edgeCount": 8},
|
|
125
|
+
"errorKind": null, "errorDescription": null, "errorInfo": null
|
|
126
|
+
}
|
|
127
|
+
```
|
|
128
|
+
|
|
129
|
+
Every request the service handles returns 200. On failure `result` is null and `errorKind` is one of:
|
|
130
|
+
|
|
131
|
+
- `InvalidLocation`: the location cannot be parsed, has no explicit protocol, or names a format other than
|
|
132
|
+
precomputed.
|
|
133
|
+
- `ProtectedLocation`: the location is the worker's output location or lies under it. Locations are compared after
|
|
134
|
+
resolving cloud-files aliases, trailing slashes, the precomputed prefix and suffix, and local symlinks.
|
|
135
|
+
- `GenerationError`: writing the dataset failed; `errorInfo` holds the exception type and message.
|
|
136
|
+
|
|
137
|
+
A body that fails validation (a missing field, a segment id of 0) gets FastAPI's standard 422 response.
|
|
138
|
+
|
|
139
|
+
The dataset is written over whatever already exists at the location without clearing it first. Skeleton files from
|
|
140
|
+
earlier calls with other segment ids remain, though the segment properties list only the latest segment.
|
|
141
|
+
|
|
142
|
+
### Running Locally
|
|
143
|
+
```bash
|
|
144
|
+
uv sync --extra numerics --extra api --extra test
|
|
145
|
+
uv run python -m nmcp -u <graphql-url> -a <key> -o <worker-output>
|
|
146
|
+
```
|
|
147
|
+
|
|
148
|
+
```bash
|
|
149
|
+
curl -s -X POST localhost:5000/generateskeleton -H "Content-Type: application/json" -d @body.json
|
|
150
|
+
```
|
|
@@ -5,7 +5,7 @@ vars:
|
|
|
5
5
|
IMAGE: allenneuraldynamics/nmcp-precomputed-worker
|
|
6
6
|
REPOIMAGE: "{{.REPOSITORY}}/{{.IMAGE}}"
|
|
7
7
|
VERSION:
|
|
8
|
-
sh:
|
|
8
|
+
sh: uv version --short
|
|
9
9
|
MAJOR:
|
|
10
10
|
sh: string={{.VERSION}} && arr=(${string//"."/ }) && echo ${arr[0]}
|
|
11
11
|
MINOR:
|
|
@@ -18,9 +18,13 @@ vars:
|
|
|
18
18
|
IMAGELATEST: "{{.REPOIMAGE}}:latest"
|
|
19
19
|
|
|
20
20
|
tasks:
|
|
21
|
+
module:
|
|
22
|
+
cmds:
|
|
23
|
+
- uv build --wheel
|
|
21
24
|
build:
|
|
22
25
|
cmds:
|
|
23
|
-
-
|
|
26
|
+
- task: module
|
|
27
|
+
- docker build --platform linux/amd64 --build-arg VERSION={{.VERSION}} --tag {{.IMAGEWITHVERSION}} .
|
|
24
28
|
- docker tag {{.IMAGEWITHVERSION}} {{.IMAGEWITHMINORVERSION}}
|
|
25
29
|
- docker tag {{.IMAGEWITHMINORVERSION}} {{.IMAGEWITHREVERSION}}
|
|
26
30
|
- docker tag {{.IMAGEWITHREVERSION}} {{.IMAGELATEST}}
|
|
@@ -12,7 +12,7 @@ exclude = [
|
|
|
12
12
|
|
|
13
13
|
[project]
|
|
14
14
|
name = "nmcp-precomputed"
|
|
15
|
-
version = "3.0
|
|
15
|
+
version = "3.1.0"
|
|
16
16
|
description = "Neuron Morphology Community Portal Precomputed Toolbox"
|
|
17
17
|
license = { text = "MIT" }
|
|
18
18
|
requires-python = ">=3.10,<3.13"
|
|
@@ -22,7 +22,7 @@ dependencies = [
|
|
|
22
22
|
|
|
23
23
|
[project.optional-dependencies]
|
|
24
24
|
numerics = [
|
|
25
|
-
"numpy
|
|
25
|
+
"numpy>=1.23.5,<2",
|
|
26
26
|
"pandas",
|
|
27
27
|
"allensdk",
|
|
28
28
|
"cloud-volume",
|
|
@@ -30,13 +30,18 @@ numerics = [
|
|
|
30
30
|
]
|
|
31
31
|
|
|
32
32
|
test = [
|
|
33
|
-
"
|
|
33
|
+
"httpx>=0.28.1",
|
|
34
|
+
"pytest==8.2.0",
|
|
34
35
|
]
|
|
35
36
|
|
|
36
37
|
publish = [
|
|
37
38
|
"build",
|
|
38
39
|
"twine"
|
|
39
40
|
]
|
|
41
|
+
api = [
|
|
42
|
+
"fastapi>=0.162.2",
|
|
43
|
+
"uvicorn>=0.54.0",
|
|
44
|
+
]
|
|
40
45
|
|
|
41
46
|
[tool.pytest.ini_options]
|
|
42
47
|
addopts = ["-W ignore::DeprecationWarning"]
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
from .export import ExportFormat, ReconstructionSpace
|
|
2
2
|
from .data import RemoteDataClient, PrecomputedEntry
|
|
3
3
|
from .precomputed import SegmentInfo, SegmentProperty, SegmentTagProperty, SomaSegmentTagProperty, SegmentPropertyValues
|
|
4
|
-
from .precomputed import (ensure_bucket_folders, create_from_json_files, create_from_reconstruction,
|
|
4
|
+
from .precomputed import (ensure_bucket_folders, create_from_json_files, create_from_reconstruction,
|
|
5
|
+
create_standalone_from_reconstruction, default_cloudvolume_info, remove_skeleton,
|
|
5
6
|
list_skeletons)
|
|
@@ -1,13 +1,13 @@
|
|
|
1
1
|
import argparse
|
|
2
2
|
|
|
3
|
-
from .
|
|
3
|
+
from .service import run
|
|
4
4
|
|
|
5
5
|
parser = argparse.ArgumentParser()
|
|
6
6
|
|
|
7
7
|
parser.add_argument("-u", "--url", help="URL of the GraphQL service")
|
|
8
8
|
parser.add_argument("-a", "--authkey", help="authorization header for GraphQL service")
|
|
9
|
-
parser.add_argument("-o", "--output", help="the output cloud volume location")
|
|
9
|
+
parser.add_argument("-o", "--output", required=True, help="the output cloud volume location")
|
|
10
10
|
|
|
11
11
|
args = parser.parse_args()
|
|
12
12
|
|
|
13
|
-
|
|
13
|
+
run(args.url, args.authkey, args.output)
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
from .app import create_app, serve
|
|
@@ -0,0 +1,90 @@
|
|
|
1
|
+
import importlib.metadata
|
|
2
|
+
import logging
|
|
3
|
+
import os
|
|
4
|
+
import threading
|
|
5
|
+
from typing import cast
|
|
6
|
+
|
|
7
|
+
import uvicorn
|
|
8
|
+
from fastapi import FastAPI
|
|
9
|
+
|
|
10
|
+
from .models import ErrorKind, SkeletonRequest, SkeletonResponse, SkeletonResult
|
|
11
|
+
from .output_location import parse_output_location
|
|
12
|
+
from ..data.portal_reconstruction import PortalReconstruction
|
|
13
|
+
from ..precomputed import create_standalone_from_reconstruction, default_cloudvolume_info
|
|
14
|
+
|
|
15
|
+
logger = logging.getLogger(__name__)
|
|
16
|
+
|
|
17
|
+
_SERVICE_NAME = "NMCP Precomputed API"
|
|
18
|
+
|
|
19
|
+
_SERVICE_VERSION = importlib.metadata.version("nmcp-precomputed")
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def create_app(worker_output: str) -> FastAPI:
|
|
23
|
+
protected = parse_output_location(worker_output, require_protocol=False)
|
|
24
|
+
|
|
25
|
+
generation_lock = threading.Lock()
|
|
26
|
+
|
|
27
|
+
app = FastAPI(title=_SERVICE_NAME)
|
|
28
|
+
|
|
29
|
+
@app.get("/")
|
|
30
|
+
def read_root() -> str:
|
|
31
|
+
return _SERVICE_NAME
|
|
32
|
+
|
|
33
|
+
# A plain `def`, so FastAPI runs the blocking generation in its threadpool rather than on the event loop.
|
|
34
|
+
@app.post("/generateskeleton")
|
|
35
|
+
def generate_skeleton(request: SkeletonRequest) -> SkeletonResponse:
|
|
36
|
+
reconstruction_id = request.reconstruction.id
|
|
37
|
+
|
|
38
|
+
def response(**kwargs) -> SkeletonResponse:
|
|
39
|
+
return SkeletonResponse(service_version=_SERVICE_VERSION, reconstruction_id=reconstruction_id, **kwargs)
|
|
40
|
+
|
|
41
|
+
logger.info(f"received on-demand skeleton request for reconstruction {reconstruction_id} "
|
|
42
|
+
f"({len(request.reconstruction.nodes)} nodes, segment {request.segment_id}) at "
|
|
43
|
+
f"'{request.location}'")
|
|
44
|
+
|
|
45
|
+
try:
|
|
46
|
+
location = parse_output_location(request.location)
|
|
47
|
+
except ValueError as ex:
|
|
48
|
+
logger.warning(f"rejected on-demand skeleton for reconstruction {reconstruction_id}: invalid output "
|
|
49
|
+
f"location '{request.location}': {ex}")
|
|
50
|
+
return response(error_kind=ErrorKind.INVALID_LOCATION,
|
|
51
|
+
error_description=f"Invalid output location for reconstruction {reconstruction_id}",
|
|
52
|
+
error_info=str(ex))
|
|
53
|
+
|
|
54
|
+
if location.is_at_or_under(protected):
|
|
55
|
+
logger.warning(f"rejected on-demand skeleton for reconstruction {reconstruction_id} at "
|
|
56
|
+
f"{location.cloudpath}: at or under the worker output {protected.cloudpath}")
|
|
57
|
+
return response(error_kind=ErrorKind.PROTECTED_LOCATION,
|
|
58
|
+
error_description=f"The output location for reconstruction {reconstruction_id} is at "
|
|
59
|
+
f"or under the worker's output location",
|
|
60
|
+
error_info=location.cloudpath)
|
|
61
|
+
|
|
62
|
+
# Chosen here so an optional request argument can replace it without touching the generation code.
|
|
63
|
+
dataset_info = default_cloudvolume_info()
|
|
64
|
+
|
|
65
|
+
reconstruction = cast(PortalReconstruction, request.reconstruction.model_dump(by_alias=True))
|
|
66
|
+
|
|
67
|
+
try:
|
|
68
|
+
with generation_lock:
|
|
69
|
+
skeleton = create_standalone_from_reconstruction(reconstruction, location.cloudpath,
|
|
70
|
+
request.segment_id, cloud_files_info=dataset_info)
|
|
71
|
+
except Exception as ex:
|
|
72
|
+
logger.exception(f"failed to generate on-demand skeleton for reconstruction {reconstruction_id} at "
|
|
73
|
+
f"{location.cloudpath}")
|
|
74
|
+
return response(error_kind=ErrorKind.GENERATION_ERROR,
|
|
75
|
+
error_description=f"Error generating precomputed skeleton for reconstruction "
|
|
76
|
+
f"{reconstruction_id}",
|
|
77
|
+
error_info=f"{type(ex).__name__}: {ex}")
|
|
78
|
+
|
|
79
|
+
logger.info(f"generated on-demand skeleton {request.segment_id} for reconstruction {reconstruction_id} at "
|
|
80
|
+
f"{location.cloudpath}")
|
|
81
|
+
|
|
82
|
+
return response(result=SkeletonResult(location=location.cloudpath, segment_id=request.segment_id,
|
|
83
|
+
node_count=len(skeleton.vertices), edge_count=len(skeleton.edges)))
|
|
84
|
+
|
|
85
|
+
return app
|
|
86
|
+
|
|
87
|
+
|
|
88
|
+
def serve(app: FastAPI) -> None:
|
|
89
|
+
uvicorn.run(app, host=os.environ.get("NMCP_PRECOMPUTED_API_HOST", "0.0.0.0"),
|
|
90
|
+
port=int(os.environ.get("NMCP_PRECOMPUTED_API_PORT", "5000")), access_log=False)
|
|
@@ -0,0 +1,68 @@
|
|
|
1
|
+
from enum import Enum
|
|
2
|
+
|
|
3
|
+
from pydantic import AliasGenerator, BaseModel, ConfigDict, Field
|
|
4
|
+
from pydantic.alias_generators import to_camel
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
class _RequestModel(BaseModel):
|
|
8
|
+
model_config = ConfigDict(alias_generator=to_camel)
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
class _ResponseModel(BaseModel):
|
|
12
|
+
model_config = ConfigDict(alias_generator=AliasGenerator(serialization_alias=to_camel))
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
class PortalSpecimenInput(_RequestModel):
|
|
16
|
+
label: str
|
|
17
|
+
genotype: str | None = None
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
class PortalNeuronInput(_RequestModel):
|
|
21
|
+
label: str
|
|
22
|
+
specimen: PortalSpecimenInput
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
class PortalNodeInput(_RequestModel):
|
|
26
|
+
index: int
|
|
27
|
+
structure: int
|
|
28
|
+
x: float
|
|
29
|
+
y: float
|
|
30
|
+
z: float
|
|
31
|
+
radius: float
|
|
32
|
+
parent_index: int
|
|
33
|
+
atlas_structure: int | None = None
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
class ReconstructionInput(_RequestModel):
|
|
37
|
+
id: str
|
|
38
|
+
neuron: PortalNeuronInput
|
|
39
|
+
nodes: list[PortalNodeInput]
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
class SkeletonRequest(_RequestModel):
|
|
43
|
+
location: str
|
|
44
|
+
# 0 is Neuroglancer's background segment, and segment ids are uint64.
|
|
45
|
+
segment_id: int = Field(default=1, ge=1, le=2 ** 64 - 1)
|
|
46
|
+
reconstruction: ReconstructionInput
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
class ErrorKind(str, Enum):
|
|
50
|
+
INVALID_LOCATION = "InvalidLocation"
|
|
51
|
+
PROTECTED_LOCATION = "ProtectedLocation"
|
|
52
|
+
GENERATION_ERROR = "GenerationError"
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
class SkeletonResult(_ResponseModel):
|
|
56
|
+
location: str
|
|
57
|
+
segment_id: int
|
|
58
|
+
node_count: int
|
|
59
|
+
edge_count: int
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
class SkeletonResponse(_ResponseModel):
|
|
63
|
+
service_version: str
|
|
64
|
+
reconstruction_id: str
|
|
65
|
+
result: SkeletonResult | None = None
|
|
66
|
+
error_kind: ErrorKind | None = None
|
|
67
|
+
error_description: str | None = None
|
|
68
|
+
error_info: str | None = None
|
|
@@ -0,0 +1,80 @@
|
|
|
1
|
+
import os
|
|
2
|
+
from dataclasses import dataclass
|
|
3
|
+
|
|
4
|
+
from cloudfiles import paths
|
|
5
|
+
|
|
6
|
+
_PRECOMPUTED_PREFIX = "precomputed://"
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
@dataclass(frozen=True)
|
|
10
|
+
class OutputLocation:
|
|
11
|
+
cloudpath: str
|
|
12
|
+
"""Protocol-only location to write to: no `precomputed://` prefix or `|precomputed:` suffix, no trailing slash."""
|
|
13
|
+
protocol: str
|
|
14
|
+
host: str | None
|
|
15
|
+
bucket: str | None
|
|
16
|
+
path: str
|
|
17
|
+
"""Comparison path: no trailing slash, and symlinks resolved for `file` locations."""
|
|
18
|
+
|
|
19
|
+
def is_at_or_under(self, other: "OutputLocation") -> bool:
|
|
20
|
+
if (self.protocol, self.host, self.bucket) != (other.protocol, other.host, other.bucket):
|
|
21
|
+
return False
|
|
22
|
+
|
|
23
|
+
if self.protocol == "file":
|
|
24
|
+
# Root-aware where a string-prefix test is not: a protected "/" covers every local path.
|
|
25
|
+
return os.path.commonpath([self.path, other.path]) == other.path
|
|
26
|
+
|
|
27
|
+
return other.path == "" or self.path == other.path or self.path.startswith(f"{other.path}/")
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
def parse_output_location(location: str, require_protocol: bool = True) -> OutputLocation:
|
|
31
|
+
"""
|
|
32
|
+
Parse a cloud-files location the way CloudFiles itself resolves it, so that comparisons see the destination that
|
|
33
|
+
will actually be written to rather than the caller's spelling. Raises `ValueError` for anything unusable.
|
|
34
|
+
"""
|
|
35
|
+
stripped = _remove_precomputed_prefix(location.strip())
|
|
36
|
+
|
|
37
|
+
if len(stripped) == 0:
|
|
38
|
+
raise ValueError("the output location is empty")
|
|
39
|
+
|
|
40
|
+
try:
|
|
41
|
+
fmt, protocol, _, _, _ = paths.extract_format_protocol(stripped, allow_defaults=False)
|
|
42
|
+
normalized = paths.normalize(stripped)
|
|
43
|
+
extracted = paths.extract(normalized)
|
|
44
|
+
except Exception as ex:
|
|
45
|
+
raise ValueError(f"unsupported location '{location}': {ex}") from ex
|
|
46
|
+
|
|
47
|
+
if fmt not in (None, "precomputed"):
|
|
48
|
+
raise ValueError(f"unsupported format '{fmt}' in location '{location}'; only precomputed is supported")
|
|
49
|
+
|
|
50
|
+
# cloud-files treats an unknown or mis-cased scheme, or a bare path, as a path relative to the working directory.
|
|
51
|
+
if require_protocol and protocol is None:
|
|
52
|
+
raise ValueError(f"the location '{location}' has no recognized protocol (e.g. s3://, gs://, file://)")
|
|
53
|
+
|
|
54
|
+
if extracted.protocol == "file":
|
|
55
|
+
comparison_path = os.path.realpath(extracted.path)
|
|
56
|
+
else:
|
|
57
|
+
comparison_path = extracted.path.rstrip("/")
|
|
58
|
+
|
|
59
|
+
return OutputLocation(_to_cloudpath(normalized), extracted.protocol, extracted.host, extracted.bucket,
|
|
60
|
+
comparison_path)
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
def _remove_precomputed_prefix(location: str) -> str:
|
|
64
|
+
if location.startswith(_PRECOMPUTED_PREFIX):
|
|
65
|
+
return location[len(_PRECOMPUTED_PREFIX):]
|
|
66
|
+
|
|
67
|
+
return location
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
def _to_cloudpath(normalized: str) -> str:
|
|
71
|
+
# `normalize` turns the `…|precomputed:` tail spelling into a `precomputed://` prefix, and the generation code
|
|
72
|
+
# adds its own, so it is removed here as well as from the raw input.
|
|
73
|
+
scheme, separator, rest = _remove_precomputed_prefix(normalized).partition("://")
|
|
74
|
+
|
|
75
|
+
trimmed = rest.rstrip("/")
|
|
76
|
+
|
|
77
|
+
if len(trimmed) == 0 and rest.startswith("/"):
|
|
78
|
+
trimmed = "/"
|
|
79
|
+
|
|
80
|
+
return f"{scheme}{separator}{trimmed}"
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import logging
|
|
2
2
|
from datetime import datetime
|
|
3
3
|
from enum import IntEnum
|
|
4
|
+
from urllib.parse import urlparse
|
|
4
5
|
|
|
5
6
|
from typing_extensions import List
|
|
6
7
|
|
|
@@ -23,8 +24,31 @@ class PrecomputedStatus(IntEnum):
|
|
|
23
24
|
FailedToGenerate = 400
|
|
24
25
|
|
|
25
26
|
|
|
27
|
+
class _GraphQLRetryFilter(logging.Filter):
|
|
28
|
+
"""
|
|
29
|
+
Drops urllib3's per-attempt "Retrying" warnings for GraphQL requests. The worker already logs one warning per
|
|
30
|
+
outage with the final reason, and every poll during an outage would otherwise add one line per retry. Requests to
|
|
31
|
+
other paths, such as cloud-files uploads, keep theirs.
|
|
32
|
+
"""
|
|
33
|
+
|
|
34
|
+
def __init__(self, path: str):
|
|
35
|
+
super().__init__()
|
|
36
|
+
self._path = path
|
|
37
|
+
|
|
38
|
+
def filter(self, record: logging.LogRecord) -> bool:
|
|
39
|
+
# urllib3 logs `"Retrying (%r) after connection broken by '%r': %s", retries, err, url`, where url is the
|
|
40
|
+
# request path.
|
|
41
|
+
is_retry = isinstance(record.msg, str) and record.msg.startswith("Retrying")
|
|
42
|
+
|
|
43
|
+
return not (is_retry and isinstance(record.args, tuple) and len(record.args) > 0 and
|
|
44
|
+
record.args[-1] == self._path)
|
|
45
|
+
|
|
46
|
+
|
|
26
47
|
class RemoteDataClient:
|
|
27
48
|
def __init__(self, url: str, auth_key: str):
|
|
49
|
+
# requests sends "/" for a URL with no path.
|
|
50
|
+
logging.getLogger("urllib3.connectionpool").addFilter(_GraphQLRetryFilter(urlparse(url).path or "/"))
|
|
51
|
+
|
|
28
52
|
transport = RequestsHTTPTransport(
|
|
29
53
|
url=url,
|
|
30
54
|
verify=True,
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
from .segment_property import SegmentProperty
|
|
2
|
-
from .segment_tag_property import SegmentTagProperty, SomaSegmentTagProperty
|
|
2
|
+
from .segment_tag_property import SegmentTagProperty, SomaSegmentTagProperty, load_structure_tree
|
|
3
3
|
from .segment_info import SegmentInfo, SegmentPropertyValues
|
|
4
4
|
from .neuron_structure import NeuronStructure
|
|
5
5
|
from .nmcp_precomputed import (ensure_bucket_folders, create_from_json_files, create_from_reconstruction,
|
|
6
|
-
remove_skeleton,
|
|
6
|
+
create_standalone_from_reconstruction, default_cloudvolume_info, remove_skeleton,
|
|
7
|
+
list_skeletons)
|